ppef 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +76 -125
- package/dist/__tests__/cli/evaluate-command.integration.test.d.ts +8 -0
- package/dist/__tests__/cli/evaluate-command.integration.test.d.ts.map +1 -0
- package/dist/__tests__/cli/evaluate-command.integration.test.js +308 -0
- package/dist/__tests__/cli/evaluate-command.integration.test.js.map +1 -0
- package/dist/__tests__/evaluators/claims-evaluator.unit.test.d.ts +8 -0
- package/dist/__tests__/evaluators/claims-evaluator.unit.test.d.ts.map +1 -0
- package/dist/__tests__/evaluators/claims-evaluator.unit.test.js +405 -0
- package/dist/__tests__/evaluators/claims-evaluator.unit.test.js.map +1 -0
- package/dist/__tests__/evaluators/metrics-evaluator.unit.test.d.ts +8 -0
- package/dist/__tests__/evaluators/metrics-evaluator.unit.test.d.ts.map +1 -0
- package/dist/__tests__/evaluators/metrics-evaluator.unit.test.js +424 -0
- package/dist/__tests__/evaluators/metrics-evaluator.unit.test.js.map +1 -0
- package/dist/__tests__/evaluators/registry.unit.test.d.ts +7 -0
- package/dist/__tests__/evaluators/registry.unit.test.d.ts.map +1 -0
- package/dist/__tests__/evaluators/registry.unit.test.js +173 -0
- package/dist/__tests__/evaluators/registry.unit.test.js.map +1 -0
- package/dist/__tests__/evaluators/robustness-evaluator.unit.test.d.ts +8 -0
- package/dist/__tests__/evaluators/robustness-evaluator.unit.test.d.ts.map +1 -0
- package/dist/__tests__/evaluators/robustness-evaluator.unit.test.js +260 -0
- package/dist/__tests__/evaluators/robustness-evaluator.unit.test.js.map +1 -0
- package/dist/__tests__/framework-pipeline.integration.test.js +36 -9
- package/dist/__tests__/framework-pipeline.integration.test.js.map +1 -1
- package/dist/__tests__/index-exports.unit.test.js +9 -12
- package/dist/__tests__/index-exports.unit.test.js.map +1 -1
- package/dist/aggregation/pipeline.d.ts.map +1 -1
- package/dist/aggregation/pipeline.js +40 -3
- package/dist/aggregation/pipeline.js.map +1 -1
- package/dist/claims/index.d.ts +6 -3
- package/dist/claims/index.d.ts.map +1 -1
- package/dist/claims/index.js +6 -3
- package/dist/claims/index.js.map +1 -1
- package/dist/cli/__tests__/aggregate.command.unit.test.js +3 -0
- package/dist/cli/__tests__/aggregate.command.unit.test.js.map +1 -1
- package/dist/cli/__tests__/binary-sut.integration.test.d.ts +8 -0
- package/dist/cli/__tests__/binary-sut.integration.test.d.ts.map +1 -0
- package/dist/cli/__tests__/binary-sut.integration.test.js +165 -0
- package/dist/cli/__tests__/binary-sut.integration.test.js.map +1 -0
- package/dist/cli/__tests__/config-loader.unit.test.d.ts +7 -0
- package/dist/cli/__tests__/config-loader.unit.test.d.ts.map +1 -0
- package/dist/cli/__tests__/config-loader.unit.test.js +611 -0
- package/dist/cli/__tests__/config-loader.unit.test.js.map +1 -0
- package/dist/cli/command-deps.d.ts +13 -1
- package/dist/cli/command-deps.d.ts.map +1 -1
- package/dist/cli/commands/aggregate.d.ts.map +1 -1
- package/dist/cli/commands/aggregate.js +3 -0
- package/dist/cli/commands/aggregate.js.map +1 -1
- package/dist/cli/commands/evaluate.d.ts +41 -0
- package/dist/cli/commands/evaluate.d.ts.map +1 -0
- package/dist/cli/commands/evaluate.js +287 -0
- package/dist/cli/commands/evaluate.js.map +1 -0
- package/dist/cli/commands/run.d.ts.map +1 -1
- package/dist/cli/commands/run.js +93 -1
- package/dist/cli/commands/run.js.map +1 -1
- package/dist/cli/index.d.ts +2 -1
- package/dist/cli/index.d.ts.map +1 -1
- package/dist/cli/index.js +3 -1
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/module-loader.d.ts +23 -1
- package/dist/cli/module-loader.d.ts.map +1 -1
- package/dist/cli/module-loader.js +19 -1
- package/dist/cli/module-loader.js.map +1 -1
- package/dist/cli/types.d.ts +19 -0
- package/dist/cli/types.d.ts.map +1 -1
- package/dist/evaluators/claims-evaluator.d.ts +87 -0
- package/dist/evaluators/claims-evaluator.d.ts.map +1 -0
- package/dist/evaluators/claims-evaluator.js +289 -0
- package/dist/evaluators/claims-evaluator.js.map +1 -0
- package/dist/evaluators/exploratory-evaluator.d.ts +136 -0
- package/dist/evaluators/exploratory-evaluator.d.ts.map +1 -0
- package/dist/evaluators/exploratory-evaluator.js +545 -0
- package/dist/evaluators/exploratory-evaluator.js.map +1 -0
- package/dist/evaluators/index.d.ts +13 -0
- package/dist/evaluators/index.d.ts.map +1 -0
- package/dist/evaluators/index.js +14 -0
- package/dist/evaluators/index.js.map +1 -0
- package/dist/evaluators/metrics-evaluator.d.ts +114 -0
- package/dist/evaluators/metrics-evaluator.d.ts.map +1 -0
- package/dist/evaluators/metrics-evaluator.js +433 -0
- package/dist/evaluators/metrics-evaluator.js.map +1 -0
- package/dist/evaluators/registry.d.ts +106 -0
- package/dist/evaluators/registry.d.ts.map +1 -0
- package/dist/evaluators/registry.js +148 -0
- package/dist/evaluators/registry.js.map +1 -0
- package/dist/evaluators/robustness-evaluator.d.ts +57 -0
- package/dist/evaluators/robustness-evaluator.d.ts.map +1 -0
- package/dist/evaluators/robustness-evaluator.js +186 -0
- package/dist/evaluators/robustness-evaluator.js.map +1 -0
- package/dist/executor/__tests__/binary-sut.unit.test.d.ts +8 -0
- package/dist/executor/__tests__/binary-sut.unit.test.d.ts.map +1 -0
- package/dist/executor/__tests__/binary-sut.unit.test.js +313 -0
- package/dist/executor/__tests__/binary-sut.unit.test.js.map +1 -0
- package/dist/executor/__tests__/checkpoint-storage.unit.test.js +43 -0
- package/dist/executor/__tests__/checkpoint-storage.unit.test.js.map +1 -1
- package/dist/executor/__tests__/executor.unit.test.js +56 -9
- package/dist/executor/__tests__/executor.unit.test.js.map +1 -1
- package/dist/executor/__tests__/resource-calculator.unit.test.d.ts +10 -0
- package/dist/executor/__tests__/resource-calculator.unit.test.d.ts.map +1 -0
- package/dist/executor/__tests__/resource-calculator.unit.test.js +104 -0
- package/dist/executor/__tests__/resource-calculator.unit.test.js.map +1 -0
- package/dist/executor/__tests__/worker-threads-executor.unit.test.d.ts +8 -0
- package/dist/executor/__tests__/worker-threads-executor.unit.test.d.ts.map +1 -0
- package/dist/executor/__tests__/worker-threads-executor.unit.test.js +276 -0
- package/dist/executor/__tests__/worker-threads-executor.unit.test.js.map +1 -0
- package/dist/executor/binary-sut.d.ts +105 -0
- package/dist/executor/binary-sut.d.ts.map +1 -0
- package/dist/executor/binary-sut.js +174 -0
- package/dist/executor/binary-sut.js.map +1 -0
- package/dist/executor/checkpoint-storage.d.ts.map +1 -1
- package/dist/executor/checkpoint-storage.js +6 -4
- package/dist/executor/checkpoint-storage.js.map +1 -1
- package/dist/executor/executor.d.ts +28 -0
- package/dist/executor/executor.d.ts.map +1 -1
- package/dist/executor/executor.js +85 -24
- package/dist/executor/executor.js.map +1 -1
- package/dist/executor/index.d.ts +4 -0
- package/dist/executor/index.d.ts.map +1 -1
- package/dist/executor/index.js +4 -0
- package/dist/executor/index.js.map +1 -1
- package/dist/executor/resource-calculator.d.ts +49 -0
- package/dist/executor/resource-calculator.d.ts.map +1 -0
- package/dist/executor/resource-calculator.js +129 -0
- package/dist/executor/resource-calculator.js.map +1 -0
- package/dist/executor/worker-entry.js +26 -10
- package/dist/executor/worker-entry.js.map +1 -1
- package/dist/executor/worker-executor.d.ts +104 -3
- package/dist/executor/worker-executor.d.ts.map +1 -1
- package/dist/executor/worker-executor.js +224 -4
- package/dist/executor/worker-executor.js.map +1 -1
- package/dist/executor/worker-threads-executor.d.ts +245 -0
- package/dist/executor/worker-threads-executor.d.ts.map +1 -0
- package/dist/executor/worker-threads-executor.js +332 -0
- package/dist/executor/worker-threads-executor.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -2
- package/dist/index.js.map +1 -1
- package/dist/renderers/latex-renderer.d.ts +60 -0
- package/dist/renderers/latex-renderer.d.ts.map +1 -1
- package/dist/renderers/latex-renderer.js +299 -0
- package/dist/renderers/latex-renderer.js.map +1 -1
- package/dist/renderers/types.d.ts +9 -0
- package/dist/renderers/types.d.ts.map +1 -1
- package/dist/renderers/types.js.map +1 -1
- package/dist/robustness/index.d.ts +5 -2
- package/dist/robustness/index.d.ts.map +1 -1
- package/dist/robustness/index.js +4 -2
- package/dist/robustness/index.js.map +1 -1
- package/dist/types/evaluator.d.ts +449 -0
- package/dist/types/evaluator.d.ts.map +1 -0
- package/dist/types/evaluator.js +9 -0
- package/dist/types/evaluator.js.map +1 -0
- package/dist/types/result.d.ts +2 -0
- package/dist/types/result.d.ts.map +1 -1
- package/package.json +1 -1
- package/dist/claims/__tests__/evaluator.unit.test.d.ts +0 -12
- package/dist/claims/__tests__/evaluator.unit.test.d.ts.map +0 -1
- package/dist/claims/__tests__/evaluator.unit.test.js +0 -801
- package/dist/claims/__tests__/evaluator.unit.test.js.map +0 -1
- package/dist/claims/evaluator.d.ts +0 -33
- package/dist/claims/evaluator.d.ts.map +0 -1
- package/dist/claims/evaluator.js +0 -174
- package/dist/claims/evaluator.js.map +0 -1
- package/dist/robustness/__tests__/analyzer.unit.test.d.ts +0 -11
- package/dist/robustness/__tests__/analyzer.unit.test.d.ts.map +0 -1
- package/dist/robustness/__tests__/analyzer.unit.test.js +0 -455
- package/dist/robustness/__tests__/analyzer.unit.test.js.map +0 -1
- package/dist/robustness/analyzer.d.ts +0 -61
- package/dist/robustness/analyzer.d.ts.map +0 -1
- package/dist/robustness/analyzer.js +0 -191
- package/dist/robustness/analyzer.js.map +0 -1
package/dist/cli/index.d.ts
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
*/
|
|
6
6
|
import { Command } from "commander";
|
|
7
7
|
import { registerAggregateCommand } from "./commands/aggregate.js";
|
|
8
|
+
import { registerEvaluateCommand } from "./commands/evaluate.js";
|
|
8
9
|
import { registerPlanCommand } from "./commands/plan.js";
|
|
9
10
|
import { registerRunCommand } from "./commands/run.js";
|
|
10
11
|
import { registerValidateCommand } from "./commands/validate.js";
|
|
@@ -22,5 +23,5 @@ export declare function createCliProgram(): Command;
|
|
|
22
23
|
* @returns Exit code (0 for success, 1 for error)
|
|
23
24
|
*/
|
|
24
25
|
export declare function runCli(): Promise<number>;
|
|
25
|
-
export { registerAggregateCommand, registerPlanCommand, registerRunCommand, registerValidateCommand, };
|
|
26
|
+
export { registerAggregateCommand, registerEvaluateCommand, registerPlanCommand, registerRunCommand, registerValidateCommand, };
|
|
26
27
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/cli/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/cli/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAAE,wBAAwB,EAAE,MAAM,yBAAyB,CAAC;AACnE,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AAEjE;;;;;;GAMG;AACH,wBAAgB,gBAAgB,IAAI,OAAO,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/cli/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAAE,wBAAwB,EAAE,MAAM,yBAAyB,CAAC;AACnE,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACjE,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AAEjE;;;;;;GAMG;AACH,wBAAgB,gBAAgB,IAAI,OAAO,CAgB1C;AAED;;;;GAIG;AACH,wBAAsB,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC,CAS9C;AAiBD,OAAO,EACN,wBAAwB,EACxB,uBAAuB,EACvB,mBAAmB,EACnB,kBAAkB,EAClB,uBAAuB,GACvB,CAAC"}
|
package/dist/cli/index.js
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
*/
|
|
6
6
|
import { Command } from "commander";
|
|
7
7
|
import { registerAggregateCommand } from "./commands/aggregate.js";
|
|
8
|
+
import { registerEvaluateCommand } from "./commands/evaluate.js";
|
|
8
9
|
import { registerPlanCommand } from "./commands/plan.js";
|
|
9
10
|
import { registerRunCommand } from "./commands/run.js";
|
|
10
11
|
import { registerValidateCommand } from "./commands/validate.js";
|
|
@@ -26,6 +27,7 @@ export function createCliProgram() {
|
|
|
26
27
|
registerValidateCommand(program);
|
|
27
28
|
registerPlanCommand(program);
|
|
28
29
|
registerAggregateCommand(program);
|
|
30
|
+
registerEvaluateCommand(program);
|
|
29
31
|
return program;
|
|
30
32
|
}
|
|
31
33
|
/**
|
|
@@ -54,5 +56,5 @@ if (import.meta.url === `file://${process.argv[1]}`) {
|
|
|
54
56
|
process.exit(1);
|
|
55
57
|
});
|
|
56
58
|
}
|
|
57
|
-
export { registerAggregateCommand, registerPlanCommand, registerRunCommand, registerValidateCommand, };
|
|
59
|
+
export { registerAggregateCommand, registerEvaluateCommand, registerPlanCommand, registerRunCommand, registerValidateCommand, };
|
|
58
60
|
//# sourceMappingURL=index.js.map
|
package/dist/cli/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/cli/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAAE,wBAAwB,EAAE,MAAM,yBAAyB,CAAC;AACnE,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AAEjE;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB;IAC/B,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;IAE9B,OAAO;SACL,IAAI,CAAC,MAAM,CAAC;SACZ,WAAW,CAAC,2EAA2E,CAAC;SACxF,OAAO,CAAC,OAAO,CAAC,CAAC;IAEnB,oBAAoB;IACpB,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAC5B,uBAAuB,CAAC,OAAO,CAAC,CAAC;IACjC,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAC7B,wBAAwB,CAAC,OAAO,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/cli/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAAE,wBAAwB,EAAE,MAAM,yBAAyB,CAAC;AACnE,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACjE,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AAEjE;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB;IAC/B,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;IAE9B,OAAO;SACL,IAAI,CAAC,MAAM,CAAC;SACZ,WAAW,CAAC,2EAA2E,CAAC;SACxF,OAAO,CAAC,OAAO,CAAC,CAAC;IAEnB,oBAAoB;IACpB,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAC5B,uBAAuB,CAAC,OAAO,CAAC,CAAC;IACjC,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAC7B,wBAAwB,CAAC,OAAO,CAAC,CAAC;IAClC,uBAAuB,CAAC,OAAO,CAAC,CAAC;IAEjC,OAAO,OAAO,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,MAAM;IAC3B,MAAM,OAAO,GAAG,gBAAgB,EAAE,CAAC;IAEnC,IAAI,CAAC;QACJ,MAAM,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACvC,OAAO,CAAC,CAAC;IACV,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,CAAC,CAAC;IACV,CAAC;AACF,CAAC;AAED;;GAEG;AACH,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,UAAU,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IACrD,MAAM,EAAE,CAAC,IAAI,CACZ,CAAC,IAAI,EAAE,EAAE;QACR,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC,EACD,CAAC,KAAK,EAAE,EAAE;QACT,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;QACrC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC,CACD,CAAC;AACH,CAAC;AAED,OAAO,EACN,wBAAwB,EACxB,uBAAuB,EACvB,mBAAmB,EACnB,kBAAkB,EAClB,uBAAuB,GACvB,CAAC"}
|
|
@@ -5,6 +5,27 @@
|
|
|
5
5
|
* case definitions, and metrics extractors.
|
|
6
6
|
*/
|
|
7
7
|
import type { CaseDefinition, SutDefinition } from "../types/index.js";
|
|
8
|
+
/**
|
|
9
|
+
* Binary SUT configuration for module loader.
|
|
10
|
+
*/
|
|
11
|
+
interface BinaryConfig {
|
|
12
|
+
/** Type discriminator */
|
|
13
|
+
type: "binary";
|
|
14
|
+
/** Command to execute */
|
|
15
|
+
command: string;
|
|
16
|
+
/** Arguments to pass to command */
|
|
17
|
+
args?: string[];
|
|
18
|
+
/** How to serialize inputs to stdin */
|
|
19
|
+
inputFormat?: "json" | "raw" | "lines";
|
|
20
|
+
/** How to deserialize stdout */
|
|
21
|
+
outputFormat?: "json" | "raw" | "lines";
|
|
22
|
+
/** Timeout per run in milliseconds */
|
|
23
|
+
timeout?: number;
|
|
24
|
+
/** Exit code that indicates success */
|
|
25
|
+
successExitCode?: number;
|
|
26
|
+
/** Working directory */
|
|
27
|
+
cwd?: string;
|
|
28
|
+
}
|
|
8
29
|
/**
|
|
9
30
|
* Load a SUT factory from a module.
|
|
10
31
|
*
|
|
@@ -22,7 +43,7 @@ export declare function loadSutFactory(modulePath: string, exportName: string, b
|
|
|
22
43
|
config: Readonly<Record<string, unknown>>;
|
|
23
44
|
tags: readonly string[];
|
|
24
45
|
description?: string;
|
|
25
|
-
}, config?: Record<string, unknown
|
|
46
|
+
}, config?: Record<string, unknown>, binaryConfig?: BinaryConfig): Promise<SutDefinition>;
|
|
26
47
|
/**
|
|
27
48
|
* Load a case definition from a module.
|
|
28
49
|
*
|
|
@@ -43,4 +64,5 @@ export declare function loadCaseDefinition(modulePath: string, exportName: strin
|
|
|
43
64
|
* @throws Error if export cannot be found or loaded
|
|
44
65
|
*/
|
|
45
66
|
export declare function loadMetricsExtractor(modulePath: string, exportName: string, baseDir: string): Promise<(result: unknown) => Record<string, number>>;
|
|
67
|
+
export {};
|
|
46
68
|
//# sourceMappingURL=module-loader.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"module-loader.d.ts","sourceRoot":"","sources":["../../src/cli/module-loader.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"module-loader.d.ts","sourceRoot":"","sources":["../../src/cli/module-loader.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAGvE;;GAEG;AACH,UAAU,YAAY;IACrB,yBAAyB;IACzB,IAAI,EAAE,QAAQ,CAAC;IAEf,yBAAyB;IACzB,OAAO,EAAE,MAAM,CAAC;IAEhB,mCAAmC;IACnC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAEhB,uCAAuC;IACvC,WAAW,CAAC,EAAE,MAAM,GAAG,KAAK,GAAG,OAAO,CAAC;IAEvC,gCAAgC;IAChC,YAAY,CAAC,EAAE,MAAM,GAAG,KAAK,GAAG,OAAO,CAAC;IAExC,sCAAsC;IACtC,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,uCAAuC;IACvC,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB,wBAAwB;IACxB,GAAG,CAAC,EAAE,MAAM,CAAC;CACb;AA6BD;;;;;;;;GAQG;AACH,wBAAsB,cAAc,CACnC,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,MAAM,EACf,YAAY,EAAE;IACb,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,SAAS,GAAG,UAAU,GAAG,QAAQ,CAAC;IACxC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IAC1C,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB,EACD,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAChC,YAAY,CAAC,EAAE,YAAY,GACzB,OAAO,CAAC,aAAa,CAAC,CAuDxB;AAED;;;;;;;;GAQG;AACH,wBAAsB,kBAAkB,CACvC,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,MAAM,GACb,OAAO,CAAC,cAAc,CAAC,CAqCzB;AAED;;;;;;;;GAQG;AACH,wBAAsB,oBAAoB,CACzC,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,MAAM,GACb,OAAO,CAAC,CAAC,MAAM,EAAE,OAAO,KAAK,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAYtD"}
|
|
@@ -40,7 +40,25 @@ async function loadModule(modulePath, baseDir) {
|
|
|
40
40
|
* @returns SUT definition ready for registration
|
|
41
41
|
* @throws Error if export cannot be found or loaded
|
|
42
42
|
*/
|
|
43
|
-
export async function loadSutFactory(modulePath, exportName, baseDir, registration, config) {
|
|
43
|
+
export async function loadSutFactory(modulePath, exportName, baseDir, registration, config, binaryConfig) {
|
|
44
|
+
// Handle binary SUTs
|
|
45
|
+
if (binaryConfig?.type === "binary") {
|
|
46
|
+
const { createBinarySut } = await import("../executor/binary-sut.js");
|
|
47
|
+
const factory = createBinarySut({
|
|
48
|
+
id: registration.id,
|
|
49
|
+
command: binaryConfig.command,
|
|
50
|
+
args: binaryConfig.args,
|
|
51
|
+
inputFormat: binaryConfig.inputFormat ?? "json",
|
|
52
|
+
outputFormat: binaryConfig.outputFormat ?? "json",
|
|
53
|
+
timeout: binaryConfig.timeout,
|
|
54
|
+
successExitCode: binaryConfig.successExitCode,
|
|
55
|
+
cwd: binaryConfig.cwd ?? resolve(baseDir),
|
|
56
|
+
});
|
|
57
|
+
return {
|
|
58
|
+
registration,
|
|
59
|
+
factory: factory,
|
|
60
|
+
};
|
|
61
|
+
}
|
|
44
62
|
const module = await loadModule(modulePath, baseDir);
|
|
45
63
|
const factoryExport = module[exportName];
|
|
46
64
|
if (typeof factoryExport !== "function") {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"module-loader.js","sourceRoot":"","sources":["../../src/cli/module-loader.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"module-loader.js","sourceRoot":"","sources":["../../src/cli/module-loader.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAkCzC;;;;;;;GAOG;AACH,KAAK,UAAU,UAAU,CAAC,UAAkB,EAAE,OAAe;IAC5D,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;IAElD,qDAAqD;IACrD,MAAM,SAAS,GAAG,aAAa,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC;IAEnD,IAAI,CAAC;QACJ,mEAAmE;QACnE,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;QACvC,+DAA+D;QAC/D,OAAO,MAAM,CAAC;IACf,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,8BAA8B,UAAU,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/E,CAAC;QACD,MAAM,KAAK,CAAC;IACb,CAAC;AACF,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,UAAkB,EAClB,UAAkB,EAClB,OAAe,EACf,YAQC,EACD,MAAgC,EAChC,YAA2B;IAE3B,qBAAqB;IACrB,IAAI,YAAY,EAAE,IAAI,KAAK,QAAQ,EAAE,CAAC;QACrC,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;QACtE,MAAM,OAAO,GAAG,eAAe,CAAC;YAC/B,EAAE,EAAE,YAAY,CAAC,EAAE;YACnB,OAAO,EAAE,YAAY,CAAC,OAAO;YAC7B,IAAI,EAAE,YAAY,CAAC,IAAI;YACvB,WAAW,EAAE,YAAY,CAAC,WAAW,IAAI,MAAM;YAC/C,YAAY,EAAE,YAAY,CAAC,YAAY,IAAI,MAAM;YACjD,OAAO,EAAE,YAAY,CAAC,OAAO;YAC7B,eAAe,EAAE,YAAY,CAAC,eAAe;YAC7C,GAAG,EAAE,YAAY,CAAC,GAAG,IAAI,OAAO,CAAC,OAAO,CAAC;SACzC,CAAC,CAAC;QAEH,OAAO;YACN,YAAY;YACZ,OAAO,EAAE,OAIR;SACD,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACrD,MAAM,aAAa,GAAG,MAAM,CAAC,UAAU,CAAiC,CAAC;IAEzE,IAAI,OAAO,aAAa,KAAK,UAAU,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CACd,WAAW,UAAU,QAAQ,UAAU,sBAAsB;YAC5D,eAAe,OAAO,aAAa,EAAE,CACtC,CAAC;IACH,CAAC;IAED,yDAAyD;IACzD,MAAM,OAAO,GAIT,CAAC,UAAoC,EAAE,EAAE;QAC5C,MAAM,QAAQ,GAAG,aAAa,CAAC,EAAE,GAAG,MAAM,EAAE,GAAG,UAAU,EAAE,CAAC,CAAC;QAC7D,MAAM,YAAY,GAAG,EAAE,GAAG,MAAM,EAAE,GAAG,UAAU,EAAE,GAAG,CAAC,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,CAAC;QAE9E,OAAO;YACN,EAAE,EAAE,YAAY,CAAC,EAAE;YACnB,MAAM,EAAE,YAAiD;YACzD,GAAG,EAAE,QAAQ,CAAC,GAA4C;SAC1D,CAAC;IACH,CAAC,CAAC;IAEF,OAAO;QACN,YAAY;QACZ,OAAO;KACP,CAAC;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACvC,UAAkB,EAClB,UAAkB,EAClB,OAAe;IAEf,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACrD,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAqC,CAAC;IAE1E,IAAI,OAAO,UAAU,KAAK,UAAU,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CACd,WAAW,UAAU,QAAQ,UAAU,sBAAsB;YAC5D,eAAe,OAAO,UAAU,EAAE,CACnC,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,UAAU,EAAE,CAAC;IAEhC,qBAAqB;IACrB,uEAAuE;IACvE,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CACd,WAAW,UAAU,QAAQ,UAAU,4CAA4C;YAClF,sBAAsB,CACvB,CAAC;IACH,CAAC;IAED,IAAI,OAAO,UAAU,CAAC,QAAQ,KAAK,UAAU,EAAE,CAAC;QAC/C,MAAM,IAAI,KAAK,CACd,WAAW,UAAU,QAAQ,UAAU,4CAA4C;YAClF,4BAA4B,CAC7B,CAAC;IACH,CAAC;IAED,IAAI,OAAO,UAAU,CAAC,SAAS,KAAK,UAAU,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CACd,WAAW,UAAU,QAAQ,UAAU,4CAA4C;YAClF,6BAA6B,CAC9B,CAAC;IACH,CAAC;IAED,OAAO,UAA4B,CAAC;AACrC,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACzC,UAAkB,EAClB,UAAkB,EAClB,OAAe;IAEf,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACrD,MAAM,eAAe,GAAG,MAAM,CAAC,UAAU,CAAuC,CAAC;IAEjF,IAAI,OAAO,eAAe,KAAK,UAAU,EAAE,CAAC;QAC3C,MAAM,IAAI,KAAK,CACd,WAAW,UAAU,QAAQ,UAAU,sBAAsB;YAC5D,eAAe,OAAO,eAAe,EAAE,CACxC,CAAC;IACH,CAAC;IAED,OAAO,eAAe,CAAC;AACxB,CAAC"}
|
package/dist/cli/types.d.ts
CHANGED
|
@@ -44,6 +44,18 @@ export interface SutConfig {
|
|
|
44
44
|
exportName: string;
|
|
45
45
|
/** Optional configuration to pass to factory */
|
|
46
46
|
config?: Record<string, unknown>;
|
|
47
|
+
/** SUT type: "module" (default) or "binary" */
|
|
48
|
+
type?: "module" | "binary";
|
|
49
|
+
/** Binary SUT: command to execute (when type="binary") */
|
|
50
|
+
binaryCommand?: string;
|
|
51
|
+
/** Binary SUT: arguments to pass to command */
|
|
52
|
+
binaryArgs?: string[];
|
|
53
|
+
/** Binary SUT: how to serialize inputs to stdin */
|
|
54
|
+
binaryInputFormat?: "json" | "raw" | "lines";
|
|
55
|
+
/** Binary SUT: how to deserialize stdout */
|
|
56
|
+
binaryOutputFormat?: "json" | "raw" | "lines";
|
|
57
|
+
/** Binary SUT: timeout per run in milliseconds */
|
|
58
|
+
binaryTimeout?: number;
|
|
47
59
|
/** SUT registration metadata */
|
|
48
60
|
registration: {
|
|
49
61
|
/** Human-readable name */
|
|
@@ -126,6 +138,13 @@ export interface CliOptions {
|
|
|
126
138
|
quiet?: boolean;
|
|
127
139
|
/** Dry run (plan without executing) */
|
|
128
140
|
dryRun?: boolean;
|
|
141
|
+
/**
|
|
142
|
+
* Run in-process without worker thread isolation.
|
|
143
|
+
*
|
|
144
|
+
* WARNING: This is unsafe and should only be used for debugging.
|
|
145
|
+
* SUT crashes can crash the CLI process.
|
|
146
|
+
*/
|
|
147
|
+
unsafeInProcess?: boolean;
|
|
129
148
|
}
|
|
130
149
|
/**
|
|
131
150
|
* Validation result.
|
package/dist/cli/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/cli/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAE/C;;GAEG;AACH,MAAM,WAAW,cAAc;IAC9B,sBAAsB;IACtB,IAAI,EAAE,MAAM,CAAC;IAEb,6BAA6B;IAC7B,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,qBAAqB;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC9B,+CAA+C;IAC/C,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B,qCAAqC;IACrC,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,uBAAuB;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,uDAAuD;IACvD,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,gDAAgD;IAChD,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAE5B,gCAAgC;IAChC,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACzB,4BAA4B;IAC5B,EAAE,EAAE,MAAM,CAAC;IAEX,oDAAoD;IACpD,MAAM,EAAE,MAAM,CAAC;IAEf,2CAA2C;IAC3C,UAAU,EAAE,MAAM,CAAC;IAEnB,gDAAgD;IAChD,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEjC,gCAAgC;IAChC,YAAY,EAAE;QACb,0BAA0B;QAC1B,IAAI,EAAE,MAAM,CAAC;QAEb,qBAAqB;QACrB,OAAO,EAAE,MAAM,CAAC;QAEhB,yBAAyB;QACzB,IAAI,EAAE,OAAO,CAAC;QAEd,sBAAsB;QACtB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;QAEhB,2BAA2B;QAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;CACF;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,6BAA6B;IAC7B,EAAE,EAAE,MAAM,CAAC;IAEX,oDAAoD;IACpD,MAAM,EAAE,MAAM,CAAC;IAEf,gDAAgD;IAChD,UAAU,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC,oDAAoD;IACpD,MAAM,EAAE,MAAM,CAAC;IAEf,qDAAqD;IACrD,UAAU,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC5B,4BAA4B;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,6CAA6C;IAC7C,MAAM,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC;IAEhC,mCAAmC;IACnC,SAAS,CAAC,EAAE,OAAO,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC,0BAA0B;IAC1B,UAAU,EAAE,cAAc,CAAC;IAE3B,6BAA6B;IAC7B,QAAQ,EAAE,cAAc,CAAC;IAEzB,uBAAuB;IACvB,IAAI,EAAE,SAAS,EAAE,CAAC;IAElB,wBAAwB;IACxB,KAAK,EAAE,UAAU,EAAE,CAAC;IAEpB,sCAAsC;IACtC,gBAAgB,EAAE,sBAAsB,CAAC;IAEzC,2BAA2B;IAC3B,MAAM,EAAE,YAAY,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,0BAA0B;IAC1B,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,gCAAgC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,oBAAoB;IACpB,MAAM,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC;IAEhC,uBAAuB;IACvB,WAAW,CAAC,EAAE,OAAO,CAAC;IAEtB,2BAA2B;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,sBAAsB;IACtB,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB,sBAAsB;IACtB,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB,uCAAuC;IACvC,MAAM,CAAC,EAAE,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/cli/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAE/C;;GAEG;AACH,MAAM,WAAW,cAAc;IAC9B,sBAAsB;IACtB,IAAI,EAAE,MAAM,CAAC;IAEb,6BAA6B;IAC7B,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,qBAAqB;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC9B,+CAA+C;IAC/C,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B,qCAAqC;IACrC,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,uBAAuB;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,uDAAuD;IACvD,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,gDAAgD;IAChD,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAE5B,gCAAgC;IAChC,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACzB,4BAA4B;IAC5B,EAAE,EAAE,MAAM,CAAC;IAEX,oDAAoD;IACpD,MAAM,EAAE,MAAM,CAAC;IAEf,2CAA2C;IAC3C,UAAU,EAAE,MAAM,CAAC;IAEnB,gDAAgD;IAChD,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEjC,+CAA+C;IAC/C,IAAI,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAE3B,0DAA0D;IAC1D,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,+CAA+C;IAC/C,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IAEtB,mDAAmD;IACnD,iBAAiB,CAAC,EAAE,MAAM,GAAG,KAAK,GAAG,OAAO,CAAC;IAE7C,4CAA4C;IAC5C,kBAAkB,CAAC,EAAE,MAAM,GAAG,KAAK,GAAG,OAAO,CAAC;IAE9C,kDAAkD;IAClD,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,gCAAgC;IAChC,YAAY,EAAE;QACb,0BAA0B;QAC1B,IAAI,EAAE,MAAM,CAAC;QAEb,qBAAqB;QACrB,OAAO,EAAE,MAAM,CAAC;QAEhB,yBAAyB;QACzB,IAAI,EAAE,OAAO,CAAC;QAEd,sBAAsB;QACtB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;QAEhB,2BAA2B;QAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;CACF;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,6BAA6B;IAC7B,EAAE,EAAE,MAAM,CAAC;IAEX,oDAAoD;IACpD,MAAM,EAAE,MAAM,CAAC;IAEf,gDAAgD;IAChD,UAAU,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC,oDAAoD;IACpD,MAAM,EAAE,MAAM,CAAC;IAEf,qDAAqD;IACrD,UAAU,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC5B,4BAA4B;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,6CAA6C;IAC7C,MAAM,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC;IAEhC,mCAAmC;IACnC,SAAS,CAAC,EAAE,OAAO,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC,0BAA0B;IAC1B,UAAU,EAAE,cAAc,CAAC;IAE3B,6BAA6B;IAC7B,QAAQ,EAAE,cAAc,CAAC;IAEzB,uBAAuB;IACvB,IAAI,EAAE,SAAS,EAAE,CAAC;IAElB,wBAAwB;IACxB,KAAK,EAAE,UAAU,EAAE,CAAC;IAEpB,sCAAsC;IACtC,gBAAgB,EAAE,sBAAsB,CAAC;IAEzC,2BAA2B;IAC3B,MAAM,EAAE,YAAY,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,0BAA0B;IAC1B,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,gCAAgC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,oBAAoB;IACpB,MAAM,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC;IAEhC,uBAAuB;IACvB,WAAW,CAAC,EAAE,OAAO,CAAC;IAEtB,2BAA2B;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,sBAAsB;IACtB,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB,sBAAsB;IACtB,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB,uCAAuC;IACvC,MAAM,CAAC,EAAE,OAAO,CAAC;IAEjB;;;;;OAKG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC,gCAAgC;IAChC,KAAK,EAAE,OAAO,CAAC;IAEf,wBAAwB;IACxB,MAAM,EAAE,MAAM,EAAE,CAAC;IAEjB,0BAA0B;IAC1B,QAAQ,EAAE,MAAM,EAAE,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC5B,sCAAsC;IACtC,MAAM,EAAE,gBAAgB,CAAC;IAEzB,wEAAwE;IACxE,OAAO,EAAE,MAAM,CAAC;IAEhB,mCAAmC;IACnC,UAAU,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,MAAM,gBAAgB,GAAG,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK;IACpE,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,GAAG,EAAE,CAAC,MAAM,EAAE,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CAC3C,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG,MAAM;IACxC,IAAI,EAAE;QACL,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,MAAM,EAAE,OAAO,CAAC;QAChB,cAAc,CAAC,EAAE,OAAO,CAAC;QACzB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;KACzB,CAAC;IACF,QAAQ,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;IACjC,SAAS,EAAE,MAAM,OAAO,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,sBAAsB,GAAG,CAAC,MAAM,EAAE,OAAO,KAAK,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC"}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Claims Evaluator
|
|
3
|
+
*
|
|
4
|
+
* Evaluates explicit hypotheses (claims) against aggregated results.
|
|
5
|
+
* Refactored from src/claims/evaluator.ts into a class-based design
|
|
6
|
+
* that implements the Evaluator interface.
|
|
7
|
+
*/
|
|
8
|
+
import type { IEvaluator, ClaimsEvaluatorConfig, ClaimsEvaluatorData, EvaluationContext, Evaluator, EvaluationOutput, EvaluationSummary, ValidationResult } from "../types/evaluator.js";
|
|
9
|
+
/**
|
|
10
|
+
* Claims evaluator - evaluates hypotheses against aggregated results.
|
|
11
|
+
*/
|
|
12
|
+
export declare class ClaimsEvaluator implements Evaluator<ClaimsEvaluatorConfig, EvaluationContext, ClaimsEvaluatorData>, IEvaluator {
|
|
13
|
+
/** Type identifier */
|
|
14
|
+
readonly type: "claims";
|
|
15
|
+
/** Schema version */
|
|
16
|
+
private static readonly VERSION;
|
|
17
|
+
/**
|
|
18
|
+
* Validate claims evaluator configuration.
|
|
19
|
+
*
|
|
20
|
+
* @param config - Configuration to validate
|
|
21
|
+
* @returns Validation result
|
|
22
|
+
*/
|
|
23
|
+
validateConfig(config: ClaimsEvaluatorConfig): ValidationResult;
|
|
24
|
+
/**
|
|
25
|
+
* Validate a single claim.
|
|
26
|
+
*
|
|
27
|
+
* @param claim - Claim to validate
|
|
28
|
+
* @param index - Index in claims array (for error messages)
|
|
29
|
+
* @returns Array of error messages
|
|
30
|
+
*/
|
|
31
|
+
private validateClaim;
|
|
32
|
+
/**
|
|
33
|
+
* Evaluate claims against aggregated results.
|
|
34
|
+
*
|
|
35
|
+
* @param config - Claims evaluator configuration
|
|
36
|
+
* @param input - Evaluation context with aggregates
|
|
37
|
+
* @returns Evaluation output
|
|
38
|
+
*/
|
|
39
|
+
evaluate(config: ClaimsEvaluatorConfig, input: EvaluationContext): EvaluationOutput<ClaimsEvaluatorData>;
|
|
40
|
+
/**
|
|
41
|
+
* Evaluate a single claim against aggregated results.
|
|
42
|
+
*
|
|
43
|
+
* @param claim - The claim to evaluate
|
|
44
|
+
* @param aggregates - Aggregated results from the pipeline
|
|
45
|
+
* @returns Claim evaluation with status and evidence
|
|
46
|
+
*/
|
|
47
|
+
private evaluateClaim;
|
|
48
|
+
/**
|
|
49
|
+
* Filter aggregates by claim scope constraints.
|
|
50
|
+
*
|
|
51
|
+
* @param aggregates - All aggregates
|
|
52
|
+
* @param claim - Claim with scope constraints
|
|
53
|
+
* @returns Filtered aggregates
|
|
54
|
+
*/
|
|
55
|
+
private filterByScope;
|
|
56
|
+
/**
|
|
57
|
+
* Create an inconclusive result with reasons.
|
|
58
|
+
*
|
|
59
|
+
* @param claim - The claim being evaluated
|
|
60
|
+
* @param reasons - Reasons for inconclusive status
|
|
61
|
+
* @returns Inconclusive claim evaluation
|
|
62
|
+
*/
|
|
63
|
+
private createInconclusiveResult;
|
|
64
|
+
/**
|
|
65
|
+
* Determine claim status based on evidence.
|
|
66
|
+
*
|
|
67
|
+
* @param claim - The claim being evaluated
|
|
68
|
+
* @param evidence - Computed evidence
|
|
69
|
+
* @returns Claim status
|
|
70
|
+
*/
|
|
71
|
+
private determineClaimStatus;
|
|
72
|
+
/**
|
|
73
|
+
* Create a claim evaluation summary.
|
|
74
|
+
*
|
|
75
|
+
* @param evaluations - Completed claim evaluations
|
|
76
|
+
* @returns Summary with counts and rates
|
|
77
|
+
*/
|
|
78
|
+
private createClaimSummary;
|
|
79
|
+
/**
|
|
80
|
+
* Summarize evaluation output.
|
|
81
|
+
*
|
|
82
|
+
* @param output - Evaluation output to summarize
|
|
83
|
+
* @returns Summary statistics
|
|
84
|
+
*/
|
|
85
|
+
summarize(output: EvaluationOutput<ClaimsEvaluatorData>): EvaluationSummary;
|
|
86
|
+
}
|
|
87
|
+
//# sourceMappingURL=claims-evaluator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"claims-evaluator.d.ts","sourceRoot":"","sources":["../../src/evaluators/claims-evaluator.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAWH,OAAO,KAAK,EACX,UAAU,EACV,qBAAqB,EACrB,mBAAmB,EACnB,iBAAiB,EACjB,SAAS,EACT,gBAAgB,EAChB,iBAAiB,EACjB,gBAAgB,EAChB,MAAM,uBAAuB,CAAC;AAE/B;;GAEG;AACH,qBAAa,eACZ,YAAW,SAAS,CAAC,qBAAqB,EAAE,iBAAiB,EAAE,mBAAmB,CAAC,EAAE,UAAU;IAE/F,sBAAsB;IACtB,QAAQ,CAAC,IAAI,EAAG,QAAQ,CAAU;IAElC,qBAAqB;IACrB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAW;IAE1C;;;;;OAKG;IACH,cAAc,CAAC,MAAM,EAAE,qBAAqB,GAAG,gBAAgB;IA4B/D;;;;;;OAMG;IACH,OAAO,CAAC,aAAa;IAgCrB;;;;;;OAMG;IACH,QAAQ,CACP,MAAM,EAAE,qBAAqB,EAC7B,KAAK,EAAE,iBAAiB,GACtB,gBAAgB,CAAC,mBAAmB,CAAC;IAuBxC;;;;;;OAMG;IACH,OAAO,CAAC,aAAa;IA8DrB;;;;;;OAMG;IACH,OAAO,CAAC,aAAa;IAsBrB;;;;;;OAMG;IACH,OAAO,CAAC,wBAAwB;IAmBhC;;;;;;OAMG;IACH,OAAO,CAAC,oBAAoB;IA4C5B;;;;;OAKG;IACH,OAAO,CAAC,kBAAkB;IAsB1B;;;;;OAKG;IACH,SAAS,CAAC,MAAM,EAAE,gBAAgB,CAAC,mBAAmB,CAAC,GAAG,iBAAiB;CAc3E"}
|
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Claims Evaluator
|
|
3
|
+
*
|
|
4
|
+
* Evaluates explicit hypotheses (claims) against aggregated results.
|
|
5
|
+
* Refactored from src/claims/evaluator.ts into a class-based design
|
|
6
|
+
* that implements the Evaluator interface.
|
|
7
|
+
*/
|
|
8
|
+
/**
|
|
9
|
+
* Claims evaluator - evaluates hypotheses against aggregated results.
|
|
10
|
+
*/
|
|
11
|
+
export class ClaimsEvaluator {
|
|
12
|
+
/** Type identifier */
|
|
13
|
+
type = "claims";
|
|
14
|
+
/** Schema version */
|
|
15
|
+
static VERSION = "1.0.0";
|
|
16
|
+
/**
|
|
17
|
+
* Validate claims evaluator configuration.
|
|
18
|
+
*
|
|
19
|
+
* @param config - Configuration to validate
|
|
20
|
+
* @returns Validation result
|
|
21
|
+
*/
|
|
22
|
+
validateConfig(config) {
|
|
23
|
+
const errors = [];
|
|
24
|
+
const warnings = [];
|
|
25
|
+
// Check claims array
|
|
26
|
+
if (!Array.isArray(config.claims)) {
|
|
27
|
+
errors.push("claims must be an array");
|
|
28
|
+
return { valid: false, errors, warnings };
|
|
29
|
+
}
|
|
30
|
+
if (config.claims.length === 0) {
|
|
31
|
+
warnings.push("No claims provided - evaluation will produce empty results");
|
|
32
|
+
}
|
|
33
|
+
// Validate each claim
|
|
34
|
+
for (let i = 0; i < config.claims.length; i++) {
|
|
35
|
+
const claim = config.claims[i];
|
|
36
|
+
const claimErrors = this.validateClaim(claim, i);
|
|
37
|
+
errors.push(...claimErrors);
|
|
38
|
+
}
|
|
39
|
+
return {
|
|
40
|
+
valid: errors.length === 0,
|
|
41
|
+
errors: errors.length > 0 ? errors : undefined,
|
|
42
|
+
warnings: warnings.length > 0 ? warnings : undefined,
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Validate a single claim.
|
|
47
|
+
*
|
|
48
|
+
* @param claim - Claim to validate
|
|
49
|
+
* @param index - Index in claims array (for error messages)
|
|
50
|
+
* @returns Array of error messages
|
|
51
|
+
*/
|
|
52
|
+
validateClaim(claim, index) {
|
|
53
|
+
const errors = [];
|
|
54
|
+
const prefix = `Claim[${index}]`;
|
|
55
|
+
if (!claim.claimId || typeof claim.claimId !== "string") {
|
|
56
|
+
errors.push(`${prefix}: claimId is required`);
|
|
57
|
+
}
|
|
58
|
+
if (!claim.description || typeof claim.description !== "string") {
|
|
59
|
+
errors.push(`${prefix}: description is required`);
|
|
60
|
+
}
|
|
61
|
+
if (!claim.sut || typeof claim.sut !== "string") {
|
|
62
|
+
errors.push(`${prefix}: sut is required`);
|
|
63
|
+
}
|
|
64
|
+
if (!claim.baseline || typeof claim.baseline !== "string") {
|
|
65
|
+
errors.push(`${prefix}: baseline is required`);
|
|
66
|
+
}
|
|
67
|
+
if (!claim.metric || typeof claim.metric !== "string") {
|
|
68
|
+
errors.push(`${prefix}: metric is required`);
|
|
69
|
+
}
|
|
70
|
+
if (!["greater", "less", "equal"].includes(claim.direction)) {
|
|
71
|
+
errors.push(`${prefix}: direction must be 'greater', 'less', or 'equal'`);
|
|
72
|
+
}
|
|
73
|
+
if (claim.threshold !== undefined && typeof claim.threshold !== "number") {
|
|
74
|
+
errors.push(`${prefix}: threshold must be a number`);
|
|
75
|
+
}
|
|
76
|
+
if (!["global", "caseClass", "parameterRange", "localStructure"].includes(claim.scope)) {
|
|
77
|
+
errors.push(`${prefix}: scope must be a valid ValidityScope`);
|
|
78
|
+
}
|
|
79
|
+
return errors;
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Evaluate claims against aggregated results.
|
|
83
|
+
*
|
|
84
|
+
* @param config - Claims evaluator configuration
|
|
85
|
+
* @param input - Evaluation context with aggregates
|
|
86
|
+
* @returns Evaluation output
|
|
87
|
+
*/
|
|
88
|
+
evaluate(config, input) {
|
|
89
|
+
const { aggregates } = input;
|
|
90
|
+
// Evaluate all claims
|
|
91
|
+
const evaluations = config.claims.map((claim) => this.evaluateClaim(claim, aggregates));
|
|
92
|
+
// Create summary
|
|
93
|
+
const summary = this.createClaimSummary(evaluations);
|
|
94
|
+
return {
|
|
95
|
+
type: "claims",
|
|
96
|
+
version: ClaimsEvaluator.VERSION,
|
|
97
|
+
timestamp: new Date().toISOString(),
|
|
98
|
+
data: summary,
|
|
99
|
+
metadata: {
|
|
100
|
+
inputSource: input.metadata?.source,
|
|
101
|
+
config,
|
|
102
|
+
},
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Evaluate a single claim against aggregated results.
|
|
107
|
+
*
|
|
108
|
+
* @param claim - The claim to evaluate
|
|
109
|
+
* @param aggregates - Aggregated results from the pipeline
|
|
110
|
+
* @returns Claim evaluation with status and evidence
|
|
111
|
+
*/
|
|
112
|
+
evaluateClaim(claim, aggregates) {
|
|
113
|
+
// Filter aggregates by scope constraints
|
|
114
|
+
const filteredAggregates = this.filterByScope(aggregates, claim);
|
|
115
|
+
// Find primary and baseline aggregates
|
|
116
|
+
const primaryAgg = filteredAggregates.find((a) => a.sut === claim.sut);
|
|
117
|
+
const baselineAgg = filteredAggregates.find((a) => a.sut === claim.baseline);
|
|
118
|
+
// Handle missing data
|
|
119
|
+
if (!primaryAgg || !baselineAgg) {
|
|
120
|
+
return this.createInconclusiveResult(claim, primaryAgg ? undefined : "Primary SUT not found", baselineAgg ? undefined : "Baseline SUT not found");
|
|
121
|
+
}
|
|
122
|
+
// Get metric values
|
|
123
|
+
const primaryMetric = claim.metric;
|
|
124
|
+
const baselineMetric = claim.metric;
|
|
125
|
+
const primaryStats = primaryAgg.metrics[primaryMetric];
|
|
126
|
+
const baselineStats = baselineAgg.metrics[baselineMetric];
|
|
127
|
+
if (!(primaryMetric in primaryAgg.metrics) || !(baselineMetric in baselineAgg.metrics)) {
|
|
128
|
+
return this.createInconclusiveResult(claim, "Metric not found in primary results", "Metric not found in baseline results");
|
|
129
|
+
}
|
|
130
|
+
// Compute evidence
|
|
131
|
+
const primaryValue = primaryStats.mean;
|
|
132
|
+
const baselineValue = baselineStats.mean;
|
|
133
|
+
const delta = primaryValue - baselineValue;
|
|
134
|
+
const ratio = baselineValue === 0 ? Infinity : primaryValue / baselineValue;
|
|
135
|
+
// Get statistical significance if available
|
|
136
|
+
const comparison = primaryAgg.comparisons?.[claim.baseline];
|
|
137
|
+
const pValue = comparison?.pValue;
|
|
138
|
+
const effectSize = comparison?.effectSize;
|
|
139
|
+
const evidence = {
|
|
140
|
+
primaryValue,
|
|
141
|
+
baselineValue,
|
|
142
|
+
delta,
|
|
143
|
+
ratio,
|
|
144
|
+
pValue,
|
|
145
|
+
effectSize,
|
|
146
|
+
n: primaryStats.n + baselineStats.n,
|
|
147
|
+
};
|
|
148
|
+
// Determine claim status
|
|
149
|
+
const status = this.determineClaimStatus(claim, evidence);
|
|
150
|
+
return {
|
|
151
|
+
claim,
|
|
152
|
+
status,
|
|
153
|
+
evidence,
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* Filter aggregates by claim scope constraints.
|
|
158
|
+
*
|
|
159
|
+
* @param aggregates - All aggregates
|
|
160
|
+
* @param claim - Claim with scope constraints
|
|
161
|
+
* @returns Filtered aggregates
|
|
162
|
+
*/
|
|
163
|
+
filterByScope(aggregates, claim) {
|
|
164
|
+
if (!claim.scopeConstraints) {
|
|
165
|
+
return aggregates;
|
|
166
|
+
}
|
|
167
|
+
return aggregates.filter((agg) => {
|
|
168
|
+
for (const [key, value] of Object.entries(claim.scopeConstraints ?? {})) {
|
|
169
|
+
if (key === "caseClass") {
|
|
170
|
+
const allowedClasses = Array.isArray(value) ? value : [value];
|
|
171
|
+
if (!allowedClasses.includes(agg.caseClass)) {
|
|
172
|
+
return false;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
// Add more scope constraint checks as needed
|
|
176
|
+
}
|
|
177
|
+
return true;
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
/**
|
|
181
|
+
* Create an inconclusive result with reasons.
|
|
182
|
+
*
|
|
183
|
+
* @param claim - The claim being evaluated
|
|
184
|
+
* @param reasons - Reasons for inconclusive status
|
|
185
|
+
* @returns Inconclusive claim evaluation
|
|
186
|
+
*/
|
|
187
|
+
createInconclusiveResult(claim, ...reasons) {
|
|
188
|
+
const validReasons = reasons.filter((r) => r !== undefined);
|
|
189
|
+
return {
|
|
190
|
+
claim,
|
|
191
|
+
status: "inconclusive",
|
|
192
|
+
evidence: {
|
|
193
|
+
primaryValue: Number.NaN,
|
|
194
|
+
baselineValue: Number.NaN,
|
|
195
|
+
delta: Number.NaN,
|
|
196
|
+
ratio: Number.NaN,
|
|
197
|
+
},
|
|
198
|
+
inconclusiveReason: validReasons.join("; "),
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
/**
|
|
202
|
+
* Determine claim status based on evidence.
|
|
203
|
+
*
|
|
204
|
+
* @param claim - The claim being evaluated
|
|
205
|
+
* @param evidence - Computed evidence
|
|
206
|
+
* @returns Claim status
|
|
207
|
+
*/
|
|
208
|
+
determineClaimStatus(claim, evidence) {
|
|
209
|
+
// Check for missing data
|
|
210
|
+
if (Number.isNaN(evidence.primaryValue) || Number.isNaN(evidence.baselineValue)) {
|
|
211
|
+
return "inconclusive";
|
|
212
|
+
}
|
|
213
|
+
// Check statistical significance if required
|
|
214
|
+
const significanceLevel = claim.significanceLevel ?? 0.05;
|
|
215
|
+
if (evidence.pValue !== undefined && evidence.pValue > significanceLevel) {
|
|
216
|
+
return "inconclusive";
|
|
217
|
+
}
|
|
218
|
+
// Check minimum effect size if required
|
|
219
|
+
if (claim.minEffectSize !== undefined &&
|
|
220
|
+
evidence.effectSize !== undefined &&
|
|
221
|
+
Math.abs(evidence.effectSize) < claim.minEffectSize) {
|
|
222
|
+
return "inconclusive";
|
|
223
|
+
}
|
|
224
|
+
// Evaluate direction
|
|
225
|
+
switch (claim.direction) {
|
|
226
|
+
case "greater": {
|
|
227
|
+
if (claim.threshold !== undefined) {
|
|
228
|
+
return evidence.delta >= claim.threshold ? "satisfied" : "violated";
|
|
229
|
+
}
|
|
230
|
+
return evidence.delta > 0 ? "satisfied" : "violated";
|
|
231
|
+
}
|
|
232
|
+
case "less": {
|
|
233
|
+
if (claim.threshold !== undefined) {
|
|
234
|
+
return evidence.delta <= -claim.threshold ? "satisfied" : "violated";
|
|
235
|
+
}
|
|
236
|
+
return evidence.delta < 0 ? "satisfied" : "violated";
|
|
237
|
+
}
|
|
238
|
+
case "equal": {
|
|
239
|
+
const epsilon = claim.threshold ?? 0.001;
|
|
240
|
+
return Math.abs(evidence.delta) <= epsilon ? "satisfied" : "violated";
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
/**
|
|
245
|
+
* Create a claim evaluation summary.
|
|
246
|
+
*
|
|
247
|
+
* @param evaluations - Completed claim evaluations
|
|
248
|
+
* @returns Summary with counts and rates
|
|
249
|
+
*/
|
|
250
|
+
createClaimSummary(evaluations) {
|
|
251
|
+
const satisfied = evaluations.filter((e) => e.status === "satisfied").length;
|
|
252
|
+
const violated = evaluations.filter((e) => e.status === "violated").length;
|
|
253
|
+
const inconclusive = evaluations.filter((e) => e.status === "inconclusive").length;
|
|
254
|
+
const definitive = satisfied + violated;
|
|
255
|
+
const satisfactionRate = definitive > 0 ? satisfied / definitive : 0;
|
|
256
|
+
return {
|
|
257
|
+
version: "1.0.0",
|
|
258
|
+
timestamp: new Date().toISOString(),
|
|
259
|
+
evaluations,
|
|
260
|
+
summary: {
|
|
261
|
+
total: evaluations.length,
|
|
262
|
+
satisfied,
|
|
263
|
+
violated,
|
|
264
|
+
inconclusive,
|
|
265
|
+
satisfactionRate,
|
|
266
|
+
},
|
|
267
|
+
};
|
|
268
|
+
}
|
|
269
|
+
/**
|
|
270
|
+
* Summarize evaluation output.
|
|
271
|
+
*
|
|
272
|
+
* @param output - Evaluation output to summarize
|
|
273
|
+
* @returns Summary statistics
|
|
274
|
+
*/
|
|
275
|
+
summarize(output) {
|
|
276
|
+
const { summary } = output.data;
|
|
277
|
+
return {
|
|
278
|
+
total: summary.total,
|
|
279
|
+
passed: summary.satisfied,
|
|
280
|
+
failed: summary.violated,
|
|
281
|
+
inconclusive: summary.inconclusive,
|
|
282
|
+
passRate: summary.satisfactionRate,
|
|
283
|
+
additional: {
|
|
284
|
+
satisfactionRate: summary.satisfactionRate,
|
|
285
|
+
},
|
|
286
|
+
};
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
//# sourceMappingURL=claims-evaluator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"claims-evaluator.js","sourceRoot":"","sources":["../../src/evaluators/claims-evaluator.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAsBH;;GAEG;AACH,MAAM,OAAO,eAAe;IAG3B,sBAAsB;IACb,IAAI,GAAG,QAAiB,CAAC;IAElC,qBAAqB;IACb,MAAM,CAAU,OAAO,GAAG,OAAO,CAAC;IAE1C;;;;;OAKG;IACH,cAAc,CAAC,MAA6B;QAC3C,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAa,EAAE,CAAC;QAE9B,qBAAqB;QACrB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YACnC,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;YACvC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;QAC3C,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChC,QAAQ,CAAC,IAAI,CAAC,4DAA4D,CAAC,CAAC;QAC7E,CAAC;QAED,sBAAsB;QACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/C,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAC/B,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YACjD,MAAM,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;QAC7B,CAAC;QAED,OAAO;YACN,KAAK,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC;YAC1B,MAAM,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;YAC9C,QAAQ,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;SACpD,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACK,aAAa,CAAC,KAAsB,EAAE,KAAa;QAC1D,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,SAAS,KAAK,GAAG,CAAC;QAEjC,IAAI,CAAC,KAAK,CAAC,OAAO,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YACzD,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,uBAAuB,CAAC,CAAC;QAC/C,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,WAAW,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;YACjE,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,2BAA2B,CAAC,CAAC;QACnD,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,OAAO,KAAK,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC;YACjD,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,mBAAmB,CAAC,CAAC;QAC3C,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,QAAQ,IAAI,OAAO,KAAK,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC3D,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,wBAAwB,CAAC,CAAC;QAChD,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YACvD,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,sBAAsB,CAAC,CAAC;QAC9C,CAAC;QACD,IAAI,CAAC,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;YAC7D,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,mDAAmD,CAAC,CAAC;QAC3E,CAAC;QACD,IAAI,KAAK,CAAC,SAAS,KAAK,SAAS,IAAI,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;YAC1E,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,8BAA8B,CAAC,CAAC;QACtD,CAAC;QACD,IAAI,CAAC,CAAC,QAAQ,EAAE,WAAW,EAAE,gBAAgB,EAAE,gBAAgB,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;YACxF,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,uCAAuC,CAAC,CAAC;QAC/D,CAAC;QAED,OAAO,MAAM,CAAC;IACf,CAAC;IAED;;;;;;OAMG;IACH,QAAQ,CACP,MAA6B,EAC7B,KAAwB;QAExB,MAAM,EAAE,UAAU,EAAE,GAAG,KAAK,CAAC;QAE7B,sBAAsB;QACtB,MAAM,WAAW,GAAsB,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAClE,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,UAAU,CAAC,CACrC,CAAC;QAEF,iBAAiB;QACjB,MAAM,OAAO,GAAG,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,CAAC;QAErD,OAAO;YACN,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,eAAe,CAAC,OAAO;YAChC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,IAAI,EAAE,OAAO;YACb,QAAQ,EAAE;gBACT,WAAW,EAAE,KAAK,CAAC,QAAQ,EAAE,MAAM;gBACnC,MAAM;aACN;SACD,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACK,aAAa,CAAC,KAAsB,EAAE,UAA8B;QAC3E,yCAAyC;QACzC,MAAM,kBAAkB,GAAG,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QAEjE,uCAAuC;QACvC,MAAM,UAAU,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,KAAK,CAAC,GAAG,CAAC,CAAC;QACvE,MAAM,WAAW,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC;QAE7E,sBAAsB;QACtB,IAAI,CAAC,UAAU,IAAI,CAAC,WAAW,EAAE,CAAC;YACjC,OAAO,IAAI,CAAC,wBAAwB,CACnC,KAAK,EACL,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,uBAAuB,EAChD,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,wBAAwB,CAClD,CAAC;QACH,CAAC;QAED,oBAAoB;QACpB,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC;QACnC,MAAM,cAAc,GAAG,KAAK,CAAC,MAAM,CAAC;QACpC,MAAM,YAAY,GAAG,UAAU,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;QACvD,MAAM,aAAa,GAAG,WAAW,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QAE1D,IAAI,CAAC,CAAC,aAAa,IAAI,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,cAAc,IAAI,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;YACxF,OAAO,IAAI,CAAC,wBAAwB,CACnC,KAAK,EACL,qCAAqC,EACrC,sCAAsC,CACtC,CAAC;QACH,CAAC;QAED,mBAAmB;QACnB,MAAM,YAAY,GAAG,YAAY,CAAC,IAAI,CAAC;QACvC,MAAM,aAAa,GAAG,aAAa,CAAC,IAAI,CAAC;QACzC,MAAM,KAAK,GAAG,YAAY,GAAG,aAAa,CAAC;QAC3C,MAAM,KAAK,GAAG,aAAa,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,YAAY,GAAG,aAAa,CAAC;QAE5E,4CAA4C;QAC5C,MAAM,UAAU,GAAG,UAAU,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,UAAU,EAAE,MAAM,CAAC;QAClC,MAAM,UAAU,GAAG,UAAU,EAAE,UAAU,CAAC;QAE1C,MAAM,QAAQ,GAAkB;YAC/B,YAAY;YACZ,aAAa;YACb,KAAK;YACL,KAAK;YACL,MAAM;YACN,UAAU;YACV,CAAC,EAAE,YAAY,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC;SACnC,CAAC;QAEF,yBAAyB;QACzB,MAAM,MAAM,GAAG,IAAI,CAAC,oBAAoB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAE1D,OAAO;YACN,KAAK;YACL,MAAM;YACN,QAAQ;SACR,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACK,aAAa,CACpB,UAA8B,EAC9B,KAAsB;QAEtB,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;YAC7B,OAAO,UAAU,CAAC;QACnB,CAAC;QAED,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE;YAChC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,gBAAgB,IAAI,EAAE,CAAC,EAAE,CAAC;gBACzE,IAAI,GAAG,KAAK,WAAW,EAAE,CAAC;oBACzB,MAAM,cAAc,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;oBAC9D,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAsB,CAAC,EAAE,CAAC;wBAC1D,OAAO,KAAK,CAAC;oBACd,CAAC;gBACF,CAAC;gBACD,6CAA6C;YAC9C,CAAC;YACD,OAAO,IAAI,CAAC;QACb,CAAC,CAAC,CAAC;IACJ,CAAC;IAED;;;;;;OAMG;IACK,wBAAwB,CAC/B,KAAsB,EACtB,GAAG,OAA+B;QAElC,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC;QAEzE,OAAO;YACN,KAAK;YACL,MAAM,EAAE,cAAc;YACtB,QAAQ,EAAE;gBACT,YAAY,EAAE,MAAM,CAAC,GAAG;gBACxB,aAAa,EAAE,MAAM,CAAC,GAAG;gBACzB,KAAK,EAAE,MAAM,CAAC,GAAG;gBACjB,KAAK,EAAE,MAAM,CAAC,GAAG;aACjB;YACD,kBAAkB,EAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;SAC3C,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACK,oBAAoB,CAAC,KAAsB,EAAE,QAAuB;QAC3E,yBAAyB;QACzB,IAAI,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;YACjF,OAAO,cAAc,CAAC;QACvB,CAAC;QAED,6CAA6C;QAC7C,MAAM,iBAAiB,GAAG,KAAK,CAAC,iBAAiB,IAAI,IAAI,CAAC;QAC1D,IAAI,QAAQ,CAAC,MAAM,KAAK,SAAS,IAAI,QAAQ,CAAC,MAAM,GAAG,iBAAiB,EAAE,CAAC;YAC1E,OAAO,cAAc,CAAC;QACvB,CAAC;QAED,wCAAwC;QACxC,IACC,KAAK,CAAC,aAAa,KAAK,SAAS;YACjC,QAAQ,CAAC,UAAU,KAAK,SAAS;YACjC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,KAAK,CAAC,aAAa,EAClD,CAAC;YACF,OAAO,cAAc,CAAC;QACvB,CAAC;QAED,qBAAqB;QACrB,QAAQ,KAAK,CAAC,SAAS,EAAE,CAAC;YACzB,KAAK,SAAS,CAAC,CAAC,CAAC;gBAChB,IAAI,KAAK,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;oBACnC,OAAO,QAAQ,CAAC,KAAK,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC;gBACrE,CAAC;gBACD,OAAO,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC;YACtD,CAAC;YAED,KAAK,MAAM,CAAC,CAAC,CAAC;gBACb,IAAI,KAAK,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;oBACnC,OAAO,QAAQ,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC;gBACtE,CAAC;gBACD,OAAO,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC;YACtD,CAAC;YAED,KAAK,OAAO,CAAC,CAAC,CAAC;gBACd,MAAM,OAAO,GAAG,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC;gBACzC,OAAO,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC;YACvE,CAAC;QACF,CAAC;IACF,CAAC;IAED;;;;;OAKG;IACK,kBAAkB,CAAC,WAA8B;QACxD,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,MAAM,CAAC;QAC7E,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,MAAM,CAAC;QAC3E,MAAM,YAAY,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAC,MAAM,CAAC;QAEnF,MAAM,UAAU,GAAG,SAAS,GAAG,QAAQ,CAAC;QACxC,MAAM,gBAAgB,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;QAErE,OAAO;YACN,OAAO,EAAE,OAAO;YAChB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,WAAW;YACX,OAAO,EAAE;gBACR,KAAK,EAAE,WAAW,CAAC,MAAM;gBACzB,SAAS;gBACT,QAAQ;gBACR,YAAY;gBACZ,gBAAgB;aAChB;SACD,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,SAAS,CAAC,MAA6C;QACtD,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC;QAEhC,OAAO;YACN,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,SAAS;YACzB,MAAM,EAAE,OAAO,CAAC,QAAQ;YACxB,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,QAAQ,EAAE,OAAO,CAAC,gBAAgB;YAClC,UAAU,EAAE;gBACX,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;aAC1C;SACD,CAAC;IACH,CAAC"}
|