ppef 1.2.1 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +270 -6
- package/dist/__tests__/cli/evaluate-command.integration.test.js +60 -0
- package/dist/__tests__/cli/evaluate-command.integration.test.js.map +1 -1
- package/dist/__tests__/examples.integration.test.d.ts +8 -0
- package/dist/__tests__/examples.integration.test.d.ts.map +1 -0
- package/dist/__tests__/examples.integration.test.js +236 -0
- package/dist/__tests__/examples.integration.test.js.map +1 -0
- package/dist/cli/__tests__/commands.unit.test.js +12 -5
- package/dist/cli/__tests__/commands.unit.test.js.map +1 -1
- package/dist/cli/__tests__/config-loader.unit.test.js +30 -25
- package/dist/cli/__tests__/config-loader.unit.test.js.map +1 -1
- package/dist/cli/__tests__/evaluator-schemas.unit.test.d.ts +9 -0
- package/dist/cli/__tests__/evaluator-schemas.unit.test.d.ts.map +1 -0
- package/dist/cli/__tests__/evaluator-schemas.unit.test.js +334 -0
- package/dist/cli/__tests__/evaluator-schemas.unit.test.js.map +1 -0
- package/dist/cli/commands/aggregate.d.ts.map +1 -1
- package/dist/cli/commands/aggregate.js +20 -12
- package/dist/cli/commands/aggregate.js.map +1 -1
- package/dist/cli/commands/evaluate.d.ts.map +1 -1
- package/dist/cli/commands/evaluate.js +130 -24
- package/dist/cli/commands/evaluate.js.map +1 -1
- package/dist/cli/commands/plan.d.ts.map +1 -1
- package/dist/cli/commands/plan.js +40 -6
- package/dist/cli/commands/plan.js.map +1 -1
- package/dist/cli/commands/run.d.ts +9 -0
- package/dist/cli/commands/run.d.ts.map +1 -1
- package/dist/cli/commands/run.js +71 -12
- package/dist/cli/commands/run.js.map +1 -1
- package/dist/cli/commands/validate.d.ts.map +1 -1
- package/dist/cli/commands/validate.js +55 -0
- package/dist/cli/commands/validate.js.map +1 -1
- package/dist/cli/config-loader.d.ts +6 -3
- package/dist/cli/config-loader.d.ts.map +1 -1
- package/dist/cli/config-loader.js +31 -106
- package/dist/cli/config-loader.js.map +1 -1
- package/dist/cli/evaluator-schemas.d.ts +395 -0
- package/dist/cli/evaluator-schemas.d.ts.map +1 -0
- package/dist/cli/evaluator-schemas.js +285 -0
- package/dist/cli/evaluator-schemas.js.map +1 -0
- package/dist/cli/index.d.ts.map +1 -1
- package/dist/cli/index.js +11 -1
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/module-loader.d.ts.map +1 -1
- package/dist/cli/module-loader.js +38 -20
- package/dist/cli/module-loader.js.map +1 -1
- package/dist/cli/type-utils.d.ts +31 -0
- package/dist/cli/type-utils.d.ts.map +1 -0
- package/dist/cli/type-utils.js +38 -0
- package/dist/cli/type-utils.js.map +1 -0
- package/dist/cli/types.d.ts +284 -94
- package/dist/cli/types.d.ts.map +1 -1
- package/dist/cli/types.js +177 -1
- package/dist/cli/types.js.map +1 -1
- package/dist/collector/schema.js.map +1 -1
- package/dist/evaluators/claims-evaluator.d.ts.map +1 -1
- package/dist/evaluators/claims-evaluator.js +1 -1
- package/dist/evaluators/claims-evaluator.js.map +1 -1
- package/dist/evaluators/exploratory-evaluator.js.map +1 -1
- package/dist/executor/__tests__/worker-entry.integration.test.d.ts.map +1 -1
- package/dist/executor/__tests__/worker-entry.integration.test.js +19 -4
- package/dist/executor/__tests__/worker-entry.integration.test.js.map +1 -1
- package/dist/executor/binary-sut.d.ts.map +1 -1
- package/dist/executor/binary-sut.js +2 -1
- package/dist/executor/binary-sut.js.map +1 -1
- package/dist/executor/checkpoint-storage.d.ts.map +1 -1
- package/dist/executor/checkpoint-storage.js +13 -4
- package/dist/executor/checkpoint-storage.js.map +1 -1
- package/dist/executor/executor.d.ts +22 -0
- package/dist/executor/executor.d.ts.map +1 -1
- package/dist/executor/executor.js +133 -6
- package/dist/executor/executor.js.map +1 -1
- package/dist/executor/parallel-executor.d.ts.map +1 -1
- package/dist/executor/parallel-executor.js +9 -2
- package/dist/executor/parallel-executor.js.map +1 -1
- package/dist/executor/worker-entry.js +3 -1
- package/dist/executor/worker-entry.js.map +1 -1
- package/dist/executor/worker-executor.d.ts +9 -0
- package/dist/executor/worker-executor.d.ts.map +1 -1
- package/dist/executor/worker-executor.js +88 -9
- package/dist/executor/worker-executor.js.map +1 -1
- package/dist/executor/worker-threads-executor.d.ts.map +1 -1
- package/dist/executor/worker-threads-executor.js +52 -18
- package/dist/executor/worker-threads-executor.js.map +1 -1
- package/dist/index.cjs +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/renderers/latex-renderer.d.ts.map +1 -1
- package/dist/renderers/latex-renderer.js +20 -12
- package/dist/renderers/latex-renderer.js.map +1 -1
- package/dist/schemas/__tests__/json-schema-validator.unit.test.d.ts +8 -0
- package/dist/schemas/__tests__/json-schema-validator.unit.test.d.ts.map +1 -0
- package/dist/schemas/__tests__/json-schema-validator.unit.test.js +170 -0
- package/dist/schemas/__tests__/json-schema-validator.unit.test.js.map +1 -0
- package/dist/schemas/index.d.ts +7 -0
- package/dist/schemas/index.d.ts.map +1 -0
- package/dist/schemas/index.js +7 -0
- package/dist/schemas/index.js.map +1 -0
- package/dist/schemas/json-schema-validator.d.ts +59 -0
- package/dist/schemas/json-schema-validator.d.ts.map +1 -0
- package/dist/schemas/json-schema-validator.js +67 -0
- package/dist/schemas/json-schema-validator.js.map +1 -0
- package/dist/types/case.d.ts +4 -0
- package/dist/types/case.d.ts.map +1 -1
- package/dist/types/result.d.ts +2 -0
- package/dist/types/result.d.ts.map +1 -1
- package/dist/types/sut.d.ts +4 -0
- package/dist/types/sut.d.ts.map +1 -1
- package/package.json +15 -5
- package/ppef.schema.json +1178 -0
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
* Validates experiment configuration files without running experiments.
|
|
5
5
|
*/
|
|
6
6
|
import { loadAndValidateConfig } from "../config-loader.js";
|
|
7
|
+
import { createValidator } from "../../schemas/index.js";
|
|
7
8
|
// Note: createLogger imported lazily in action handler to avoid circular dependency
|
|
8
9
|
/**
|
|
9
10
|
* Execute validate command with injected dependencies.
|
|
@@ -51,6 +52,60 @@ export async function executeValidate(configFile, dependencies) {
|
|
|
51
52
|
logger.info(` Path: ${output.path ?? "./results"}`);
|
|
52
53
|
logger.info(` Format: ${output.format ?? "json-pretty"}`);
|
|
53
54
|
logger.info(` Aggregate: ${output.aggregate ?? "true"}`);
|
|
55
|
+
// Validate JSON Schemas compile correctly
|
|
56
|
+
const schemaErrors = [];
|
|
57
|
+
if (loaded.config.schemas?.input) {
|
|
58
|
+
try {
|
|
59
|
+
createValidator(loaded.config.schemas.input);
|
|
60
|
+
const props = loaded.config.schemas.input.properties;
|
|
61
|
+
const propCount = typeof props === "object" && props !== null ? Object.keys(props).length : 0;
|
|
62
|
+
logger.info("");
|
|
63
|
+
logger.info(`Input schema: compiles OK (${propCount} properties)`);
|
|
64
|
+
}
|
|
65
|
+
catch (e) {
|
|
66
|
+
schemaErrors.push(`schemas.input: ${e instanceof Error ? e.message : String(e)}`);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
if (loaded.config.schemas?.output) {
|
|
70
|
+
try {
|
|
71
|
+
createValidator(loaded.config.schemas.output);
|
|
72
|
+
const props = loaded.config.schemas.output.properties;
|
|
73
|
+
const propCount = typeof props === "object" && props !== null ? Object.keys(props).length : 0;
|
|
74
|
+
logger.info(`Output schema: compiles OK (${propCount} properties)`);
|
|
75
|
+
}
|
|
76
|
+
catch (e) {
|
|
77
|
+
schemaErrors.push(`schemas.output: ${e instanceof Error ? e.message : String(e)}`);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
for (const sut of loaded.config.suts) {
|
|
81
|
+
if (sut.outputSchema) {
|
|
82
|
+
try {
|
|
83
|
+
createValidator(sut.outputSchema);
|
|
84
|
+
logger.info(`SUT "${sut.id}" output schema: compiles OK`);
|
|
85
|
+
}
|
|
86
|
+
catch (e) {
|
|
87
|
+
schemaErrors.push(`suts[${sut.id}].outputSchema: ${e instanceof Error ? e.message : String(e)}`);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
for (const testCase of loaded.config.cases) {
|
|
92
|
+
if (testCase.inputSchema) {
|
|
93
|
+
try {
|
|
94
|
+
createValidator(testCase.inputSchema);
|
|
95
|
+
logger.info(`Case "${testCase.id}" input schema: compiles OK`);
|
|
96
|
+
}
|
|
97
|
+
catch (e) {
|
|
98
|
+
schemaErrors.push(`cases[${testCase.id}].inputSchema: ${e instanceof Error ? e.message : String(e)}`);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
if (schemaErrors.length > 0) {
|
|
103
|
+
logger.subheader("Schema Compilation Errors");
|
|
104
|
+
for (const err of schemaErrors) {
|
|
105
|
+
logger.error(` ${err}`);
|
|
106
|
+
}
|
|
107
|
+
processExit(1);
|
|
108
|
+
}
|
|
54
109
|
logger.info("");
|
|
55
110
|
logger.info("Configuration is valid!");
|
|
56
111
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"validate.js","sourceRoot":"","sources":["../../../src/cli/commands/validate.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"validate.js","sourceRoot":"","sources":["../../../src/cli/commands/validate.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAE5D,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAEzD,oFAAoF;AAEpF;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACpC,UAAkB,EAClB,YAIC;IAED,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,WAAW,EAAE,GAAG,YAAY,CAAC;IAE3D,IAAI,CAAC;QACJ,MAAM,CAAC,MAAM,CAAC,0BAA0B,CAAC,CAAC;QAE1C,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,qBAAqB,CAAC,UAAU,CAAC,CAAC;QAEpE,MAAM,CAAC,IAAI,CAAC,uBAAuB,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;QACxD,MAAM,CAAC,IAAI,CAAC,mBAAmB,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;QACjD,MAAM,CAAC,IAAI,CAAC,eAAe,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;QAE5D,IAAI,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,EAAE,CAAC;YAC1C,MAAM,CAAC,IAAI,CAAC,gBAAgB,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,EAAE,CAAC,CAAC;QACrE,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC,YAAY,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC;QAC7D,CAAC;QAED,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACzB,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,YAAY,CAAC,IAAI,KAAK,GAAG,CAAC,YAAY,CAAC,OAAO,GAAG,CAAC,CAAC;YACrF,MAAM,CAAC,IAAI,CAAC,eAAe,GAAG,CAAC,MAAM,OAAO,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;YAC9D,MAAM,CAAC,IAAI,CAAC,aAAa,GAAG,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;QACnD,CAAC;QAED,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAC1B,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YAC5C,MAAM,CAAC,IAAI,CAAC,OAAO,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC;YAClC,MAAM,CAAC,IAAI,CAAC,eAAe,QAAQ,CAAC,MAAM,OAAO,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QACzE,CAAC;QAED,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QAC7B,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC;QACxC,MAAM,CAAC,IAAI,CAAC,kBAAkB,QAAQ,CAAC,WAAW,IAAI,SAAS,EAAE,CAAC,CAAC;QACnE,MAAM,CAAC,IAAI,CAAC,gBAAgB,QAAQ,CAAC,QAAQ,IAAI,SAAS,EAAE,CAAC,CAAC;QAC9D,MAAM,CAAC,IAAI,CAAC,cAAc,QAAQ,CAAC,SAAS,IAAI,SAAS,IAAI,CAAC,CAAC;QAC/D,MAAM,CAAC,IAAI,CAAC,kBAAkB,QAAQ,CAAC,WAAW,IAAI,YAAY,EAAE,CAAC,CAAC;QACtE,MAAM,CAAC,IAAI,CAAC,wBAAwB,QAAQ,CAAC,eAAe,IAAI,SAAS,EAAE,CAAC,CAAC;QAC7E,MAAM,CAAC,IAAI,CAAC,yBAAyB,QAAQ,CAAC,iBAAiB,IAAI,SAAS,EAAE,CAAC,CAAC;QAEhF,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QAC3B,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC;QACpC,MAAM,CAAC,IAAI,CAAC,WAAW,MAAM,CAAC,IAAI,IAAI,WAAW,EAAE,CAAC,CAAC;QACrD,MAAM,CAAC,IAAI,CAAC,aAAa,MAAM,CAAC,MAAM,IAAI,aAAa,EAAE,CAAC,CAAC;QAC3D,MAAM,CAAC,IAAI,CAAC,gBAAgB,MAAM,CAAC,SAAS,IAAI,MAAM,EAAE,CAAC,CAAC;QAE1D,0CAA0C;QAC1C,MAAM,YAAY,GAAa,EAAE,CAAC;QAElC,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC;YAClC,IAAI,CAAC;gBACJ,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBAC7C,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC;gBACrD,MAAM,SAAS,GACd,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC7E,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAChB,MAAM,CAAC,IAAI,CAAC,8BAA8B,SAAS,cAAc,CAAC,CAAC;YACpE,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACZ,YAAY,CAAC,IAAI,CAAC,kBAAkB,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACnF,CAAC;QACF,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACnC,IAAI,CAAC;gBACJ,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAC9C,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC;gBACtD,MAAM,SAAS,GACd,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC7E,MAAM,CAAC,IAAI,CAAC,+BAA+B,SAAS,cAAc,CAAC,CAAC;YACrE,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACZ,YAAY,CAAC,IAAI,CAAC,mBAAmB,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACpF,CAAC;QACF,CAAC;QAED,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACtC,IAAI,GAAG,CAAC,YAAY,EAAE,CAAC;gBACtB,IAAI,CAAC;oBACJ,eAAe,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;oBAClC,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,EAAE,8BAA8B,CAAC,CAAC;gBAC3D,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACZ,YAAY,CAAC,IAAI,CAChB,QAAQ,GAAG,CAAC,EAAE,mBAAmB,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAC7E,CAAC;gBACH,CAAC;YACF,CAAC;QACF,CAAC;QAED,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YAC5C,IAAI,QAAQ,CAAC,WAAW,EAAE,CAAC;gBAC1B,IAAI,CAAC;oBACJ,eAAe,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;oBACtC,MAAM,CAAC,IAAI,CAAC,SAAS,QAAQ,CAAC,EAAE,6BAA6B,CAAC,CAAC;gBAChE,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACZ,YAAY,CAAC,IAAI,CAChB,SAAS,QAAQ,CAAC,EAAE,kBAAkB,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAClF,CAAC;gBACH,CAAC;YACF,CAAC;QACF,CAAC;QAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,CAAC,SAAS,CAAC,2BAA2B,CAAC,CAAC;YAC9C,KAAK,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;gBAChC,MAAM,CAAC,KAAK,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;YAC1B,CAAC;YACD,WAAW,CAAC,CAAC,CAAC,CAAC;QAChB,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAChB,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;IACxC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC5B,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC7B,CAAC;aAAM,CAAC;YACP,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QAC7B,CAAC;QACD,WAAW,CAAC,CAAC,CAAC,CAAC;IAChB,CAAC;AACF,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,uBAAuB,CAAC,OAAgB;IACvD,OAAO;SACL,OAAO,CAAC,UAAU,CAAC;SACnB,WAAW,CAAC,2CAA2C,CAAC;SACxD,QAAQ,CAAC,eAAe,EAAE,4CAA4C,CAAC;SACvE,MAAM,CAAC,KAAK,EAAE,UAAkB,EAAE,EAAE;QACpC,uDAAuD;QACvD,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC;QACtD,MAAM,MAAM,GAAG,YAAY,EAAE,CAAC;QAE9B,MAAM,eAAe,CAAC,UAAU,EAAE;YACjC,MAAM;YACN,YAAY,EAAE,EAAE,qBAAqB,EAAE;YACvC,WAAW,EAAE,CAAC,IAAY,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;SACjD,CAAC,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Loads and validates experiment configuration from JSON files.
|
|
5
5
|
*/
|
|
6
|
-
import type {
|
|
6
|
+
import type { LoadedConfig, ValidationResult } from "./types.js";
|
|
7
7
|
/**
|
|
8
8
|
* Load and parse experiment configuration from a JSON file.
|
|
9
9
|
*
|
|
@@ -15,10 +15,13 @@ export declare function loadConfig(configPath: string): Promise<LoadedConfig>;
|
|
|
15
15
|
/**
|
|
16
16
|
* Validate experiment configuration.
|
|
17
17
|
*
|
|
18
|
-
*
|
|
18
|
+
* Accepts raw parsed JSON (unknown) and validates it against the Zod schema.
|
|
19
|
+
* Returns a ValidationResult with errors and warnings for backward compatibility.
|
|
20
|
+
*
|
|
21
|
+
* @param config - Configuration to validate (unknown input from JSON)
|
|
19
22
|
* @returns Validation result with errors and warnings
|
|
20
23
|
*/
|
|
21
|
-
export declare function validateConfig(config:
|
|
24
|
+
export declare function validateConfig(config: unknown): ValidationResult;
|
|
22
25
|
/**
|
|
23
26
|
* Load and validate experiment configuration.
|
|
24
27
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config-loader.d.ts","sourceRoot":"","sources":["../../src/cli/config-loader.ts"],"names":[],"mappings":"AAAA;;;;GAIG;
|
|
1
|
+
{"version":3,"file":"config-loader.d.ts","sourceRoot":"","sources":["../../src/cli/config-loader.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAMH,OAAO,KAAK,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AASjE;;;;;;GAMG;AACH,wBAAsB,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAiB1E;AAED;;;;;;;;GAQG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,OAAO,GAAG,gBAAgB,CAyChE;AAED;;;;;;GAMG;AACH,wBAAsB,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAkBrF"}
|
|
@@ -5,6 +5,13 @@
|
|
|
5
5
|
*/
|
|
6
6
|
import { readFile } from "node:fs/promises";
|
|
7
7
|
import { resolve, dirname } from "node:path";
|
|
8
|
+
import { ExperimentConfig } from "./types.js";
|
|
9
|
+
/**
|
|
10
|
+
* Type guard for plain objects (Record<string, unknown>).
|
|
11
|
+
*/
|
|
12
|
+
function isRecord(value) {
|
|
13
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
14
|
+
}
|
|
8
15
|
/**
|
|
9
16
|
* Load and parse experiment configuration from a JSON file.
|
|
10
17
|
*
|
|
@@ -17,6 +24,7 @@ export async function loadConfig(configPath) {
|
|
|
17
24
|
const absolutePath = resolve(configPath);
|
|
18
25
|
// Read and parse JSON
|
|
19
26
|
const content = await readFile(absolutePath, "utf-8");
|
|
27
|
+
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment -- JSON.parse returns any; Zod validates in validateExperimentConfig
|
|
20
28
|
const config = JSON.parse(content);
|
|
21
29
|
// Get base directory (for resolving module paths)
|
|
22
30
|
const baseDir = dirname(absolutePath);
|
|
@@ -29,126 +37,43 @@ export async function loadConfig(configPath) {
|
|
|
29
37
|
/**
|
|
30
38
|
* Validate experiment configuration.
|
|
31
39
|
*
|
|
32
|
-
*
|
|
40
|
+
* Accepts raw parsed JSON (unknown) and validates it against the Zod schema.
|
|
41
|
+
* Returns a ValidationResult with errors and warnings for backward compatibility.
|
|
42
|
+
*
|
|
43
|
+
* @param config - Configuration to validate (unknown input from JSON)
|
|
33
44
|
* @returns Validation result with errors and warnings
|
|
34
45
|
*/
|
|
35
46
|
export function validateConfig(config) {
|
|
36
47
|
const errors = [];
|
|
37
48
|
const warnings = [];
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
// Validate executor config
|
|
44
|
-
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
|
45
|
-
if (!config.executor) {
|
|
46
|
-
errors.push("executor configuration is required");
|
|
47
|
-
}
|
|
48
|
-
else {
|
|
49
|
-
if (config.executor.repetitions !== undefined && config.executor.repetitions < 1) {
|
|
50
|
-
errors.push("executor.repetitions must be at least 1");
|
|
51
|
-
}
|
|
52
|
-
if (config.executor.seedBase !== undefined && config.executor.seedBase < 0) {
|
|
53
|
-
errors.push("executor.seedBase must be non-negative");
|
|
54
|
-
}
|
|
55
|
-
if (config.executor.timeoutMs !== undefined && config.executor.timeoutMs < 0) {
|
|
56
|
-
errors.push("executor.timeoutMs must be non-negative");
|
|
57
|
-
}
|
|
58
|
-
if (config.executor.concurrency !== undefined && config.executor.concurrency < 1) {
|
|
59
|
-
errors.push("executor.concurrency must be at least 1");
|
|
49
|
+
const result = ExperimentConfig.safeParse(config);
|
|
50
|
+
if (!result.success) {
|
|
51
|
+
for (const issue of result.error.issues) {
|
|
52
|
+
const path = issue.path.join(".");
|
|
53
|
+
errors.push(path ? `${path}: ${issue.message}` : issue.message);
|
|
60
54
|
}
|
|
61
55
|
}
|
|
62
|
-
//
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
}
|
|
67
|
-
else {
|
|
68
|
-
for (let i = 0; i < config.suts.length; i++) {
|
|
69
|
-
const sut = config.suts[i];
|
|
70
|
-
if (!sut.id) {
|
|
71
|
-
errors.push(`suts[${i}].id is required`);
|
|
72
|
-
}
|
|
73
|
-
if (!sut.module) {
|
|
74
|
-
errors.push(`suts[${i}].module is required`);
|
|
75
|
-
}
|
|
76
|
-
if (!sut.exportName) {
|
|
77
|
-
errors.push(`suts[${i}].exportName is required`);
|
|
78
|
-
}
|
|
79
|
-
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
|
80
|
-
if (!sut.registration?.name) {
|
|
81
|
-
errors.push(`suts[${i}].registration.name is required`);
|
|
82
|
-
}
|
|
83
|
-
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
|
84
|
-
if (!sut.registration?.version) {
|
|
85
|
-
errors.push(`suts[${i}].registration.version is required`);
|
|
86
|
-
}
|
|
87
|
-
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
|
88
|
-
if (!sut.registration?.role) {
|
|
89
|
-
errors.push(`suts[${i}].registration.role is required`);
|
|
90
|
-
}
|
|
91
|
-
else if (!["primary", "baseline", "oracle"].includes(sut.registration.role)) {
|
|
92
|
-
errors.push(`suts[${i}].registration.role must be one of: primary, baseline, oracle`);
|
|
93
|
-
}
|
|
56
|
+
// Post-parse warnings (Zod doesn't have a warnings concept)
|
|
57
|
+
if (result.success) {
|
|
58
|
+
if (result.data.suts.length === 0) {
|
|
59
|
+
warnings.push("No SUTs configured - experiment will have nothing to execute");
|
|
94
60
|
}
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
for (const sut of config.suts) {
|
|
98
|
-
if (sut.id && sutIds.has(sut.id)) {
|
|
99
|
-
errors.push(`Duplicate SUT ID: ${sut.id}`);
|
|
100
|
-
}
|
|
101
|
-
sutIds.add(sut.id);
|
|
61
|
+
if (result.data.cases.length === 0) {
|
|
62
|
+
warnings.push("No cases configured - experiment will have nothing to execute");
|
|
102
63
|
}
|
|
103
64
|
}
|
|
104
|
-
// Validate cases
|
|
105
|
-
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
|
106
|
-
if (!config.cases || config.cases.length === 0) {
|
|
107
|
-
warnings.push("No cases configured - experiment will have nothing to execute");
|
|
108
|
-
}
|
|
109
65
|
else {
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
if (!
|
|
113
|
-
|
|
114
|
-
}
|
|
115
|
-
if (!testCase.module) {
|
|
116
|
-
errors.push(`cases[${i}].module is required`);
|
|
66
|
+
// Even on parse failure, try to extract warnings from the raw input
|
|
67
|
+
if (isRecord(config)) {
|
|
68
|
+
if (!config.output) {
|
|
69
|
+
warnings.push("No output configuration specified - using defaults");
|
|
117
70
|
}
|
|
118
|
-
if (
|
|
119
|
-
|
|
71
|
+
if (Array.isArray(config.suts) && config.suts.length === 0) {
|
|
72
|
+
warnings.push("No SUTs configured - experiment will have nothing to execute");
|
|
120
73
|
}
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
const caseIds = new Set();
|
|
124
|
-
for (const testCase of config.cases) {
|
|
125
|
-
if (testCase.id && caseIds.has(testCase.id)) {
|
|
126
|
-
errors.push(`Duplicate case ID: ${testCase.id}`);
|
|
74
|
+
if (Array.isArray(config.cases) && config.cases.length === 0) {
|
|
75
|
+
warnings.push("No cases configured - experiment will have nothing to execute");
|
|
127
76
|
}
|
|
128
|
-
caseIds.add(testCase.id);
|
|
129
|
-
}
|
|
130
|
-
}
|
|
131
|
-
// Validate metrics extractor
|
|
132
|
-
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
|
133
|
-
if (!config.metricsExtractor) {
|
|
134
|
-
errors.push("metricsExtractor configuration is required");
|
|
135
|
-
}
|
|
136
|
-
else {
|
|
137
|
-
if (!config.metricsExtractor.module) {
|
|
138
|
-
errors.push("metricsExtractor.module is required");
|
|
139
|
-
}
|
|
140
|
-
if (!config.metricsExtractor.exportName) {
|
|
141
|
-
errors.push("metricsExtractor.exportName is required");
|
|
142
|
-
}
|
|
143
|
-
}
|
|
144
|
-
// Validate output config
|
|
145
|
-
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
|
146
|
-
if (!config.output) {
|
|
147
|
-
warnings.push("No output configuration specified - using defaults");
|
|
148
|
-
}
|
|
149
|
-
else {
|
|
150
|
-
if (config.output.format && !["json", "json-pretty"].includes(config.output.format)) {
|
|
151
|
-
errors.push('output.format must be one of: "json", "json-pretty"');
|
|
152
77
|
}
|
|
153
78
|
}
|
|
154
79
|
return {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config-loader.js","sourceRoot":"","sources":["../../src/cli/config-loader.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"config-loader.js","sourceRoot":"","sources":["../../src/cli/config-loader.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAE7C,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAG9C;;GAEG;AACH,SAAS,QAAQ,CAAC,KAAc;IAC/B,OAAO,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC7E,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,UAAkB;IAClD,wBAAwB;IACxB,MAAM,YAAY,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAEzC,sBAAsB;IACtB,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IACtD,wIAAwI;IACxI,MAAM,MAAM,GAAqB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAErD,kDAAkD;IAClD,MAAM,OAAO,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IAEtC,OAAO;QACN,MAAM;QACN,OAAO;QACP,UAAU,EAAE,YAAY;KACxB,CAAC;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,cAAc,CAAC,MAAe;IAC7C,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAElD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACrB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YACzC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAClC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACjE,CAAC;IACF,CAAC;IAED,4DAA4D;IAC5D,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnC,QAAQ,CAAC,IAAI,CAAC,8DAA8D,CAAC,CAAC;QAC/E,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACpC,QAAQ,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;QAChF,CAAC;IACF,CAAC;SAAM,CAAC;QACP,oEAAoE;QACpE,IAAI,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACtB,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;gBACpB,QAAQ,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAC;YACrE,CAAC;YACD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC5D,QAAQ,CAAC,IAAI,CAAC,8DAA8D,CAAC,CAAC;YAC/E,CAAC;YACD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC9D,QAAQ,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;YAChF,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO;QACN,KAAK,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC;QAC1B,MAAM;QACN,QAAQ;KACR,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,UAAkB;IAC7D,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC,CAAC;IAC5C,MAAM,UAAU,GAAG,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAEjD,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACd,qCAAqC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC1F,CAAC;IACH,CAAC;IAED,wBAAwB;IACxB,IAAI,UAAU,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpC,KAAK,MAAM,OAAO,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC;YAC3C,OAAO,CAAC,IAAI,CAAC,YAAY,OAAO,EAAE,CAAC,CAAC;QACrC,CAAC;IACF,CAAC;IAED,OAAO,MAAM,CAAC;AACf,CAAC"}
|