ppef 1.0.1 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/ppef.mjs +20 -0
- package/dist/__tests__/framework-pipeline.integration.test.js +13 -11
- package/dist/__tests__/framework-pipeline.integration.test.js.map +1 -1
- package/dist/__tests__/index-exports.unit.test.d.ts +8 -0
- package/dist/__tests__/index-exports.unit.test.d.ts.map +1 -0
- package/dist/__tests__/index-exports.unit.test.js +127 -0
- package/dist/__tests__/index-exports.unit.test.js.map +1 -0
- package/dist/__tests__/registry-executor.integration.test.js +12 -9
- package/dist/__tests__/registry-executor.integration.test.js.map +1 -1
- package/dist/aggregation/__tests__/aggregators.unit.test.d.ts +7 -0
- package/dist/aggregation/__tests__/aggregators.unit.test.d.ts.map +1 -0
- package/dist/aggregation/__tests__/aggregators.unit.test.js +350 -0
- package/dist/aggregation/__tests__/aggregators.unit.test.js.map +1 -0
- package/dist/aggregation/__tests__/pipeline.unit.test.d.ts +7 -0
- package/dist/aggregation/__tests__/pipeline.unit.test.d.ts.map +1 -0
- package/dist/aggregation/__tests__/pipeline.unit.test.js +213 -0
- package/dist/aggregation/__tests__/pipeline.unit.test.js.map +1 -0
- package/dist/aggregation/aggregators.d.ts +9 -0
- package/dist/aggregation/aggregators.d.ts.map +1 -1
- package/dist/aggregation/aggregators.js +1 -1
- package/dist/aggregation/aggregators.js.map +1 -1
- package/dist/aggregation/index.d.ts +1 -1
- package/dist/aggregation/index.d.ts.map +1 -1
- package/dist/aggregation/index.js +1 -1
- package/dist/aggregation/index.js.map +1 -1
- package/dist/claims/__tests__/evaluator.unit.test.d.ts +12 -0
- package/dist/claims/__tests__/evaluator.unit.test.d.ts.map +1 -0
- package/dist/claims/__tests__/evaluator.unit.test.js +801 -0
- package/dist/claims/__tests__/evaluator.unit.test.js.map +1 -0
- package/dist/cli/__tests__/aggregate.command.unit.test.d.ts +7 -0
- package/dist/cli/__tests__/aggregate.command.unit.test.d.ts.map +1 -0
- package/dist/cli/__tests__/aggregate.command.unit.test.js +396 -0
- package/dist/cli/__tests__/aggregate.command.unit.test.js.map +1 -0
- package/dist/cli/__tests__/commands.unit.test.d.ts +10 -0
- package/dist/cli/__tests__/commands.unit.test.d.ts.map +1 -0
- package/dist/cli/__tests__/commands.unit.test.js +217 -0
- package/dist/cli/__tests__/commands.unit.test.js.map +1 -0
- package/dist/cli/__tests__/index.unit.test.d.ts +10 -0
- package/dist/cli/__tests__/index.unit.test.d.ts.map +1 -0
- package/dist/cli/__tests__/index.unit.test.js +65 -0
- package/dist/cli/__tests__/index.unit.test.js.map +1 -0
- package/dist/cli/__tests__/logger.unit.test.d.ts +11 -0
- package/dist/cli/__tests__/logger.unit.test.d.ts.map +1 -0
- package/dist/cli/__tests__/logger.unit.test.js +180 -0
- package/dist/cli/__tests__/logger.unit.test.js.map +1 -0
- package/dist/cli/__tests__/module-loader.unit.test.d.ts +11 -0
- package/dist/cli/__tests__/module-loader.unit.test.d.ts.map +1 -0
- package/dist/cli/__tests__/module-loader.unit.test.js +262 -0
- package/dist/cli/__tests__/module-loader.unit.test.js.map +1 -0
- package/dist/cli/__tests__/output-writer.unit.test.d.ts +10 -0
- package/dist/cli/__tests__/output-writer.unit.test.d.ts.map +1 -0
- package/dist/cli/__tests__/output-writer.unit.test.js +216 -0
- package/dist/cli/__tests__/output-writer.unit.test.js.map +1 -0
- package/dist/cli/__tests__/plan.command.unit.test.d.ts +7 -0
- package/dist/cli/__tests__/plan.command.unit.test.d.ts.map +1 -0
- package/dist/cli/__tests__/plan.command.unit.test.js +289 -0
- package/dist/cli/__tests__/plan.command.unit.test.js.map +1 -0
- package/dist/cli/__tests__/run.command.unit.test.d.ts +7 -0
- package/dist/cli/__tests__/run.command.unit.test.d.ts.map +1 -0
- package/dist/cli/__tests__/run.command.unit.test.js +422 -0
- package/dist/cli/__tests__/run.command.unit.test.js.map +1 -0
- package/dist/cli/__tests__/validate.command.unit.test.d.ts +7 -0
- package/dist/cli/__tests__/validate.command.unit.test.d.ts.map +1 -0
- package/dist/cli/__tests__/validate.command.unit.test.js +226 -0
- package/dist/cli/__tests__/validate.command.unit.test.js.map +1 -0
- package/dist/cli/command-deps.d.ts +125 -0
- package/dist/cli/command-deps.d.ts.map +1 -0
- package/dist/cli/command-deps.js +7 -0
- package/dist/cli/command-deps.js.map +1 -0
- package/dist/cli/commands/aggregate.d.ts +35 -0
- package/dist/cli/commands/aggregate.d.ts.map +1 -0
- package/dist/cli/commands/aggregate.js +121 -0
- package/dist/cli/commands/aggregate.js.map +1 -0
- package/dist/cli/commands/plan.d.ts +36 -0
- package/dist/cli/commands/plan.d.ts.map +1 -0
- package/dist/cli/commands/plan.js +109 -0
- package/dist/cli/commands/plan.js.map +1 -0
- package/dist/cli/commands/run.d.ts +33 -0
- package/dist/cli/commands/run.d.ts.map +1 -0
- package/dist/cli/commands/run.js +185 -0
- package/dist/cli/commands/run.js.map +1 -0
- package/dist/cli/commands/validate.d.ts +27 -0
- package/dist/cli/commands/validate.d.ts.map +1 -0
- package/dist/cli/commands/validate.js +88 -0
- package/dist/cli/commands/validate.js.map +1 -0
- package/dist/cli/config-loader.d.ts +30 -0
- package/dist/cli/config-loader.d.ts.map +1 -0
- package/dist/cli/config-loader.js +181 -0
- package/dist/cli/config-loader.js.map +1 -0
- package/dist/cli/index.d.ts +26 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/cli/index.js +58 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/cli/logger.d.ts +75 -0
- package/dist/cli/logger.d.ts.map +1 -0
- package/dist/cli/logger.js +131 -0
- package/dist/cli/logger.js.map +1 -0
- package/dist/cli/module-loader.d.ts +46 -0
- package/dist/cli/module-loader.d.ts.map +1 -0
- package/dist/cli/module-loader.js +116 -0
- package/dist/cli/module-loader.js.map +1 -0
- package/dist/cli/output-writer.d.ts +51 -0
- package/dist/cli/output-writer.d.ts.map +1 -0
- package/dist/cli/output-writer.js +65 -0
- package/dist/cli/output-writer.js.map +1 -0
- package/dist/cli/types.d.ts +174 -0
- package/dist/cli/types.d.ts.map +1 -0
- package/dist/cli/types.js +7 -0
- package/dist/cli/types.js.map +1 -0
- package/dist/collector/__tests__/result-collector.unit.test.d.ts +7 -0
- package/dist/collector/__tests__/result-collector.unit.test.d.ts.map +1 -0
- package/dist/collector/__tests__/result-collector.unit.test.js +1021 -0
- package/dist/collector/__tests__/result-collector.unit.test.js.map +1 -0
- package/dist/collector/__tests__/schema.unit.test.d.ts +7 -0
- package/dist/collector/__tests__/schema.unit.test.d.ts.map +1 -0
- package/dist/collector/__tests__/schema.unit.test.js +360 -0
- package/dist/collector/__tests__/schema.unit.test.js.map +1 -0
- package/dist/executor/__tests__/checkpoint-manager.unit.test.js +83 -1
- package/dist/executor/__tests__/checkpoint-manager.unit.test.js.map +1 -1
- package/dist/executor/__tests__/checkpoint-merge-bug.diagnostic.test.d.ts +3 -6
- package/dist/executor/__tests__/checkpoint-merge-bug.diagnostic.test.d.ts.map +1 -1
- package/dist/executor/__tests__/checkpoint-merge-bug.diagnostic.test.js +428 -159
- package/dist/executor/__tests__/checkpoint-merge-bug.diagnostic.test.js.map +1 -1
- package/dist/executor/__tests__/checkpoint-storage.unit.test.js +105 -1
- package/dist/executor/__tests__/checkpoint-storage.unit.test.js.map +1 -1
- package/dist/executor/__tests__/executor.unit.test.js +69 -1
- package/dist/executor/__tests__/executor.unit.test.js.map +1 -1
- package/dist/executor/__tests__/memory-monitor.unit.test.d.ts +7 -0
- package/dist/executor/__tests__/memory-monitor.unit.test.d.ts.map +1 -0
- package/dist/executor/__tests__/memory-monitor.unit.test.js +285 -0
- package/dist/executor/__tests__/memory-monitor.unit.test.js.map +1 -0
- package/dist/executor/__tests__/parallel-executor.unit.test.d.ts +2 -1
- package/dist/executor/__tests__/parallel-executor.unit.test.d.ts.map +1 -1
- package/dist/executor/__tests__/parallel-executor.unit.test.js +426 -156
- package/dist/executor/__tests__/parallel-executor.unit.test.js.map +1 -1
- package/dist/executor/__tests__/run-id.unit.test.d.ts +8 -0
- package/dist/executor/__tests__/run-id.unit.test.d.ts.map +1 -0
- package/dist/executor/__tests__/run-id.unit.test.js +156 -0
- package/dist/executor/__tests__/run-id.unit.test.js.map +1 -0
- package/dist/executor/__tests__/worker-entry.integration.test.d.ts +24 -0
- package/dist/executor/__tests__/worker-entry.integration.test.d.ts.map +1 -0
- package/dist/executor/__tests__/worker-entry.integration.test.js +82 -0
- package/dist/executor/__tests__/worker-entry.integration.test.js.map +1 -0
- package/dist/executor/__tests__/worker-entry.unit.test.d.ts +7 -0
- package/dist/executor/__tests__/worker-entry.unit.test.d.ts.map +1 -0
- package/dist/executor/__tests__/worker-entry.unit.test.js +364 -0
- package/dist/executor/__tests__/worker-entry.unit.test.js.map +1 -0
- package/dist/executor/parallel-executor.d.ts +186 -0
- package/dist/executor/parallel-executor.d.ts.map +1 -1
- package/dist/executor/parallel-executor.js +218 -83
- package/dist/executor/parallel-executor.js.map +1 -1
- package/dist/executor/run-id.d.ts.map +1 -1
- package/dist/executor/run-id.js +8 -1
- package/dist/executor/run-id.js.map +1 -1
- package/dist/executor/worker-entry.d.ts +2 -0
- package/dist/executor/worker-entry.d.ts.map +1 -1
- package/dist/executor/worker-entry.js +29 -54
- package/dist/executor/worker-entry.js.map +1 -1
- package/dist/executor/worker-executor.d.ts +156 -0
- package/dist/executor/worker-executor.d.ts.map +1 -0
- package/dist/executor/worker-executor.js +88 -0
- package/dist/executor/worker-executor.js.map +1 -0
- package/dist/robustness/__tests__/analyzer.unit.test.d.ts +11 -0
- package/dist/robustness/__tests__/analyzer.unit.test.d.ts.map +1 -0
- package/dist/robustness/__tests__/analyzer.unit.test.js +455 -0
- package/dist/robustness/__tests__/analyzer.unit.test.js.map +1 -0
- package/dist/robustness/__tests__/perturbations.unit.test.d.ts +11 -0
- package/dist/robustness/__tests__/perturbations.unit.test.d.ts.map +1 -0
- package/dist/robustness/__tests__/perturbations.unit.test.js +284 -0
- package/dist/robustness/__tests__/perturbations.unit.test.js.map +1 -0
- package/dist/statistical/__tests__/mann-whitney-u.unit.test.d.ts +7 -0
- package/dist/statistical/__tests__/mann-whitney-u.unit.test.d.ts.map +1 -0
- package/dist/statistical/__tests__/mann-whitney-u.unit.test.js +185 -0
- package/dist/statistical/__tests__/mann-whitney-u.unit.test.js.map +1 -0
- package/package.json +8 -1
|
@@ -0,0 +1,1021 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Unit tests for ResultCollector
|
|
3
|
+
*
|
|
4
|
+
* Tests result collection, validation, querying, and serialization.
|
|
5
|
+
*/
|
|
6
|
+
import { beforeEach, describe, it } from "node:test";
|
|
7
|
+
import { strict as assert } from "node:assert";
|
|
8
|
+
import { ResultCollector } from "../result-collector.js";
|
|
9
|
+
import { createMinimalValidResult, createMockResult, createMockResults, } from "../../__tests__/test-helpers.js";
|
|
10
|
+
describe("ResultCollector", () => {
|
|
11
|
+
let collector;
|
|
12
|
+
beforeEach(() => {
|
|
13
|
+
collector = new ResultCollector();
|
|
14
|
+
});
|
|
15
|
+
describe("record", () => {
|
|
16
|
+
it("should add a valid result", () => {
|
|
17
|
+
const result = createMockResult();
|
|
18
|
+
collector.record(result);
|
|
19
|
+
assert.strictEqual(collector.count, 1);
|
|
20
|
+
});
|
|
21
|
+
it("should throw on result with empty runId", () => {
|
|
22
|
+
const result = createMockResult({
|
|
23
|
+
run: { ...createMockResult().run, runId: "" },
|
|
24
|
+
});
|
|
25
|
+
assert.throws(() => {
|
|
26
|
+
collector.record(result);
|
|
27
|
+
}, /Invalid result/);
|
|
28
|
+
});
|
|
29
|
+
it("should throw on result with empty sut", () => {
|
|
30
|
+
const result = createMockResult({
|
|
31
|
+
run: { ...createMockResult().run, sut: "" },
|
|
32
|
+
});
|
|
33
|
+
assert.throws(() => {
|
|
34
|
+
collector.record(result);
|
|
35
|
+
}, /Invalid result/);
|
|
36
|
+
});
|
|
37
|
+
it("should throw on result with empty caseId", () => {
|
|
38
|
+
const result = createMockResult({
|
|
39
|
+
run: { ...createMockResult().run, caseId: "" },
|
|
40
|
+
});
|
|
41
|
+
assert.throws(() => {
|
|
42
|
+
collector.record(result);
|
|
43
|
+
}, /Invalid result/);
|
|
44
|
+
});
|
|
45
|
+
it("should throw on result with no numeric metrics", () => {
|
|
46
|
+
const result = createMockResult({
|
|
47
|
+
metrics: { numeric: {} },
|
|
48
|
+
});
|
|
49
|
+
assert.throws(() => {
|
|
50
|
+
collector.record(result);
|
|
51
|
+
}, /Invalid result/);
|
|
52
|
+
});
|
|
53
|
+
it("should throw on result with empty platform", () => {
|
|
54
|
+
const result = createMockResult({
|
|
55
|
+
provenance: {
|
|
56
|
+
...createMockResult().provenance,
|
|
57
|
+
runtime: {
|
|
58
|
+
...createMockResult().provenance.runtime,
|
|
59
|
+
platform: "",
|
|
60
|
+
},
|
|
61
|
+
},
|
|
62
|
+
});
|
|
63
|
+
assert.throws(() => {
|
|
64
|
+
collector.record(result);
|
|
65
|
+
}, /Invalid result/);
|
|
66
|
+
});
|
|
67
|
+
it("should throw on result with multiple validation errors", () => {
|
|
68
|
+
const result = createMockResult({
|
|
69
|
+
run: { runId: "", sut: "", sutRole: "primary", caseId: "" },
|
|
70
|
+
metrics: { numeric: {} },
|
|
71
|
+
provenance: {
|
|
72
|
+
runtime: {
|
|
73
|
+
platform: "",
|
|
74
|
+
arch: "arm64",
|
|
75
|
+
nodeVersion: "20.0.0",
|
|
76
|
+
},
|
|
77
|
+
timestamp: new Date().toISOString(),
|
|
78
|
+
},
|
|
79
|
+
});
|
|
80
|
+
assert.throws(() => {
|
|
81
|
+
collector.record(result);
|
|
82
|
+
}, /Invalid result/);
|
|
83
|
+
});
|
|
84
|
+
it("should accept minimal valid result with at least one metric", () => {
|
|
85
|
+
const result = createMinimalValidResult();
|
|
86
|
+
// createMinimalValidResult creates empty metrics, so add one
|
|
87
|
+
result.metrics.numeric = { test: 1 };
|
|
88
|
+
collector.record(result);
|
|
89
|
+
assert.strictEqual(collector.count, 1);
|
|
90
|
+
});
|
|
91
|
+
it("should accept result with undefined caseClass", () => {
|
|
92
|
+
const result = createMockResult({
|
|
93
|
+
run: { ...createMockResult().run, caseClass: undefined },
|
|
94
|
+
});
|
|
95
|
+
collector.record(result);
|
|
96
|
+
assert.strictEqual(collector.count, 1);
|
|
97
|
+
});
|
|
98
|
+
it("should accept result with optional fields missing", () => {
|
|
99
|
+
const result = {
|
|
100
|
+
run: {
|
|
101
|
+
runId: "test-001",
|
|
102
|
+
sut: "test-sut",
|
|
103
|
+
sutRole: "primary",
|
|
104
|
+
caseId: "case-001",
|
|
105
|
+
},
|
|
106
|
+
correctness: {
|
|
107
|
+
expectedExists: true,
|
|
108
|
+
producedOutput: true,
|
|
109
|
+
valid: true,
|
|
110
|
+
matchesExpected: true,
|
|
111
|
+
},
|
|
112
|
+
outputs: { summary: {} },
|
|
113
|
+
metrics: { numeric: { time: 100 } },
|
|
114
|
+
provenance: {
|
|
115
|
+
runtime: {
|
|
116
|
+
platform: "test",
|
|
117
|
+
arch: "test",
|
|
118
|
+
nodeVersion: "20.0.0",
|
|
119
|
+
},
|
|
120
|
+
timestamp: new Date().toISOString(),
|
|
121
|
+
},
|
|
122
|
+
};
|
|
123
|
+
collector.record(result);
|
|
124
|
+
assert.strictEqual(collector.count, 1);
|
|
125
|
+
});
|
|
126
|
+
});
|
|
127
|
+
describe("recordBatch", () => {
|
|
128
|
+
it("should add multiple valid results", () => {
|
|
129
|
+
const results = createMockResults(5, "sut-v1", "primary", "test-class");
|
|
130
|
+
collector.recordBatch(results);
|
|
131
|
+
assert.strictEqual(collector.count, 5);
|
|
132
|
+
});
|
|
133
|
+
it("should throw when batch contains invalid result", () => {
|
|
134
|
+
const invalidResult = createMockResult({
|
|
135
|
+
run: { runId: "", sut: "bad", sutRole: "primary", caseId: "bad" },
|
|
136
|
+
metrics: { numeric: {} },
|
|
137
|
+
provenance: {
|
|
138
|
+
runtime: {
|
|
139
|
+
platform: "",
|
|
140
|
+
arch: "arm64",
|
|
141
|
+
nodeVersion: "20.0.0",
|
|
142
|
+
},
|
|
143
|
+
timestamp: new Date().toISOString(),
|
|
144
|
+
},
|
|
145
|
+
});
|
|
146
|
+
assert.throws(() => {
|
|
147
|
+
collector.recordBatch([invalidResult]);
|
|
148
|
+
});
|
|
149
|
+
assert.strictEqual(collector.count, 0);
|
|
150
|
+
});
|
|
151
|
+
it("should handle empty batch", () => {
|
|
152
|
+
collector.recordBatch([]);
|
|
153
|
+
assert.strictEqual(collector.count, 0);
|
|
154
|
+
});
|
|
155
|
+
it("should add results from different SUTs", () => {
|
|
156
|
+
const batch1 = createMockResults(2, "sut-1", "primary");
|
|
157
|
+
const batch2 = createMockResults(3, "sut-2", "baseline");
|
|
158
|
+
collector.recordBatch([...batch1, ...batch2]);
|
|
159
|
+
assert.strictEqual(collector.count, 5);
|
|
160
|
+
});
|
|
161
|
+
it("should preserve insertion order", () => {
|
|
162
|
+
const results = createMockResults(3, "sut-v1");
|
|
163
|
+
collector.recordBatch(results);
|
|
164
|
+
const all = collector.getAll();
|
|
165
|
+
assert.strictEqual(all[0].run.runId, results[0].run.runId);
|
|
166
|
+
assert.strictEqual(all[1].run.runId, results[1].run.runId);
|
|
167
|
+
assert.strictEqual(all[2].run.runId, results[2].run.runId);
|
|
168
|
+
});
|
|
169
|
+
});
|
|
170
|
+
describe("validate", () => {
|
|
171
|
+
it("should return empty array for valid result", () => {
|
|
172
|
+
const result = createMockResult();
|
|
173
|
+
const errors = collector.validate(result);
|
|
174
|
+
assert.deepStrictEqual(errors, []);
|
|
175
|
+
});
|
|
176
|
+
it("should detect empty runId", () => {
|
|
177
|
+
const result = createMockResult({
|
|
178
|
+
run: { ...createMockResult().run, runId: "" },
|
|
179
|
+
});
|
|
180
|
+
const errors = collector.validate(result);
|
|
181
|
+
assert.ok(errors.some((e) => e.field === "run.runId"));
|
|
182
|
+
});
|
|
183
|
+
it("should detect empty sut", () => {
|
|
184
|
+
const result = createMockResult({
|
|
185
|
+
run: { ...createMockResult().run, sut: "" },
|
|
186
|
+
});
|
|
187
|
+
const errors = collector.validate(result);
|
|
188
|
+
assert.ok(errors.some((e) => e.field === "run.sut"));
|
|
189
|
+
});
|
|
190
|
+
it("should detect empty caseId", () => {
|
|
191
|
+
const result = createMockResult({
|
|
192
|
+
run: { ...createMockResult().run, caseId: "" },
|
|
193
|
+
});
|
|
194
|
+
const errors = collector.validate(result);
|
|
195
|
+
assert.ok(errors.some((e) => e.field === "run.caseId"));
|
|
196
|
+
});
|
|
197
|
+
it("should detect empty numeric metrics", () => {
|
|
198
|
+
const result = createMockResult({
|
|
199
|
+
metrics: { numeric: {} },
|
|
200
|
+
});
|
|
201
|
+
const errors = collector.validate(result);
|
|
202
|
+
assert.ok(errors.some((e) => e.field === "metrics.numeric"));
|
|
203
|
+
});
|
|
204
|
+
it("should detect empty platform", () => {
|
|
205
|
+
const result = createMockResult({
|
|
206
|
+
provenance: {
|
|
207
|
+
...createMockResult().provenance,
|
|
208
|
+
runtime: {
|
|
209
|
+
...createMockResult().provenance.runtime,
|
|
210
|
+
platform: "",
|
|
211
|
+
},
|
|
212
|
+
},
|
|
213
|
+
});
|
|
214
|
+
const errors = collector.validate(result);
|
|
215
|
+
assert.ok(errors.some((e) => e.field === "provenance.runtime.platform"));
|
|
216
|
+
});
|
|
217
|
+
it("should return multiple errors for invalid result", () => {
|
|
218
|
+
const result = createMockResult({
|
|
219
|
+
run: { runId: "", sut: "", sutRole: "primary", caseId: "" },
|
|
220
|
+
metrics: { numeric: {} },
|
|
221
|
+
provenance: {
|
|
222
|
+
runtime: {
|
|
223
|
+
platform: "",
|
|
224
|
+
arch: "arm64",
|
|
225
|
+
nodeVersion: "20.0.0",
|
|
226
|
+
},
|
|
227
|
+
timestamp: new Date().toISOString(),
|
|
228
|
+
},
|
|
229
|
+
});
|
|
230
|
+
const errors = collector.validate(result);
|
|
231
|
+
assert.ok(errors.length >= 4);
|
|
232
|
+
});
|
|
233
|
+
it("should not validate optional fields like caseClass", () => {
|
|
234
|
+
const result = createMockResult({
|
|
235
|
+
run: { ...createMockResult().run, caseClass: undefined },
|
|
236
|
+
});
|
|
237
|
+
const errors = collector.validate(result);
|
|
238
|
+
assert.deepStrictEqual(errors, []);
|
|
239
|
+
});
|
|
240
|
+
});
|
|
241
|
+
describe("query", () => {
|
|
242
|
+
beforeEach(() => {
|
|
243
|
+
const results = [
|
|
244
|
+
createMockResult({
|
|
245
|
+
run: {
|
|
246
|
+
runId: "run-001",
|
|
247
|
+
sut: "sut-1",
|
|
248
|
+
sutRole: "primary",
|
|
249
|
+
caseId: "case-001",
|
|
250
|
+
caseClass: "class-a",
|
|
251
|
+
},
|
|
252
|
+
correctness: {
|
|
253
|
+
expectedExists: true,
|
|
254
|
+
producedOutput: true,
|
|
255
|
+
valid: true,
|
|
256
|
+
matchesExpected: true,
|
|
257
|
+
},
|
|
258
|
+
metrics: { numeric: { time: 100, memory: 50 } },
|
|
259
|
+
}),
|
|
260
|
+
createMockResult({
|
|
261
|
+
run: {
|
|
262
|
+
runId: "run-002",
|
|
263
|
+
sut: "sut-1",
|
|
264
|
+
sutRole: "primary",
|
|
265
|
+
caseId: "case-002",
|
|
266
|
+
caseClass: "class-b",
|
|
267
|
+
},
|
|
268
|
+
correctness: {
|
|
269
|
+
expectedExists: true,
|
|
270
|
+
producedOutput: true,
|
|
271
|
+
valid: false,
|
|
272
|
+
matchesExpected: false,
|
|
273
|
+
},
|
|
274
|
+
metrics: { numeric: { time: 200 } },
|
|
275
|
+
}),
|
|
276
|
+
createMockResult({
|
|
277
|
+
run: {
|
|
278
|
+
runId: "run-003",
|
|
279
|
+
sut: "sut-2",
|
|
280
|
+
sutRole: "baseline",
|
|
281
|
+
caseId: "case-001",
|
|
282
|
+
caseClass: "class-a",
|
|
283
|
+
},
|
|
284
|
+
correctness: {
|
|
285
|
+
expectedExists: true,
|
|
286
|
+
producedOutput: true,
|
|
287
|
+
valid: true,
|
|
288
|
+
matchesExpected: true,
|
|
289
|
+
},
|
|
290
|
+
metrics: { numeric: { time: 150, memory: 75 } },
|
|
291
|
+
}),
|
|
292
|
+
];
|
|
293
|
+
collector.recordBatch(results);
|
|
294
|
+
});
|
|
295
|
+
it("should return all results with no filter", () => {
|
|
296
|
+
const results = collector.query();
|
|
297
|
+
assert.strictEqual(results.length, 3);
|
|
298
|
+
});
|
|
299
|
+
it("should filter by sut", () => {
|
|
300
|
+
const results = collector.query({ sut: "sut-1" });
|
|
301
|
+
assert.strictEqual(results.length, 2);
|
|
302
|
+
assert.ok(results.every((r) => r.run.sut === "sut-1"));
|
|
303
|
+
});
|
|
304
|
+
it("should filter by sutRole", () => {
|
|
305
|
+
const results = collector.query({ sutRole: "baseline" });
|
|
306
|
+
assert.strictEqual(results.length, 1);
|
|
307
|
+
assert.strictEqual(results[0].run.sutRole, "baseline");
|
|
308
|
+
});
|
|
309
|
+
it("should filter by caseId", () => {
|
|
310
|
+
const results = collector.query({ caseId: "case-001" });
|
|
311
|
+
assert.strictEqual(results.length, 2);
|
|
312
|
+
assert.ok(results.every((r) => r.run.caseId === "case-001"));
|
|
313
|
+
});
|
|
314
|
+
it("should filter by caseClass", () => {
|
|
315
|
+
const results = collector.query({ caseClass: "class-a" });
|
|
316
|
+
assert.strictEqual(results.length, 2);
|
|
317
|
+
assert.ok(results.every((r) => r.run.caseClass === "class-a"));
|
|
318
|
+
});
|
|
319
|
+
it("should filter by valid", () => {
|
|
320
|
+
const results = collector.query({ valid: true });
|
|
321
|
+
assert.strictEqual(results.length, 2);
|
|
322
|
+
assert.ok(results.every((r) => r.correctness.valid));
|
|
323
|
+
});
|
|
324
|
+
it("should filter by hasMetric", () => {
|
|
325
|
+
const results = collector.query({ hasMetric: "memory" });
|
|
326
|
+
assert.strictEqual(results.length, 2);
|
|
327
|
+
assert.ok(results.every((r) => "memory" in r.metrics.numeric));
|
|
328
|
+
});
|
|
329
|
+
it("should filter by custom predicate", () => {
|
|
330
|
+
const results = collector.query({
|
|
331
|
+
predicate: (r) => r.metrics.numeric.time > 120,
|
|
332
|
+
});
|
|
333
|
+
assert.strictEqual(results.length, 2);
|
|
334
|
+
});
|
|
335
|
+
it("should combine multiple filters", () => {
|
|
336
|
+
const results = collector.query({
|
|
337
|
+
sut: "sut-1",
|
|
338
|
+
caseClass: "class-a",
|
|
339
|
+
valid: true,
|
|
340
|
+
});
|
|
341
|
+
assert.strictEqual(results.length, 1);
|
|
342
|
+
assert.strictEqual(results[0].run.runId, "run-001");
|
|
343
|
+
});
|
|
344
|
+
it("should return empty array for non-matching filter", () => {
|
|
345
|
+
const results = collector.query({ sut: "non-existent" });
|
|
346
|
+
assert.deepStrictEqual(results, []);
|
|
347
|
+
});
|
|
348
|
+
it("should handle undefined caseClass in filter", () => {
|
|
349
|
+
const result = createMockResult({
|
|
350
|
+
run: {
|
|
351
|
+
runId: "run-no-class",
|
|
352
|
+
sut: "sut-3",
|
|
353
|
+
sutRole: "primary",
|
|
354
|
+
caseId: "case-003",
|
|
355
|
+
},
|
|
356
|
+
metrics: { numeric: { time: 100 } },
|
|
357
|
+
});
|
|
358
|
+
collector.record(result);
|
|
359
|
+
const withClass = collector.query({ caseClass: "class-a" });
|
|
360
|
+
const withoutClass = collector.query({ sut: "sut-3" });
|
|
361
|
+
assert.strictEqual(withClass.length, 2);
|
|
362
|
+
assert.strictEqual(withoutClass.length, 1);
|
|
363
|
+
});
|
|
364
|
+
});
|
|
365
|
+
describe("getBySut", () => {
|
|
366
|
+
beforeEach(() => {
|
|
367
|
+
const batch1 = createMockResults(3, "sut-1", "primary", "class-a");
|
|
368
|
+
const batch2 = createMockResults(2, "sut-2", "baseline", "class-b");
|
|
369
|
+
collector.recordBatch([...batch1, ...batch2]);
|
|
370
|
+
});
|
|
371
|
+
it("should return all results for specified SUT", () => {
|
|
372
|
+
const results = collector.getBySut("sut-1");
|
|
373
|
+
assert.strictEqual(results.length, 3);
|
|
374
|
+
assert.ok(results.every((r) => r.run.sut === "sut-1"));
|
|
375
|
+
});
|
|
376
|
+
it("should return empty array for non-existent SUT", () => {
|
|
377
|
+
const results = collector.getBySut("non-existent");
|
|
378
|
+
assert.deepStrictEqual(results, []);
|
|
379
|
+
});
|
|
380
|
+
it("should return results with different roles for same SUT", () => {
|
|
381
|
+
const result1 = createMockResult({
|
|
382
|
+
run: {
|
|
383
|
+
runId: "run-001",
|
|
384
|
+
sut: "multi-role-sut",
|
|
385
|
+
sutRole: "primary",
|
|
386
|
+
caseId: "case-001",
|
|
387
|
+
},
|
|
388
|
+
metrics: { numeric: { time: 100 } },
|
|
389
|
+
});
|
|
390
|
+
const result2 = createMockResult({
|
|
391
|
+
run: {
|
|
392
|
+
runId: "run-002",
|
|
393
|
+
sut: "multi-role-sut",
|
|
394
|
+
sutRole: "baseline",
|
|
395
|
+
caseId: "case-002",
|
|
396
|
+
},
|
|
397
|
+
metrics: { numeric: { time: 100 } },
|
|
398
|
+
});
|
|
399
|
+
collector.recordBatch([result1, result2]);
|
|
400
|
+
const results = collector.getBySut("multi-role-sut");
|
|
401
|
+
assert.strictEqual(results.length, 2);
|
|
402
|
+
});
|
|
403
|
+
});
|
|
404
|
+
describe("getByCaseClass", () => {
|
|
405
|
+
beforeEach(() => {
|
|
406
|
+
const results = [
|
|
407
|
+
createMockResult({
|
|
408
|
+
run: {
|
|
409
|
+
runId: "run-001",
|
|
410
|
+
sut: "sut-1",
|
|
411
|
+
sutRole: "primary",
|
|
412
|
+
caseId: "case-001",
|
|
413
|
+
caseClass: "graph",
|
|
414
|
+
},
|
|
415
|
+
metrics: { numeric: { time: 100 } },
|
|
416
|
+
}),
|
|
417
|
+
createMockResult({
|
|
418
|
+
run: {
|
|
419
|
+
runId: "run-002",
|
|
420
|
+
sut: "sut-1",
|
|
421
|
+
sutRole: "primary",
|
|
422
|
+
caseId: "case-002",
|
|
423
|
+
caseClass: "graph",
|
|
424
|
+
},
|
|
425
|
+
metrics: { numeric: { time: 100 } },
|
|
426
|
+
}),
|
|
427
|
+
createMockResult({
|
|
428
|
+
run: {
|
|
429
|
+
runId: "run-003",
|
|
430
|
+
sut: "sut-2",
|
|
431
|
+
sutRole: "baseline",
|
|
432
|
+
caseId: "case-003",
|
|
433
|
+
caseClass: "tree",
|
|
434
|
+
},
|
|
435
|
+
metrics: { numeric: { time: 100 } },
|
|
436
|
+
}),
|
|
437
|
+
createMockResult({
|
|
438
|
+
run: {
|
|
439
|
+
runId: "run-004",
|
|
440
|
+
sut: "sut-1",
|
|
441
|
+
sutRole: "primary",
|
|
442
|
+
caseId: "case-004",
|
|
443
|
+
},
|
|
444
|
+
metrics: { numeric: { time: 100 } },
|
|
445
|
+
}),
|
|
446
|
+
];
|
|
447
|
+
collector.recordBatch(results);
|
|
448
|
+
});
|
|
449
|
+
it("should return all results for specified case class", () => {
|
|
450
|
+
const results = collector.getByCaseClass("graph");
|
|
451
|
+
assert.strictEqual(results.length, 2);
|
|
452
|
+
assert.ok(results.every((r) => r.run.caseClass === "graph"));
|
|
453
|
+
});
|
|
454
|
+
it("should return empty array for non-existent case class", () => {
|
|
455
|
+
const results = collector.getByCaseClass("non-existent");
|
|
456
|
+
assert.deepStrictEqual(results, []);
|
|
457
|
+
});
|
|
458
|
+
it("should not include results with undefined caseClass", () => {
|
|
459
|
+
const results = collector.getByCaseClass("graph");
|
|
460
|
+
assert.strictEqual(results.length, 2);
|
|
461
|
+
assert.ok(results.every((r) => r.run.caseClass !== undefined));
|
|
462
|
+
});
|
|
463
|
+
});
|
|
464
|
+
describe("getUniqueSuts", () => {
|
|
465
|
+
it("should return empty array when no results", () => {
|
|
466
|
+
const suts = collector.getUniqueSuts();
|
|
467
|
+
assert.deepStrictEqual(suts, []);
|
|
468
|
+
});
|
|
469
|
+
it("should return unique SUT IDs", () => {
|
|
470
|
+
const batch1 = createMockResults(3, "sut-1", "primary");
|
|
471
|
+
const batch2 = createMockResults(2, "sut-2", "baseline");
|
|
472
|
+
collector.recordBatch([...batch1, ...batch2]);
|
|
473
|
+
const suts = collector.getUniqueSuts();
|
|
474
|
+
assert.strictEqual(suts.length, 2);
|
|
475
|
+
assert.ok(suts.includes("sut-1"));
|
|
476
|
+
assert.ok(suts.includes("sut-2"));
|
|
477
|
+
});
|
|
478
|
+
it("should preserve insertion order", () => {
|
|
479
|
+
collector.recordBatch(createMockResults(1, "sut-c", "primary"));
|
|
480
|
+
collector.recordBatch(createMockResults(1, "sut-a", "primary"));
|
|
481
|
+
collector.recordBatch(createMockResults(1, "sut-b", "primary"));
|
|
482
|
+
const suts = collector.getUniqueSuts();
|
|
483
|
+
assert.deepStrictEqual(suts, ["sut-c", "sut-a", "sut-b"]);
|
|
484
|
+
});
|
|
485
|
+
it("should handle same SUT with different roles", () => {
|
|
486
|
+
const result1 = createMockResult({
|
|
487
|
+
run: {
|
|
488
|
+
runId: "run-001",
|
|
489
|
+
sut: "same-sut",
|
|
490
|
+
sutRole: "primary",
|
|
491
|
+
caseId: "case-001",
|
|
492
|
+
},
|
|
493
|
+
metrics: { numeric: { time: 100 } },
|
|
494
|
+
});
|
|
495
|
+
const result2 = createMockResult({
|
|
496
|
+
run: {
|
|
497
|
+
runId: "run-002",
|
|
498
|
+
sut: "same-sut",
|
|
499
|
+
sutRole: "baseline",
|
|
500
|
+
caseId: "case-002",
|
|
501
|
+
},
|
|
502
|
+
metrics: { numeric: { time: 100 } },
|
|
503
|
+
});
|
|
504
|
+
collector.recordBatch([result1, result2]);
|
|
505
|
+
const suts = collector.getUniqueSuts();
|
|
506
|
+
assert.deepStrictEqual(suts, ["same-sut"]);
|
|
507
|
+
});
|
|
508
|
+
});
|
|
509
|
+
describe("getUniqueCaseClasses", () => {
|
|
510
|
+
it("should return empty array when no results", () => {
|
|
511
|
+
const classes = collector.getUniqueCaseClasses();
|
|
512
|
+
assert.deepStrictEqual(classes, []);
|
|
513
|
+
});
|
|
514
|
+
it("should return unique case classes", () => {
|
|
515
|
+
const results = [
|
|
516
|
+
createMockResult({
|
|
517
|
+
run: {
|
|
518
|
+
runId: "run-001",
|
|
519
|
+
sut: "sut-1",
|
|
520
|
+
sutRole: "primary",
|
|
521
|
+
caseId: "case-001",
|
|
522
|
+
caseClass: "graph",
|
|
523
|
+
},
|
|
524
|
+
metrics: { numeric: { time: 100 } },
|
|
525
|
+
}),
|
|
526
|
+
createMockResult({
|
|
527
|
+
run: {
|
|
528
|
+
runId: "run-002",
|
|
529
|
+
sut: "sut-1",
|
|
530
|
+
sutRole: "primary",
|
|
531
|
+
caseId: "case-002",
|
|
532
|
+
caseClass: "tree",
|
|
533
|
+
},
|
|
534
|
+
metrics: { numeric: { time: 100 } },
|
|
535
|
+
}),
|
|
536
|
+
createMockResult({
|
|
537
|
+
run: {
|
|
538
|
+
runId: "run-003",
|
|
539
|
+
sut: "sut-2",
|
|
540
|
+
sutRole: "baseline",
|
|
541
|
+
caseId: "case-003",
|
|
542
|
+
caseClass: "graph",
|
|
543
|
+
},
|
|
544
|
+
metrics: { numeric: { time: 100 } },
|
|
545
|
+
}),
|
|
546
|
+
];
|
|
547
|
+
collector.recordBatch(results);
|
|
548
|
+
const classes = collector.getUniqueCaseClasses();
|
|
549
|
+
assert.strictEqual(classes.length, 2);
|
|
550
|
+
assert.ok(classes.includes("graph"));
|
|
551
|
+
assert.ok(classes.includes("tree"));
|
|
552
|
+
});
|
|
553
|
+
it("should exclude undefined caseClass", () => {
|
|
554
|
+
const results = [
|
|
555
|
+
createMockResult({
|
|
556
|
+
run: {
|
|
557
|
+
runId: "run-001",
|
|
558
|
+
sut: "sut-1",
|
|
559
|
+
sutRole: "primary",
|
|
560
|
+
caseId: "case-001",
|
|
561
|
+
caseClass: "graph",
|
|
562
|
+
},
|
|
563
|
+
metrics: { numeric: { time: 100 } },
|
|
564
|
+
}),
|
|
565
|
+
createMockResult({
|
|
566
|
+
run: {
|
|
567
|
+
runId: "run-002",
|
|
568
|
+
sut: "sut-1",
|
|
569
|
+
sutRole: "primary",
|
|
570
|
+
caseId: "case-002",
|
|
571
|
+
caseClass: undefined,
|
|
572
|
+
},
|
|
573
|
+
metrics: { numeric: { time: 100 } },
|
|
574
|
+
}),
|
|
575
|
+
];
|
|
576
|
+
collector.recordBatch(results);
|
|
577
|
+
const classes = collector.getUniqueCaseClasses();
|
|
578
|
+
assert.deepStrictEqual(classes, ["graph"]);
|
|
579
|
+
});
|
|
580
|
+
it("should preserve insertion order", () => {
|
|
581
|
+
collector.record(createMockResult({
|
|
582
|
+
run: {
|
|
583
|
+
runId: "run-001",
|
|
584
|
+
sut: "sut-1",
|
|
585
|
+
sutRole: "primary",
|
|
586
|
+
caseId: "case-001",
|
|
587
|
+
caseClass: "class-c",
|
|
588
|
+
},
|
|
589
|
+
metrics: { numeric: { time: 100 } },
|
|
590
|
+
}));
|
|
591
|
+
collector.record(createMockResult({
|
|
592
|
+
run: {
|
|
593
|
+
runId: "run-002",
|
|
594
|
+
sut: "sut-1",
|
|
595
|
+
sutRole: "primary",
|
|
596
|
+
caseId: "case-002",
|
|
597
|
+
caseClass: "class-a",
|
|
598
|
+
},
|
|
599
|
+
metrics: { numeric: { time: 100 } },
|
|
600
|
+
}));
|
|
601
|
+
collector.record(createMockResult({
|
|
602
|
+
run: {
|
|
603
|
+
runId: "run-003",
|
|
604
|
+
sut: "sut-2",
|
|
605
|
+
sutRole: "baseline",
|
|
606
|
+
caseId: "case-003",
|
|
607
|
+
caseClass: "class-b",
|
|
608
|
+
},
|
|
609
|
+
metrics: { numeric: { time: 100 } },
|
|
610
|
+
}));
|
|
611
|
+
const classes = collector.getUniqueCaseClasses();
|
|
612
|
+
assert.deepStrictEqual(classes, ["class-c", "class-a", "class-b"]);
|
|
613
|
+
});
|
|
614
|
+
});
|
|
615
|
+
describe("getUniqueMetrics", () => {
|
|
616
|
+
it("should return empty array when no results", () => {
|
|
617
|
+
const metrics = collector.getUniqueMetrics();
|
|
618
|
+
assert.deepStrictEqual(metrics, []);
|
|
619
|
+
});
|
|
620
|
+
it("should return unique metric names across all results", () => {
|
|
621
|
+
const results = [
|
|
622
|
+
createMockResult({
|
|
623
|
+
run: {
|
|
624
|
+
runId: "run-001",
|
|
625
|
+
sut: "sut-1",
|
|
626
|
+
sutRole: "primary",
|
|
627
|
+
caseId: "case-001",
|
|
628
|
+
},
|
|
629
|
+
metrics: { numeric: { time: 100, memory: 50 } },
|
|
630
|
+
}),
|
|
631
|
+
createMockResult({
|
|
632
|
+
run: {
|
|
633
|
+
runId: "run-002",
|
|
634
|
+
sut: "sut-1",
|
|
635
|
+
sutRole: "primary",
|
|
636
|
+
caseId: "case-002",
|
|
637
|
+
},
|
|
638
|
+
metrics: { numeric: { time: 200, nodes: 10 } },
|
|
639
|
+
}),
|
|
640
|
+
];
|
|
641
|
+
collector.recordBatch(results);
|
|
642
|
+
const metrics = collector.getUniqueMetrics();
|
|
643
|
+
assert.strictEqual(metrics.length, 3);
|
|
644
|
+
assert.ok(metrics.includes("time"));
|
|
645
|
+
assert.ok(metrics.includes("memory"));
|
|
646
|
+
assert.ok(metrics.includes("nodes"));
|
|
647
|
+
});
|
|
648
|
+
it("should preserve insertion order of first occurrence", () => {
|
|
649
|
+
collector.record(createMockResult({
|
|
650
|
+
run: {
|
|
651
|
+
runId: "run-001",
|
|
652
|
+
sut: "sut-1",
|
|
653
|
+
sutRole: "primary",
|
|
654
|
+
caseId: "case-001",
|
|
655
|
+
},
|
|
656
|
+
metrics: { numeric: { z: 1, a: 2 } },
|
|
657
|
+
}));
|
|
658
|
+
collector.record(createMockResult({
|
|
659
|
+
run: {
|
|
660
|
+
runId: "run-002",
|
|
661
|
+
sut: "sut-1",
|
|
662
|
+
sutRole: "primary",
|
|
663
|
+
caseId: "case-002",
|
|
664
|
+
},
|
|
665
|
+
metrics: { numeric: { b: 3 } },
|
|
666
|
+
}));
|
|
667
|
+
const metrics = collector.getUniqueMetrics();
|
|
668
|
+
assert.deepStrictEqual(metrics, ["z", "a", "b"]);
|
|
669
|
+
});
|
|
670
|
+
});
|
|
671
|
+
describe("getAll", () => {
|
|
672
|
+
it("should return empty array when no results", () => {
|
|
673
|
+
const results = collector.getAll();
|
|
674
|
+
assert.deepStrictEqual(results, []);
|
|
675
|
+
});
|
|
676
|
+
it("should return copy of all results", () => {
|
|
677
|
+
const original = createMockResults(3, "sut-1");
|
|
678
|
+
collector.recordBatch(original);
|
|
679
|
+
const retrieved = collector.getAll();
|
|
680
|
+
assert.strictEqual(retrieved.length, 3);
|
|
681
|
+
assert.deepStrictEqual(retrieved, original);
|
|
682
|
+
});
|
|
683
|
+
it("should not return reference to internal array", () => {
|
|
684
|
+
collector.recordBatch(createMockResults(2, "sut-1"));
|
|
685
|
+
const results1 = collector.getAll();
|
|
686
|
+
const results2 = collector.getAll();
|
|
687
|
+
assert.notStrictEqual(results1, results2);
|
|
688
|
+
});
|
|
689
|
+
it("should preserve insertion order", () => {
|
|
690
|
+
const results = createMockResults(3, "sut-1");
|
|
691
|
+
collector.recordBatch(results);
|
|
692
|
+
const retrieved = collector.getAll();
|
|
693
|
+
assert.strictEqual(retrieved[0].run.runId, results[0].run.runId);
|
|
694
|
+
assert.strictEqual(retrieved[1].run.runId, results[1].run.runId);
|
|
695
|
+
assert.strictEqual(retrieved[2].run.runId, results[2].run.runId);
|
|
696
|
+
});
|
|
697
|
+
});
|
|
698
|
+
describe("count getter", () => {
|
|
699
|
+
it("should return 0 when empty", () => {
|
|
700
|
+
assert.strictEqual(collector.count, 0);
|
|
701
|
+
});
|
|
702
|
+
it("should return number of results", () => {
|
|
703
|
+
collector.recordBatch(createMockResults(5, "sut-1"));
|
|
704
|
+
assert.strictEqual(collector.count, 5);
|
|
705
|
+
});
|
|
706
|
+
it("should update after adding results", () => {
|
|
707
|
+
assert.strictEqual(collector.count, 0);
|
|
708
|
+
collector.record(createMockResult());
|
|
709
|
+
assert.strictEqual(collector.count, 1);
|
|
710
|
+
collector.recordBatch(createMockResults(3, "sut-1"));
|
|
711
|
+
assert.strictEqual(collector.count, 4);
|
|
712
|
+
});
|
|
713
|
+
it("should update after clearing", () => {
|
|
714
|
+
collector.recordBatch(createMockResults(5, "sut-1"));
|
|
715
|
+
assert.strictEqual(collector.count, 5);
|
|
716
|
+
collector.clear();
|
|
717
|
+
assert.strictEqual(collector.count, 0);
|
|
718
|
+
});
|
|
719
|
+
});
|
|
720
|
+
describe("isEmpty getter", () => {
|
|
721
|
+
it("should return true when empty", () => {
|
|
722
|
+
assert.strictEqual(collector.isEmpty, true);
|
|
723
|
+
});
|
|
724
|
+
it("should return false when has results", () => {
|
|
725
|
+
collector.record(createMockResult());
|
|
726
|
+
assert.strictEqual(collector.isEmpty, false);
|
|
727
|
+
});
|
|
728
|
+
it("should return true after clearing", () => {
|
|
729
|
+
collector.record(createMockResult());
|
|
730
|
+
assert.strictEqual(collector.isEmpty, false);
|
|
731
|
+
collector.clear();
|
|
732
|
+
assert.strictEqual(collector.isEmpty, true);
|
|
733
|
+
});
|
|
734
|
+
});
|
|
735
|
+
describe("clear", () => {
|
|
736
|
+
it("should remove all results", () => {
|
|
737
|
+
collector.recordBatch(createMockResults(5, "sut-1"));
|
|
738
|
+
assert.strictEqual(collector.count, 5);
|
|
739
|
+
collector.clear();
|
|
740
|
+
assert.strictEqual(collector.count, 0);
|
|
741
|
+
assert.deepStrictEqual(collector.getAll(), []);
|
|
742
|
+
});
|
|
743
|
+
it("should be idempotent", () => {
|
|
744
|
+
collector.recordBatch(createMockResults(3, "sut-1"));
|
|
745
|
+
collector.clear();
|
|
746
|
+
collector.clear();
|
|
747
|
+
assert.strictEqual(collector.count, 0);
|
|
748
|
+
});
|
|
749
|
+
it("should clear empty collector", () => {
|
|
750
|
+
collector.clear();
|
|
751
|
+
assert.strictEqual(collector.count, 0);
|
|
752
|
+
});
|
|
753
|
+
});
|
|
754
|
+
describe("serialize", () => {
|
|
755
|
+
beforeEach(() => {
|
|
756
|
+
const results = createMockResults(3, "sut-1", "primary", "test-class");
|
|
757
|
+
collector.recordBatch(results);
|
|
758
|
+
});
|
|
759
|
+
it("should create ResultBatch with version", () => {
|
|
760
|
+
const batch = collector.serialize();
|
|
761
|
+
assert.strictEqual(batch.version, "1.0.0");
|
|
762
|
+
});
|
|
763
|
+
it("should include timestamp", () => {
|
|
764
|
+
const before = new Date().toISOString();
|
|
765
|
+
const batch = collector.serialize();
|
|
766
|
+
const after = new Date().toISOString();
|
|
767
|
+
assert.ok(batch.timestamp >= before);
|
|
768
|
+
assert.ok(batch.timestamp <= after);
|
|
769
|
+
});
|
|
770
|
+
it("should include all results", () => {
|
|
771
|
+
const batch = collector.serialize();
|
|
772
|
+
assert.strictEqual(batch.results.length, 3);
|
|
773
|
+
});
|
|
774
|
+
it("should have undefined metadata by default", () => {
|
|
775
|
+
const batch = collector.serialize();
|
|
776
|
+
assert.strictEqual(batch.metadata, undefined);
|
|
777
|
+
});
|
|
778
|
+
it("should include custom metadata", () => {
|
|
779
|
+
const metadata = {
|
|
780
|
+
experimentId: "exp-001",
|
|
781
|
+
totalRuns: 10,
|
|
782
|
+
successRate: 0.95,
|
|
783
|
+
};
|
|
784
|
+
const batch = collector.serialize(metadata);
|
|
785
|
+
assert.deepStrictEqual(batch.metadata, metadata);
|
|
786
|
+
});
|
|
787
|
+
it("should create independent copy of results", () => {
|
|
788
|
+
const batch = collector.serialize();
|
|
789
|
+
const originalResults = collector.getAll();
|
|
790
|
+
assert.notStrictEqual(batch.results, originalResults);
|
|
791
|
+
assert.deepStrictEqual(batch.results, originalResults);
|
|
792
|
+
});
|
|
793
|
+
});
|
|
794
|
+
describe("load", () => {
|
|
795
|
+
it("should replace existing results when append is false", () => {
|
|
796
|
+
collector.recordBatch(createMockResults(3, "sut-1", "primary"));
|
|
797
|
+
assert.strictEqual(collector.count, 3);
|
|
798
|
+
const batch = {
|
|
799
|
+
version: "1.0.0",
|
|
800
|
+
timestamp: new Date().toISOString(),
|
|
801
|
+
results: createMockResults(2, "sut-2", "baseline"),
|
|
802
|
+
};
|
|
803
|
+
collector.load(batch);
|
|
804
|
+
assert.strictEqual(collector.count, 2);
|
|
805
|
+
const suts = collector.getUniqueSuts();
|
|
806
|
+
assert.deepStrictEqual(suts, ["sut-2"]);
|
|
807
|
+
});
|
|
808
|
+
it("should append results when append is true", () => {
|
|
809
|
+
collector.recordBatch(createMockResults(2, "sut-1", "primary"));
|
|
810
|
+
assert.strictEqual(collector.count, 2);
|
|
811
|
+
const batch = {
|
|
812
|
+
version: "1.0.0",
|
|
813
|
+
timestamp: new Date().toISOString(),
|
|
814
|
+
results: createMockResults(3, "sut-2", "baseline"),
|
|
815
|
+
};
|
|
816
|
+
collector.load(batch, true);
|
|
817
|
+
assert.strictEqual(collector.count, 5);
|
|
818
|
+
const suts = collector.getUniqueSuts();
|
|
819
|
+
assert.ok(suts.includes("sut-1"));
|
|
820
|
+
assert.ok(suts.includes("sut-2"));
|
|
821
|
+
});
|
|
822
|
+
it("should validate results during load", () => {
|
|
823
|
+
const batch = {
|
|
824
|
+
version: "1.0.0",
|
|
825
|
+
timestamp: new Date().toISOString(),
|
|
826
|
+
results: [
|
|
827
|
+
createMockResult(),
|
|
828
|
+
createMockResult({
|
|
829
|
+
run: { runId: "", sut: "", sutRole: "primary", caseId: "" },
|
|
830
|
+
metrics: { numeric: {} },
|
|
831
|
+
provenance: {
|
|
832
|
+
runtime: {
|
|
833
|
+
platform: "",
|
|
834
|
+
arch: "arm64",
|
|
835
|
+
nodeVersion: "20.0.0",
|
|
836
|
+
},
|
|
837
|
+
timestamp: new Date().toISOString(),
|
|
838
|
+
},
|
|
839
|
+
}),
|
|
840
|
+
],
|
|
841
|
+
};
|
|
842
|
+
assert.throws(() => {
|
|
843
|
+
collector.load(batch);
|
|
844
|
+
});
|
|
845
|
+
});
|
|
846
|
+
it("should handle empty batch", () => {
|
|
847
|
+
collector.recordBatch(createMockResults(3, "sut-1"));
|
|
848
|
+
const batch = {
|
|
849
|
+
version: "1.0.0",
|
|
850
|
+
timestamp: new Date().toISOString(),
|
|
851
|
+
results: [],
|
|
852
|
+
};
|
|
853
|
+
collector.load(batch);
|
|
854
|
+
assert.strictEqual(collector.count, 0);
|
|
855
|
+
});
|
|
856
|
+
it("should preserve results when append is true and load fails", () => {
|
|
857
|
+
const originalResults = createMockResults(2, "sut-1");
|
|
858
|
+
collector.recordBatch(originalResults);
|
|
859
|
+
const batch = {
|
|
860
|
+
version: "1.0.0",
|
|
861
|
+
timestamp: new Date().toISOString(),
|
|
862
|
+
results: [
|
|
863
|
+
createMockResult({
|
|
864
|
+
run: { runId: "", sut: "", sutRole: "primary", caseId: "" },
|
|
865
|
+
metrics: { numeric: {} },
|
|
866
|
+
provenance: {
|
|
867
|
+
runtime: {
|
|
868
|
+
platform: "",
|
|
869
|
+
arch: "arm64",
|
|
870
|
+
nodeVersion: "20.0.0",
|
|
871
|
+
},
|
|
872
|
+
timestamp: new Date().toISOString(),
|
|
873
|
+
},
|
|
874
|
+
}),
|
|
875
|
+
],
|
|
876
|
+
};
|
|
877
|
+
assert.throws(() => {
|
|
878
|
+
collector.load(batch, true);
|
|
879
|
+
});
|
|
880
|
+
assert.strictEqual(collector.count, 2);
|
|
881
|
+
});
|
|
882
|
+
});
|
|
883
|
+
describe("extractMetric", () => {
|
|
884
|
+
beforeEach(() => {
|
|
885
|
+
const results = [
|
|
886
|
+
createMockResult({
|
|
887
|
+
run: {
|
|
888
|
+
runId: "run-001",
|
|
889
|
+
sut: "sut-1",
|
|
890
|
+
sutRole: "primary",
|
|
891
|
+
caseId: "case-001",
|
|
892
|
+
},
|
|
893
|
+
metrics: { numeric: { time: 100, memory: 50 } },
|
|
894
|
+
}),
|
|
895
|
+
createMockResult({
|
|
896
|
+
run: {
|
|
897
|
+
runId: "run-002",
|
|
898
|
+
sut: "sut-1",
|
|
899
|
+
sutRole: "primary",
|
|
900
|
+
caseId: "case-002",
|
|
901
|
+
},
|
|
902
|
+
metrics: { numeric: { time: 200 } },
|
|
903
|
+
}),
|
|
904
|
+
createMockResult({
|
|
905
|
+
run: {
|
|
906
|
+
runId: "run-003",
|
|
907
|
+
sut: "sut-2",
|
|
908
|
+
sutRole: "baseline",
|
|
909
|
+
caseId: "case-003",
|
|
910
|
+
},
|
|
911
|
+
metrics: { numeric: { memory: 75 } },
|
|
912
|
+
}),
|
|
913
|
+
];
|
|
914
|
+
collector.recordBatch(results);
|
|
915
|
+
});
|
|
916
|
+
it("should extract metric values across all results", () => {
|
|
917
|
+
const extracted = collector.extractMetric("time");
|
|
918
|
+
assert.strictEqual(extracted.length, 2);
|
|
919
|
+
assert.deepStrictEqual(extracted, [
|
|
920
|
+
{ runId: "run-001", value: 100 },
|
|
921
|
+
{ runId: "run-002", value: 200 },
|
|
922
|
+
]);
|
|
923
|
+
});
|
|
924
|
+
it("should return empty array for non-existent metric", () => {
|
|
925
|
+
const extracted = collector.extractMetric("non-existent");
|
|
926
|
+
assert.deepStrictEqual(extracted, []);
|
|
927
|
+
});
|
|
928
|
+
it("should preserve order of insertion", () => {
|
|
929
|
+
const extracted = collector.extractMetric("time");
|
|
930
|
+
assert.strictEqual(extracted[0].runId, "run-001");
|
|
931
|
+
assert.strictEqual(extracted[1].runId, "run-002");
|
|
932
|
+
});
|
|
933
|
+
it("should extract metric that exists in all results", () => {
|
|
934
|
+
const results = [
|
|
935
|
+
createMockResult({
|
|
936
|
+
run: {
|
|
937
|
+
runId: "run-001",
|
|
938
|
+
sut: "sut-1",
|
|
939
|
+
sutRole: "primary",
|
|
940
|
+
caseId: "case-001",
|
|
941
|
+
},
|
|
942
|
+
metrics: { numeric: { score: 0.8 } },
|
|
943
|
+
}),
|
|
944
|
+
createMockResult({
|
|
945
|
+
run: {
|
|
946
|
+
runId: "run-002",
|
|
947
|
+
sut: "sut-1",
|
|
948
|
+
sutRole: "primary",
|
|
949
|
+
caseId: "case-002",
|
|
950
|
+
},
|
|
951
|
+
metrics: { numeric: { score: 0.9 } },
|
|
952
|
+
}),
|
|
953
|
+
];
|
|
954
|
+
collector.clear();
|
|
955
|
+
collector.recordBatch(results);
|
|
956
|
+
const extracted = collector.extractMetric("score");
|
|
957
|
+
assert.strictEqual(extracted.length, 2);
|
|
958
|
+
});
|
|
959
|
+
});
|
|
960
|
+
describe("getMetricValues", () => {
|
|
961
|
+
beforeEach(() => {
|
|
962
|
+
const results = [
|
|
963
|
+
createMockResult({
|
|
964
|
+
run: {
|
|
965
|
+
runId: "run-001",
|
|
966
|
+
sut: "sut-1",
|
|
967
|
+
sutRole: "primary",
|
|
968
|
+
caseId: "case-001",
|
|
969
|
+
},
|
|
970
|
+
metrics: { numeric: { time: 100, memory: 50 } },
|
|
971
|
+
}),
|
|
972
|
+
createMockResult({
|
|
973
|
+
run: {
|
|
974
|
+
runId: "run-002",
|
|
975
|
+
sut: "sut-1",
|
|
976
|
+
sutRole: "primary",
|
|
977
|
+
caseId: "case-002",
|
|
978
|
+
},
|
|
979
|
+
metrics: { numeric: { time: 200 } },
|
|
980
|
+
}),
|
|
981
|
+
createMockResult({
|
|
982
|
+
run: {
|
|
983
|
+
runId: "run-003",
|
|
984
|
+
sut: "sut-2",
|
|
985
|
+
sutRole: "baseline",
|
|
986
|
+
caseId: "case-003",
|
|
987
|
+
},
|
|
988
|
+
metrics: { numeric: { time: 150 } },
|
|
989
|
+
}),
|
|
990
|
+
];
|
|
991
|
+
collector.recordBatch(results);
|
|
992
|
+
});
|
|
993
|
+
it("should return metric values for specified SUT", () => {
|
|
994
|
+
const values = collector.getMetricValues("sut-1", "time");
|
|
995
|
+
assert.strictEqual(values.length, 2);
|
|
996
|
+
assert.deepStrictEqual(values, [100, 200]);
|
|
997
|
+
});
|
|
998
|
+
it("should return empty array for non-existent SUT", () => {
|
|
999
|
+
const values = collector.getMetricValues("non-existent", "time");
|
|
1000
|
+
assert.deepStrictEqual(values, []);
|
|
1001
|
+
});
|
|
1002
|
+
it("should return empty array for non-existent metric", () => {
|
|
1003
|
+
const values = collector.getMetricValues("sut-1", "non-existent");
|
|
1004
|
+
assert.deepStrictEqual(values, []);
|
|
1005
|
+
});
|
|
1006
|
+
it("should return empty array when SUT has metric but other SUTs do not", () => {
|
|
1007
|
+
const values = collector.getMetricValues("sut-1", "memory");
|
|
1008
|
+
assert.deepStrictEqual(values, [50]);
|
|
1009
|
+
});
|
|
1010
|
+
it("should preserve insertion order", () => {
|
|
1011
|
+
const values = collector.getMetricValues("sut-1", "time");
|
|
1012
|
+
assert.strictEqual(values[0], 100);
|
|
1013
|
+
assert.strictEqual(values[1], 200);
|
|
1014
|
+
});
|
|
1015
|
+
it("should handle SUT with single result", () => {
|
|
1016
|
+
const values = collector.getMetricValues("sut-2", "time");
|
|
1017
|
+
assert.deepStrictEqual(values, [150]);
|
|
1018
|
+
});
|
|
1019
|
+
});
|
|
1020
|
+
});
|
|
1021
|
+
//# sourceMappingURL=result-collector.unit.test.js.map
|