ppef 1.0.0 → 1.0.1

package/dist/executor/__tests__/checkpoint-merge-bug.diagnostic.test.js

@@ -0,0 +1,224 @@
+/**
+ * Diagnostic Tests for Checkpoint Integration Bug
+ *
+ * Tests to diagnose why parallel workers aren't properly using checkpoints.
+ * Symptoms:
+ * - Workers report "No checkpoint"
+ * - Workers report "Total runs: 0, From checkpoint: 0, New this run: 0"
+ * - Main checkpoint has 123/132 runs but workers start fresh
+ */
+import { createHash, randomBytes } from "node:crypto";
+import { rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { strict as assert } from "node:assert";
+import { afterEach, beforeEach, describe, it } from "node:test";
+import { CheckpointManager } from "../checkpoint-manager.js";
+import { FileStorage } from "../checkpoint-storage.js";
+describe("Checkpoint Integration Bug Diagnostics", () => {
+    let testDir;
+    let checkpoint;
+    let checkpointPath;
+    beforeEach(() => {
+        testDir = join(tmpdir(), `checkpoint-test-${randomBytes(8).toString("hex")}`);
+        checkpointPath = join(testDir, "checkpoint.json");
+        checkpoint = new CheckpointManager({ storage: new FileStorage(checkpointPath) });
+    });
+    afterEach(() => {
+        rmSync(testDir, { recursive: true, force: true });
+    });
+    it("diagnostic-1: should save and load single run", async () => {
+        const sut = createMockSut();
+        const testCase = createMockCase("case-001");
+        const { Executor } = await import("../executor.js");
+        const executor = new Executor({
+            repetitions: 1,
+            seedBase: 42,
+            timeoutMs: 5000,
+            collectProvenance: false,
+            onResult: async (result) => {
+                await checkpoint.saveIncremental(result);
+            },
+        });
+        // Execute
+        const summary = await executor.execute([sut], [testCase], () => ({}));
+        // Verify
+        assert.strictEqual(summary.successfulRuns, 1);
+        const results = checkpoint.getResults();
+        assert.strictEqual(results.length, 1);
+        // Load fresh checkpoint
+        const fresh = new CheckpointManager({ storage: new FileStorage(checkpointPath) });
+        await fresh.load();
+        assert.strictEqual(fresh.getResults().length, 1);
+    });
+    it("diagnostic-2: should detect config hash mismatch", async () => {
+        // Save with one config
+        const result = {
+            run: {
+                runId: "test-run-001",
+                sut: "test-sut",
+                sutRole: "primary",
+                sutVersion: "1.0.0",
+                caseId: "case-001",
+                caseClass: "test-class",
+                seed: 42,
+                repetition: 0,
+            },
+            correctness: {
+                expectedExists: false,
+                producedOutput: true,
+                valid: true,
+                matchesExpected: null,
+            },
+            outputs: { summary: {} },
+            metrics: { numeric: { test: 1 } },
+            provenance: {
+                runtime: { platform: "linux", arch: "x64", nodeVersion: "v22.0.0" },
+            },
+        };
+        await checkpoint.saveIncremental(result);
+        // Try to load with different config
+        const fresh = new CheckpointManager({ storage: new FileStorage(checkpointPath) });
+        await fresh.load();
+        // Check if stale with different config
+        const isStale = fresh.isStale([createMockSut()], [createMockCase("case-001")], {
+            repetitions: 2, // Different from original (1)
+            seedBase: 42,
+            timeoutMs: 5000,
+            collectProvenance: false,
+        }, 1);
+        assert.strictEqual(isStale, true);
+    });
+    it("diagnostic-3: should find worker shards", async () => {
+        // Create mock worker shards
+        const shard1 = join(testDir, "checkpoint-worker-00.json");
+        const shard2 = join(testDir, "checkpoint-worker-01.json");
+        const storage1 = new FileStorage(shard1);
+        const storage2 = new FileStorage(shard2);
+        await storage1.save({
+            configHash: "test-hash",
+            createdAt: new Date().toISOString(),
+            updatedAt: new Date().toISOString(),
+            completedRunIds: ["run-001", "run-002"],
+            results: {},
+            totalPlanned: 0,
+        });
+        await storage2.save({
+            configHash: "test-hash",
+            createdAt: new Date().toISOString(),
+            updatedAt: new Date().toISOString(),
+            completedRunIds: ["run-003", "run-004"],
+            results: {},
+            totalPlanned: 0,
+        });
+        // Find shards
+        const shards = await FileStorage.findShards(testDir);
+        assert.strictEqual(shards.length, 2);
+        assert.ok(shards.includes(shard1));
+        assert.ok(shards.includes(shard2));
+    });
+    it("diagnostic-4: should merge shards without duplicates", async () => {
+        // Create main checkpoint
+        const result1 = {
+            run: {
+                runId: "run-001",
+                sut: "sut-1",
+                sutRole: "primary",
+                sutVersion: "1.0.0",
+                caseId: "case-001",
+                caseClass: "test-class",
+                seed: 42,
+                repetition: 0,
+            },
+            correctness: {
+                expectedExists: false,
+                producedOutput: true,
+                valid: true,
+                matchesExpected: null,
+            },
+            outputs: { summary: {} },
+            metrics: { numeric: { test: 1 } },
+            provenance: {
+                runtime: { platform: "linux", arch: "x64", nodeVersion: "v22.0.0" },
+            },
+        };
+        await checkpoint.saveIncremental(result1);
+        // Create worker shard with overlapping run
+        const workerStorage = new FileStorage(join(testDir, "checkpoint-worker-00.json"));
+        await workerStorage.save({
+            configHash: "test-hash",
+            createdAt: new Date().toISOString(),
+            updatedAt: new Date().toISOString(),
+            completedRunIds: ["run-001", "run-002"], // run-001 overlaps with main
+            results: {},
+            totalPlanned: 0,
+        });
+        // Merge
+        const shards = await FileStorage.findShards(testDir);
+        const merged = await checkpoint.mergeShards(shards);
+        // Verify no duplicates
+        const uniqueIds = new Set(merged.completedRunIds);
+        assert.strictEqual(uniqueIds.size, 2); // run-001, run-002 (no duplicates)
+        assert.strictEqual(merged.completedRunIds.length, 2);
+    });
+    it("diagnostic-5: config hash should include all executor config properties", async () => {
+        // This test verifies which properties are included in the hash
+        const config1 = {
+            continueOnError: true,
+            repetitions: 1,
+            seedBase: 42,
+            timeoutMs: 300_000,
+            collectProvenance: true,
+        };
+        const config2 = {
+            ...config1,
+            concurrency: 12, // Additional property
+        };
+        const hash1 = createHash("sha256")
+            .update(JSON.stringify(config1, Object.keys(config1).sort()))
+            .digest("hex");
+        const hash2 = createHash("sha256")
+            .update(JSON.stringify(config2, Object.keys(config2).sort()))
+            .digest("hex");
+        console.log("Hash without concurrency:", hash1);
+        console.log("Hash with concurrency:", hash2);
+        console.log("Match:", hash1 === hash2);
+        // They should be different because concurrency is different
+        assert.notStrictEqual(hash1, hash2);
+    });
+});
+/**
+ * Create a mock SUT for testing.
+ */
+const createMockSut = () => ({
+    registration: {
+        id: "mock-sut-v1.0.0",
+        name: "Mock SUT",
+        version: "1.0.0",
+        role: "primary",
+        config: {},
+        tags: ["test"],
+    },
+    factory: () => ({
+        id: "mock-sut-v1.0.0",
+        config: {},
+        run: async () => ({ mockResult: true }),
+    }),
+});
+/**
+ * Create a mock case for testing.
+ * @param id
+ */
+const createMockCase = (id) => ({
+    case: {
+        caseId: id,
+        name: `Mock Case ${id}`,
+        caseClass: "test",
+        inputs: { summary: { test: id } },
+        tags: ["test"],
+        version: "1.0.0",
+    },
+    getInput: async () => ({ mockInput: true }),
+    getInputs: () => ({ mockInputs: true }),
+});
+//# sourceMappingURL=checkpoint-merge-bug.diagnostic.test.js.map

package/dist/executor/__tests__/checkpoint-merge-bug.diagnostic.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"checkpoint-merge-bug.diagnostic.test.js","sourceRoot":"","sources":["../../../src/executor/__tests__/checkpoint-merge-bug.diagnostic.test.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,MAAM,IAAI,MAAM,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,WAAW,CAAC;AAIhE,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAEvD,QAAQ,CAAC,wCAAwC,EAAE,GAAG,EAAE;IACvD,IAAI,OAAe,CAAC;IACpB,IAAI,UAA6B,CAAC;IAClC,IAAI,cAAsB,CAAC;IAE3B,UAAU,CAAC,GAAG,EAAE;QACf,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,mBAAmB,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC9E,cAAc,GAAG,IAAI,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;QAClD,UAAU,GAAG,IAAI,iBAAiB,CAAC,EAAE,OAAO,EAAE,IAAI,WAAW,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;IAClF,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,GAAG,EAAE;QACd,MAAM,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;QAC9D,MAAM,GAAG,GAAG,aAAa,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC;QAE5C,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;QACpD,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC;YAC7B,WAAW,EAAE,CAAC;YACd,QAAQ,EAAE,EAAE;YACZ,SAAS,EAAE,IAAI;YACf,iBAAiB,EAAE,KAAK;YACxB,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;gBAC1B,MAAM,UAAU,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;YAC1C,CAAC;SACD,CAAC,CAAC;QAEH,UAAU;QACV,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAEtE,SAAS;QACT,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;QAC9C,MAAM,OAAO,GAAG,UAAU,CAAC,UAAU,EAAE,CAAC;QACxC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAEtC,wBAAwB;QACxB,MAAM,KAAK,GAAG,IAAI,iBAAiB,CAAC,EAAE,OAAO,EAAE,IAAI,WAAW,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;QAClF,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;QACjE,uBAAuB;QACvB,MAAM,MAAM,GAAqB;YAChC,GAAG,EAAE;gBACJ,KAAK,EAAE,cAAc;gBACrB,GAAG,EAAE,UAAU;gBACf,OAAO,EAAE,SAAS;gBAClB,UAAU,EAAE,OAAO;gBACnB,MAAM,EAAE,UAAU;gBAClB,SAAS,EAAE,YAAY;gBACvB,IAAI,EAAE,EAAE;gBACR,UAAU,EAAE,CAAC;aACb;YACD,WAAW,EAAE;gBACZ,cAAc,EAAE,KAAK;gBACrB,cAAc,EAAE,IAAI;gBACpB,KAAK,EAAE,IAAI;gBACX,eAAe,EAAE,IAAI;aACrB;YACD,OAAO,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE;YACxB,OAAO,EAAE,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE;YACjC,UAAU,EAAE;gBACX,OAAO,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE;aACnE;SACD,CAAC;QAEF,MAAM,UAAU,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;QAEzC,oCAAoC;QACpC,MAAM,KAAK,GAAG,IAAI,iBAAiB,CAAC,EAAE,OAAO,EAAE,IAAI,WAAW,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;QAClF,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QAEnB,uCAAuC;QACvC,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAC5B,CAAC,aAAa,EAAE,CAAC,EACjB,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC,EAC5B;YACC,WAAW,EAAE,CAAC,EAAE,8BAA8B;YAC9C,QAAQ,EAAE,EAAE;YACZ,SAAS,EAAE,IAAI;YACf,iBAAiB,EAAE,KAAK;SACxB,EACD,CAAC,CACD,CAAC;QAEF,MAAM,CAAC,WAAW,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IACnC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;QACxD,4BAA4B;QAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,2BAA2B,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,2BAA2B,CAAC,CAAC;QAE1D,MAAM,QAAQ,GAAG,IAAI,WAAW,CAAC,MAAM,CAAC,CAAC;QACzC,MAAM,QAAQ,GAAG,IAAI,WAAW,CAAC,MAAM,CAAC,CAAC;QAEzC,MAAM,QAAQ,CAAC,IAAI,CAAC;YACnB,UAAU,EAAE,WAAW;YACvB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,eAAe,EAAE,CAAC,SAAS,EAAE,SAAS,CAAC;YACvC,OAAO,EAAE,EAAE;YACX,YAAY,EAAE,CAAC;SACf,CAAC,CAAC;QAEH,MAAM,QAAQ,CAAC,IAAI,CAAC;YACnB,UAAU,EAAE,WAAW;YACvB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,eAAe,EAAE,CAAC,SAAS,EAAE,SAAS,CAAC;YACvC,OAAO,EAAE,EAAE;YACX,YAAY,EAAE,CAAC;SACf,CAAC,CAAC;QAEH,cAAc;QACd,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACrD,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;QACnC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;QACrE,yBAAyB;QACzB,MAAM,OAAO,GAAqB;YACjC,GAAG,EAAE;gBACJ,KAAK,EAAE,SAAS;gBAChB,GAAG,EAAE,OAAO;gBACZ,OAAO,EAAE,SAAS;gBAClB,UAAU,EAAE,OAAO;gBACnB,MAAM,EAAE,UAAU;gBAClB,SAAS,EAAE,YAAY;gBACvB,IAAI,EAAE,EAAE;gBACR,UAAU,EAAE,CAAC;aACb;YACD,WAAW,EAAE;gBACZ,cAAc,EAAE,KAAK;gBACrB,cAAc,EAAE,IAAI;gBACpB,KAAK,EAAE,IAAI;gBACX,eAAe,EAAE,IAAI;aACrB;YACD,OAAO,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE;YACxB,OAAO,EAAE,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE;YACjC,UAAU,EAAE;gBACX,OAAO,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE;aACnE;SACD,CAAC;QAEF,MAAM,UAAU,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QAE1C,2CAA2C;QAC3C,MAAM,aAAa,GAAG,IAAI,WAAW,CAAC,IAAI,CAAC,OAAO,EAAE,2BAA2B,CAAC,CAAC,CAAC;QAElF,MAAM,aAAa,CAAC,IAAI,CAAC;YACxB,UAAU,EAAE,WAAW;YACvB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,eAAe,EAAE,CAAC,SAAS,EAAE,SAAS,CAAC,EAAE,6BAA6B;YACtE,OAAO,EAAE,EAAE;YACX,YAAY,EAAE,CAAC;SACf,CAAC,CAAC;QAEH,QAAQ;QACR,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACrD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QAEpD,uBAAuB;QACvB,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;QAClD,MAAM,CAAC,WAAW,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,mCAAmC;QAC1E,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yEAAyE,EAAE,KAAK,IAAI,EAAE;QACxF,+DAA+D;QAC/D,MAAM,OAAO,GAAG;YACf,eAAe,EAAE,IAAI;YACrB,WAAW,EAAE,CAAC;YACd,QAAQ,EAAE,EAAE;YACZ,SAAS,EAAE,OAAO;YAClB,iBAAiB,EAAE,IAAI;SACvB,CAAC;QAEF,MAAM,OAAO,GAAG;YACf,GAAG,OAAO;YACV,WAAW,EAAE,EAAE,EAAE,sBAAsB;SACvC,CAAC;QAEF,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC;aAChC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;aAC5D,MAAM,CAAC,KAAK,CAAC,CAAC;QAEhB,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC;aAChC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;aAC5D,MAAM,CAAC,KAAK,CAAC,CAAC;QAEhB,OAAO,CAAC,GAAG,CAAC,2BAA2B,EAAE,KAAK,CAAC,CAAC;QAChD,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,KAAK,CAAC,CAAC;QAC7C,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,KAAK,KAAK,CAAC,CAAC;QAEvC,4DAA4D;QAC5D,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IACrC,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH;;GAEG;AACH,MAAM,aAAa,GAAG,GAAkB,EAAE,CAAC,CAAC;IAC3C,YAAY,EAAE;QACb,EAAE,EAAE,iBAAiB;QACrB,IAAI,EAAE,UAAU;QAChB,OAAO,EAAE,OAAO;QAChB,IAAI,EAAE,SAAS;QACf,MAAM,EAAE,EAAE;QACV,IAAI,EAAE,CAAC,MAAM,CAAC;KACd;IACD,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;QACf,EAAE,EAAE,iBAAiB;QACrB,MAAM,EAAE,EAAE;QACV,GAAG,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;KACvC,CAAC;CACF,CAAC,CAAC;AAEH;;;GAGG;AACH,MAAM,cAAc,GAAG,CAAC,EAAU,EAAkB,EAAE,CAAC,CAAC;IACvD,IAAI,EAAE;QACL,MAAM,EAAE,EAAE;QACV,IAAI,EAAE,aAAa,EAAE,EAAE;QACvB,SAAS,EAAE,MAAM;QACjB,MAAM,EAAE,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE;QACjC,IAAI,EAAE,CAAC,MAAM,CAAC;QACd,OAAO,EAAE,OAAO;KAChB;IACD,QAAQ,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IAC3C,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;CACvC,CAAC,CAAC"}

package/dist/executor/__tests__/checkpoint-merge-bug.unit.test.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Diagnostic Unit Tests for Checkpoint Merge Bug
+ *
+ * Tests to reproduce the data loss bug where mergeShards() overwrites
+ * the main checkpoint instead of combining it with shard results.
+ */
+export {};
+//# sourceMappingURL=checkpoint-merge-bug.unit.test.d.ts.map

package/dist/executor/__tests__/checkpoint-merge-bug.unit.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"checkpoint-merge-bug.unit.test.d.ts","sourceRoot":"","sources":["../../../src/executor/__tests__/checkpoint-merge-bug.unit.test.ts"],"names":[],"mappings":"AAAA;;;;;GAKG"}

package/dist/executor/__tests__/checkpoint-merge-bug.unit.test.js

@@ -0,0 +1,164 @@
+/**
+ * Diagnostic Unit Tests for Checkpoint Merge Bug
+ *
+ * Tests to reproduce the data loss bug where mergeShards() overwrites
+ * the main checkpoint instead of combining it with shard results.
+ */
+import { randomBytes } from "node:crypto";
+import { readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { CheckpointManager } from "../checkpoint-manager.js";
+import { FileStorage } from "../checkpoint-storage.js";
+describe("Checkpoint Merge Bug Diagnostics", () => {
+    let testDir;
+    let mainCheckpoint;
+    let mainPath;
+    beforeEach(() => {
+        testDir = join(tmpdir(), `checkpoint-merge-test-${randomBytes(8).toString("hex")}`);
+        mainPath = join(testDir, "checkpoint.json");
+        mainCheckpoint = new CheckpointManager({ storage: new FileStorage(mainPath) });
+    });
+    /**
+     * Helper to read checkpoint data directly from file
+     * @param path
+     */
+    const readCheckpointFile = (path) => {
+        const data = JSON.parse(readFileSync(path, "utf-8"));
+        return data;
+    };
+    afterEach(() => {
+        rmSync(testDir, { recursive: true, force: true });
+    });
+    it("diagnostic-1: mergeShards should combine main checkpoint with shards", async () => {
+        // Setup: Create main checkpoint with 3 completed runs
+        const run1 = createMockResult("run-001", "sut-1", "case-1");
+        const run2 = createMockResult("run-002", "sut-1", "case-2");
+        const run3 = createMockResult("run-003", "sut-1", "case-3");
+        await mainCheckpoint.saveIncremental(run1);
+        await mainCheckpoint.saveIncremental(run2);
+        await mainCheckpoint.saveIncremental(run3);
+        const mainData = readCheckpointFile(mainPath);
+        assert.strictEqual(mainData.completedRunIds.length, 3);
+        // Create worker shard with 1 additional run
+        const shardPath = join(testDir, "checkpoint-worker-00.json");
+        const shardStorage = new FileStorage(shardPath);
+        await shardStorage.save({
+            configHash: "test-hash",
+            createdAt: new Date().toISOString(),
+            updatedAt: new Date().toISOString(),
+            completedRunIds: ["run-004"],
+            results: {
+                "run-004": createMockResult("run-004", "sut-1", "case-4"),
+            },
+            totalPlanned: 4,
+        });
+        // Load checkpoint to reset internal state
+        await mainCheckpoint.load();
+        // BUG: mergeShards will lose the 3 runs from main checkpoint!
+        const merged = await mainCheckpoint.mergeShards([shardPath]);
+        const mainAfterMerge = readCheckpointFile(mainPath);
+        console.log("Main checkpoint runs:", mainAfterMerge.completedRunIds.length);
+        console.log("Merged runs:", merged.completedRunIds.length);
+        console.log("Expected: 4 runs (3 from main + 1 from shard)");
+        // This should be 4, but will fail showing the bug
+        assert.strictEqual(merged.completedRunIds.length, 4);
+    });
+    it("diagnostic-2: demonstrates mergeShards creates new checkpoint instead of combining", async () => {
+        // Setup main checkpoint with data
+        await mainCheckpoint.saveIncremental(createMockResult("main-001", "sut-1", "case-1"));
+        const mainBeforeMerge = readCheckpointFile(mainPath);
+        console.log("Main checkpoint before merge:", mainBeforeMerge.completedRunIds.length);
+        // Create empty shard (simulating worker that completed no runs)
+        const shardPath = join(testDir, "checkpoint-worker-00.json");
+        const shardStorage = new FileStorage(shardPath);
+        await shardStorage.save({
+            configHash: "test-hash",
+            createdAt: new Date().toISOString(),
+            updatedAt: new Date().toISOString(),
+            completedRunIds: [],
+            results: {},
+            totalPlanned: 132,
+        });
+        await mainCheckpoint.load();
+        // Merge empty shard
+        await mainCheckpoint.mergeShards([shardPath]);
+        const mainAfterMerge = readCheckpointFile(mainPath);
+        console.log("Main checkpoint after merge:", mainAfterMerge.completedRunIds.length);
+        // BUG: The main checkpoint's run is lost!
+        assert.ok(mainAfterMerge.completedRunIds.length > 0);
+    });
+    it("diagnostic-3: shows mergeShards doesn't preserve main checkpoint data", async () => {
+        // Create main checkpoint
+        await mainCheckpoint.saveIncremental(createMockResult("main-001", "sut-1", "case-1"));
+        await mainCheckpoint.saveIncremental(createMockResult("main-002", "sut-2", "case-2"));
+        const mainDataBefore = readCheckpointFile(mainPath);
+        const mainRuns = mainDataBefore.completedRunIds;
+        console.log("Main checkpoint has runs:", mainRuns);
+        // Create worker shard
+        const shardPath = join(testDir, "checkpoint-worker-00.json");
+        const shardStorage = new FileStorage(shardPath);
+        await shardStorage.save({
+            configHash: "test-hash",
+            createdAt: new Date().toISOString(),
+            updatedAt: new Date().toISOString(),
+            completedRunIds: ["shard-001"],
+            results: {
+                "shard-001": createMockResult("shard-001", "sut-3", "case-3"),
+            },
+            totalPlanned: 132,
+        });
+        // Reload main checkpoint
+        await mainCheckpoint.load();
+        // Merge
+        const merged = await mainCheckpoint.mergeShards([shardPath]);
+        const mainDataAfter = readCheckpointFile(mainPath);
+        console.log("After merge:");
+        console.log("  Main checkpoint runs:", mainDataAfter.completedRunIds);
+        console.log("  Merged runs:", merged.completedRunIds);
+        console.log("  Expected: 3 runs (2 from main + 1 from shard)");
+        console.log("  Actual:", merged.completedRunIds.length, "runs");
+        // Should have both main runs AND shard run
+        const expectedRunIds = new Set([...mainRuns, "shard-001"]);
+        const actualRunIds = new Set(merged.completedRunIds);
+        console.log("  Missing runs:", [...expectedRunIds].filter((x) => !actualRunIds.has(x)));
+        // Check all expected runs are present
+        for (const runId of expectedRunIds) {
+            assert.ok(actualRunIds.has(runId), `Expected run ${runId} to be in merged results`);
+        }
+        // Check counts match
+        assert.strictEqual(actualRunIds.size, expectedRunIds.size);
+    });
+});
+/**
+ * Create a mock evaluation result for testing
+ * @param runId
+ * @param sut
+ * @param caseId
+ */
+const createMockResult = (runId, sut, caseId) => ({
+    run: {
+        runId,
+        sut,
+        sutRole: "primary",
+        sutVersion: "1.0.0",
+        caseId,
+        caseClass: "test-class",
+        seed: 42,
+        repetition: 0,
+    },
+    correctness: {
+        expectedExists: false,
+        producedOutput: true,
+        valid: true,
+        matchesExpected: null,
+    },
+    outputs: { summary: {} },
+    metrics: { numeric: { test: 1 } },
+    provenance: {
+        runtime: { platform: "linux", arch: "x64", nodeVersion: "v22.0.0" },
+    },
+});
+//# sourceMappingURL=checkpoint-merge-bug.unit.test.js.map

package/dist/executor/__tests__/checkpoint-merge-bug.unit.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"checkpoint-merge-bug.unit.test.js","sourceRoot":"","sources":["../../../src/executor/__tests__/checkpoint-merge-bug.unit.test.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAC/C,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,WAAW,CAAC;AAChE,OAAO,MAAM,MAAM,oBAAoB,CAAC;AAIxC,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAEvD,QAAQ,CAAC,kCAAkC,EAAE,GAAG,EAAE;IACjD,IAAI,OAAe,CAAC;IACpB,IAAI,cAAiC,CAAC;IACtC,IAAI,QAAgB,CAAC;IAErB,UAAU,CAAC,GAAG,EAAE;QACf,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,yBAAyB,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACpF,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;QAC5C,cAAc,GAAG,IAAI,iBAAiB,CAAC,EAAE,OAAO,EAAE,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAChF,CAAC,CAAC,CAAC;IAEH;;;OAGG;IACH,MAAM,kBAAkB,GAAG,CAAC,IAAY,EAAkB,EAAE;QAC3D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;QACrD,OAAO,IAAsB,CAAC;IAC/B,CAAC,CAAC;IAEF,SAAS,CAAC,GAAG,EAAE;QACd,MAAM,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sEAAsE,EAAE,KAAK,IAAI,EAAE;QACrF,sDAAsD;QACtD,MAAM,IAAI,GAAqB,gBAAgB,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;QAC9E,MAAM,IAAI,GAAqB,gBAAgB,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;QAC9E,MAAM,IAAI,GAAqB,gBAAgB,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;QAE9E,MAAM,cAAc,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;QAC3C,MAAM,cAAc,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;QAC3C,MAAM,cAAc,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;QAE3C,MAAM,QAAQ,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QAC9C,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAEvD,4CAA4C;QAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,EAAE,2BAA2B,CAAC,CAAC;QAC7D,MAAM,YAAY,GAAG,IAAI,WAAW,CAAC,SAAS,CAAC,CAAC;QAChD,MAAM,YAAY,CAAC,IAAI,CAAC;YACvB,UAAU,EAAE,WAAW;YACvB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,eAAe,EAAE,CAAC,SAAS,CAAC;YAC5B,OAAO,EAAE;gBACR,SAAS,EAAE,gBAAgB,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC;aACzD;YACD,YAAY,EAAE,CAAC;SACf,CAAC,CAAC;QAEH,0CAA0C;QAC1C,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QAE5B,8DAA8D;QAC9D,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;QAE7D,MAAM,cAAc,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QACpD,OAAO,CAAC,GAAG,CAAC,uBAAuB,EAAE,cAAc,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;QAC5E,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;QAC3D,OAAO,CAAC,GAAG,CAAC,+CAA+C,CAAC,CAAC;QAE7D,kDAAkD;QAClD,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oFAAoF,EAAE,KAAK,IAAI,EAAE;QACnG,kCAAkC;QAClC,MAAM,cAAc,CAAC,eAAe,CAAC,gBAAgB,CAAC,UAAU,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;QAEtF,MAAM,eAAe,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,+BAA+B,EAAE,eAAe,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;QAErF,gEAAgE;QAChE,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,EAAE,2BAA2B,CAAC,CAAC;QAC7D,MAAM,YAAY,GAAG,IAAI,WAAW,CAAC,SAAS,CAAC,CAAC;QAChD,MAAM,YAAY,CAAC,IAAI,CAAC;YACvB,UAAU,EAAE,WAAW;YACvB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,eAAe,EAAE,EAAE;YACnB,OAAO,EAAE,EAAE;YACX,YAAY,EAAE,GAAG;SACjB,CAAC,CAAC;QAEH,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QAE5B,oBAAoB;QACpB,MAAM,cAAc,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;QAE9C,MAAM,cAAc,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QACpD,OAAO,CAAC,GAAG,CAAC,8BAA8B,EAAE,cAAc,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;QAEnF,0CAA0C;QAC1C,MAAM,CAAC,EAAE,CAAC,cAAc,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uEAAuE,EAAE,KAAK,IAAI,EAAE;QACtF,yBAAyB;QACzB,MAAM,cAAc,CAAC,eAAe,CAAC,gBAAgB,CAAC,UAAU,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;QACtF,MAAM,cAAc,CAAC,eAAe,CAAC,gBAAgB,CAAC,UAAU,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;QAEtF,MAAM,cAAc,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QACpD,MAAM,QAAQ,GAAG,cAAc,CAAC,eAAe,CAAC;QAChD,OAAO,CAAC,GAAG,CAAC,2BAA2B,EAAE,QAAQ,CAAC,CAAC;QAEnD,sBAAsB;QACtB,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,EAAE,2BAA2B,CAAC,CAAC;QAC7D,MAAM,YAAY,GAAG,IAAI,WAAW,CAAC,SAAS,CAAC,CAAC;QAChD,MAAM,YAAY,CAAC,IAAI,CAAC;YACvB,UAAU,EAAE,WAAW;YACvB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,eAAe,EAAE,CAAC,WAAW,CAAC;YAC9B,OAAO,EAAE;gBACR,WAAW,EAAE,gBAAgB,CAAC,WAAW,EAAE,OAAO,EAAE,QAAQ,CAAC;aAC7D;YACD,YAAY,EAAE,GAAG;SACjB,CAAC,CAAC;QAEH,yBAAyB;QACzB,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QAE5B,QAAQ;QACR,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;QAE7D,MAAM,aAAa,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QACnD,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,yBAAyB,EAAE,aAAa,CAAC,eAAe,CAAC,CAAC;QACtE,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,MAAM,CAAC,eAAe,CAAC,CAAC;QACtD,OAAO,CAAC,GAAG,CAAC,iDAAiD,CAAC,CAAC;QAC/D,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,MAAM,CAAC,eAAe,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAEhE,2CAA2C;QAC3C,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,QAAQ,EAAE,WAAW,CAAC,CAAC,CAAC;QAC3D,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;QAErD,OAAO,CAAC,GAAG,CACV,iBAAiB,EACjB,CAAC,GAAG,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CACvD,CAAC;QAEF,sCAAsC;QACtC,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;YACpC,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,gBAAgB,KAAK,0BAA0B,CAAC,CAAC;QACrF,CAAC;QACD,qBAAqB;QACrB,MAAM,CAAC,WAAW,CAAC,YAAY,CAAC,IAAI,EAAE,cAAc,CAAC,IAAI,CAAC,CAAC;IAC5D,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH;;;;;GAKG;AACH,MAAM,gBAAgB,GAAG,CAAC,KAAa,EAAE,GAAW,EAAE,MAAc,EAAoB,EAAE,CAAC,CAAC;IAC3F,GAAG,EAAE;QACJ,KAAK;QACL,GAAG;QACH,OAAO,EAAE,SAAkB;QAC3B,UAAU,EAAE,OAAO;QACnB,MAAM;QACN,SAAS,EAAE,YAAY;QACvB,IAAI,EAAE,EAAE;QACR,UAAU,EAAE,CAAC;KACb;IACD,WAAW,EAAE;QACZ,cAAc,EAAE,KAAK;QACrB,cAAc,EAAE,IAAI;QACpB,KAAK,EAAE,IAAI;QACX,eAAe,EAAE,IAAI;KACrB;IACD,OAAO,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE;IACxB,OAAO,EAAE,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE;IACjC,UAAU,EAAE;QACX,OAAO,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE;KACnE;CACD,CAAC,CAAC"}

package/dist/executor/__tests__/checkpoint-storage.unit.test.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Unit tests for CheckpointStorage
+ *
+ * Tests FileStorage, GitStorage, FileSystem, and Lock implementations.
+ */
+export {};
+//# sourceMappingURL=checkpoint-storage.unit.test.d.ts.map

package/dist/executor/__tests__/checkpoint-storage.unit.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"checkpoint-storage.unit.test.d.ts","sourceRoot":"","sources":["../../../src/executor/__tests__/checkpoint-storage.unit.test.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}