ppef 1.0.1 → 1.2.0

package/dist/executor/worker-threads-executor.js

@@ -0,0 +1,332 @@
+/**
+ * Worker Threads Executor
+ *
+ * Executes experiments using Node.js worker threads for parallelism.
+ * Each worker runs in isolation, preventing SUT crashes from affecting the main process.
+ *
+ * Architecture:
+ * - Main thread spawns N worker threads (75% of CPU cores by default)
+ * - Each worker receives a batch of runs via postMessage
+ * - Workers write results to sharded checkpoint files
+ * - Main thread merges shards after all workers complete
+ *
+ * Benefits:
+ * - Main thread isolation: SUT crashes don't crash the CLI
+ * - True parallelism: Workers run on separate OS threads
+ * - Resource managed: Automatic 75% resource allocation
+ * - Fault tolerance: Single worker failure doesn't stop others
+ *
+ * Dependency Injection:
+ * - Logger interface enables testing of log output
+ * - WorkerFactory interface enables mocking Worker for testing
+ * - WorkerEntryPath interface enables testing worker path resolution
+ */
+import { resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+/**
+ * Production logger using console.
+ */
+export class ConsoleLogger {
+    log(message) {
+        console.log(message);
+    }
+    debug(message) {
+        console.log(message);
+    }
+    info(message) {
+        console.log(message);
+    }
+    warn(message) {
+        console.warn(message);
+    }
+}
+import { Worker } from "node:worker_threads";
+/**
+ * Production worker factory using node:worker_threads.
+ * Registers the tsx loader when spawning workers from TypeScript source.
+ */
+export class WorkerFactory {
+    create(workerPath) {
+        const execArgv = workerPath.endsWith(".ts") ? ["--import", "tsx"] : [];
+        return new Worker(workerPath, { execArgv });
+    }
+}
+/**
+ * Production path resolver using import.meta.url.
+ * Handles both compiled (dist/*.js) and source (src/*.ts via tsx) contexts.
+ */
+export class WorkerEntryPath {
+    getWorkerEntryPath() {
+        const __filename = fileURLToPath(import.meta.url);
+        const __dirname = resolve(__filename, "..");
+        const ext = __filename.endsWith(".ts") ? ".ts" : ".js";
+        return resolve(__dirname, `worker-entry${ext}`);
+    }
+}
+/**
+ * Worker threads executor class with dependency injection.
+ *
+ * Executes experiments using Node.js worker threads for parallelism.
+ * Each worker runs in isolation, preventing SUT crashes from affecting the main process.
+ */
+export class WorkerThreadsExecutor {
+    logger;
+    workerFactory;
+    workerEntryPath;
+    baseDir;
+    useRegistryManifest;
+    constructor(options = {}) {
+        this.logger = options.logger ?? new ConsoleLogger();
+        this.workerFactory = options.workerFactory ?? new WorkerFactory();
+        this.workerEntryPath = options.workerEntryPath ?? new WorkerEntryPath();
+        this.baseDir = options.baseDir ?? process.cwd();
+        this.useRegistryManifest = options.useRegistryManifest ?? false;
+    }
+    /**
+     * Split runs into batches for parallel processing.
+     * @param runs - Runs to distribute
+     * @param numberWorkers - Number of worker threads
+     * @returns Array of run batches
+     */
+    _createBatches(runs, numberWorkers) {
+        const batchSize = Math.ceil(runs.length / numberWorkers);
+        const batches = [];
+        for (let i = 0; i < runs.length; i += batchSize) {
+            const batch = runs.slice(i, i + batchSize);
+            const batchIndex = Math.floor(i / batchSize);
+            const runIds = batch.map((r) => r.runId);
+            batches.push({
+                index: batchIndex,
+                runIds,
+                runs: batch,
+                firstRunId: batch[0]?.runId ?? "none",
+                lastRunId: batch.at(-1)?.runId ?? "none",
+            });
+        }
+        return batches;
+    }
+    /**
+     * Execute runs using multiple worker threads.
+     *
+     * Each worker writes to its own sharded checkpoint file to avoid race conditions.
+     * After all workers complete, results are aggregated and returned.
+     *
+     * @param runs - Planned runs to execute
+     * @param suts - SUT definitions (serialized and passed to workers)
+     * @param cases - Case definitions (serialized and passed to workers)
+     * @param config - Executor configuration
+     * @param options - Worker threads executor options
+     * @param checkpointDir - Checkpoint directory for shard files
+     * @returns Execution results
+     */
+    async execute(runs, suts, cases, config, options = {}, checkpointDir = resolve(process.cwd(), "results/execute")) {
+        const workerCount = options.workers ?? 1;
+        this.logger.info(`WorkerThreadsExecutor: Spawning ${workerCount} workers for ${runs.length} runs`);
+        this.logger.info(`Checkpoint directory: ${checkpointDir}`);
+        // Split runs into batches
+        const batches = this._createBatches(runs, workerCount);
+        // Log batch information
+        for (const batch of batches) {
+            this.logger.debug(`Batch ${batch.index} has ${batch.runIds.length} runs`);
+            this.logger.debug(`  First run: ${batch.firstRunId}, Last run: ${batch.lastRunId}`);
+        }
+        // Get worker entry path
+        const workerPath = this.workerEntryPath.getWorkerEntryPath();
+        this.logger.debug(`Worker entry path: ${workerPath}`);
+        // Spawn workers
+        const workerStates = this._spawnWorkers(batches, workerPath, checkpointDir, config, suts, cases);
+        // Wait for all workers to complete
+        await this._waitForWorkers(workerStates);
+        // Aggregate results
+        const allResults = [];
+        const allErrors = [];
+        for (const state of workerStates) {
+            allResults.push(...state.results);
+            allErrors.push(...state.errors);
+        }
+        this.logger.info(`All workers completed: ${allResults.length} results, ${allErrors.length} errors`);
+        return { results: allResults, errors: allErrors };
+    }
+    /**
+     * Spawn worker threads for all batches.
+     * @param batches - Run batches
+     * @param workerPath - Path to worker entry point
+     * @param checkpointDir - Checkpoint directory
+     * @param config - Executor configuration
+     * @param suts - SUT definitions to serialize
+     * @param cases - Case definitions to serialize
+     * @returns Array of worker states
+     */
+    _spawnWorkers(batches, workerPath, checkpointDir, config, suts, cases) {
+        const workerStates = [];
+        // Serialize SUTs for WorkerMessage
+        const serializedSuts = suts.map((sut) => ({
+            id: sut.registration.id,
+            module: `./dist/suts/${sut.registration.id}.js`,
+            exportName: "createSut",
+            registration: {
+                name: sut.registration.name,
+                version: sut.registration.version,
+                role: sut.registration.role,
+            },
+        }));
+        // Serialize cases for WorkerMessage
+        const serializedCases = cases.map((c) => ({
+            caseId: c.case.caseId,
+            module: `./dist/cases/${c.case.caseId}.js`,
+            exportName: "createCase",
+        }));
+        for (const batch of batches) {
+            const checkpointPath = resolve(checkpointDir, `checkpoint-worker-${String(batch.index).padStart(2, "0")}.json`);
+            const worker = this.workerFactory.create(workerPath);
+            const state = {
+                worker,
+                index: batch.index,
+                batch,
+                checkpointPath,
+                completed: false,
+                results: [],
+                errors: [],
+            };
+            // Set up message handler
+            worker.on("message", (data) => {
+                this._handleWorkerMessage(state, data, config);
+            });
+            // Set up error handler
+            worker.on("error", (error) => {
+                this.logger.warn(`Worker ${batch.index} error: ${error.message}`);
+                state.completed = true;
+                // Add error for all runs in this batch
+                for (const runId of batch.runIds) {
+                    state.errors.push({ runId, error: error.message });
+                }
+            });
+            // Send initial message to worker with serialized SUTs and cases
+            const workerMessage = {
+                runs: batch.runs.map((run) => ({
+                    runId: run.runId,
+                    sutId: run.sutId,
+                    caseId: run.caseId,
+                    repetition: run.repetition,
+                    config: run.config,
+                })),
+                config: {
+                    repetitions: config.repetitions,
+                    seedBase: config.seedBase,
+                    continueOnError: config.continueOnError,
+                    timeoutMs: config.timeoutMs,
+                    collectProvenance: config.collectProvenance,
+                },
+                baseDir: this.baseDir,
+                suts: serializedSuts,
+                cases: serializedCases,
+            };
+            // Add registry manifest if useRegistryManifest is enabled
+            if (this.useRegistryManifest) {
+                const registryManifest = {
+                    suts: suts.map((sut) => ({
+                        id: sut.registration.id,
+                        name: sut.registration.name,
+                        version: sut.registration.version,
+                        role: sut.registration.role,
+                        config: {},
+                        tags: [],
+                    })),
+                    sharedCode: "", // Registry code not bundled in this implementation
+                    sutModules: Object.fromEntries(suts.map((sut) => [sut.registration.id, `./dist/suts/${sut.registration.id}.js`])),
+                    exportName: "createSut",
+                };
+                workerMessage.registryManifest = registryManifest;
+            }
+            worker.postMessage(workerMessage);
+            workerStates.push(state);
+        }
+        return workerStates;
+    }
+    /**
+     * Handle a message from a worker.
+     * @param state - Worker state
+     * @param data - Message data
+     * @param config - Executor configuration
+     */
+    _handleWorkerMessage(state, data, config) {
+        // Use type guards to discriminate the union type
+        const isError = (msg) => msg.type === "error";
+        const isSuccess = (msg) => msg.type === "done";
+        // Handle WorkerErrorMessage
+        if (isError(data)) {
+            this.logger.warn(`Worker ${state.index} error: ${data.error}`);
+            state.completed = true;
+            for (const runId of state.batch.runIds) {
+                state.errors.push({ runId, error: data.error });
+            }
+            return;
+        }
+        // Handle WorkerSuccessMessage
+        if (isSuccess(data)) {
+            state.results.push(...data.results);
+            state.errors.push(...data.errors);
+            state.completed = true;
+            this.logger.debug(`Worker ${state.index} completed: ${data.results.length} results`);
+            // Call onResult callback for each result
+            if (config.onResult) {
+                for (const result of data.results) {
+                    try {
+                        void config.onResult(result);
+                    }
+                    catch (error) {
+                        this.logger.warn(`Error in onResult callback: ${String(error)}`);
+                    }
+                }
+            }
+        }
+    }
+    /**
+     * Wait for all workers to complete.
+     * @param workerStates - Worker states
+     * @returns Promise that resolves when all workers complete
+     */
+    async _waitForWorkers(workerStates) {
+        return new Promise((resolve, reject) => {
+            let completedCount = 0;
+            const totalWorkers = workerStates.length;
+            const checkCompletion = () => {
+                completedCount = workerStates.filter((s) => s.completed).length;
+                if (completedCount === totalWorkers) {
+                    clearInterval(interval);
+                    clearTimeout(timeout);
+                    resolve();
+                }
+            };
+            // Poll for completion
+            const interval = setInterval(checkCompletion, 100);
+            // Timeout after 1 hour
+            const timeout = setTimeout(() => {
+                clearInterval(interval);
+                const incomplete = workerStates.filter((s) => !s.completed).map((s) => s.index);
+                reject(new Error(`Worker timeout after 1 hour. Incomplete workers: ${incomplete.join(", ")}`));
+            }, 3_600_000);
+        });
+    }
+}
+/**
+ * Execute runs using multiple worker threads.
+ *
+ * Each worker writes to its own sharded checkpoint file to avoid race conditions.
+ * After all workers complete, results are aggregated and returned.
+ *
+ * This is a convenience function that creates a WorkerThreadsExecutor with default dependencies.
+ * For testing or custom behavior, use the WorkerThreadsExecutor class directly.
+ *
+ * @param runs - Planned runs to execute
+ * @param suts - SUT definitions (serialized and passed to workers)
+ * @param cases - Case definitions (serialized and passed to workers)
+ * @param config - Executor configuration
+ * @param options - Worker threads executor options
+ * @returns Execution results
+ */
+export const executeWithWorkerThreads = async (runs, suts, cases, config, options = {}) => {
+    const executor = new WorkerThreadsExecutor(options);
+    return executor.execute(runs, suts, cases, config, options);
+};
+//# sourceMappingURL=worker-threads-executor.js.map

package/dist/executor/worker-threads-executor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker-threads-executor.js","sourceRoot":"","sources":["../../src/executor/worker-threads-executor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AA0BzC;;GAEG;AACH,MAAM,OAAO,aAAa;IACzB,GAAG,CAAC,OAAe;QAClB,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,OAAe;QACpB,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACtB,CAAC;IAED,IAAI,CAAC,OAAe;QACnB,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACtB,CAAC;IAED,IAAI,CAAC,OAAe;QACnB,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvB,CAAC;CACD;AAqBD,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAE7C;;;GAGG;AACH,MAAM,OAAO,aAAa;IACzB,MAAM,CAAC,UAAkB;QACxB,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,OAAO,IAAI,MAAM,CAAC,UAAU,EAAE,EAAE,QAAQ,EAAE,CAAY,CAAC;IACxD,CAAC;CACD;AAaD;;;GAGG;AACH,MAAM,OAAO,eAAe;IAC3B,kBAAkB;QACjB,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClD,MAAM,SAAS,GAAG,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QAC5C,MAAM,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;QACvD,OAAO,OAAO,CAAC,SAAS,EAAE,eAAe,GAAG,EAAE,CAAC,CAAC;IACjD,CAAC;CACD;AA6ED;;;;;GAKG;AACH,MAAM,OAAO,qBAAqB;IAChB,MAAM,CAAU;IAChB,aAAa,CAAiB;IAC9B,eAAe,CAAmB;IAClC,OAAO,CAAS;IAChB,mBAAmB,CAAU;IAE9C,YAAY,UAAwC,EAAE;QACrD,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;QACpD,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,IAAI,aAAa,EAAE,CAAC;QAClE,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,IAAI,eAAe,EAAE,CAAC;QACxE,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;QAChD,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,IAAI,KAAK,CAAC;IACjE,CAAC;IAED;;;;;OAKG;IACK,cAAc,CAAC,IAAkB,EAAE,aAAqB;QAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,aAAa,CAAC,CAAC;QACzD,MAAM,OAAO,GAAe,EAAE,CAAC;QAE/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YACjD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YAC3C,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;YAC7C,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YAEzC,OAAO,CAAC,IAAI,CAAC;gBACZ,KAAK,EAAE,UAAU;gBACjB,MAAM;gBACN,IAAI,EAAE,KAAK;gBACX,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,MAAM;gBACrC,SAAS,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,MAAM;aACxC,CAAC,CAAC;QACJ,CAAC;QAED,OAAO,OAAO,CAAC;IAChB,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,OAAO,CACZ,IAAkB,EAClB,IAGG,EACH,KAIG,EACH,MAA0E,EAC1E,UAAwC,EAAE,EAC1C,aAAa,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,iBAAiB,CAAC;QAEzD,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC;QAEzC,IAAI,CAAC,MAAM,CAAC,IAAI,CACf,mCAAmC,WAAW,gBAAgB,IAAI,CAAC,MAAM,OAAO,CAChF,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,aAAa,EAAE,CAAC,CAAC;QAE3D,0BAA0B;QAC1B,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QAEvD,wBAAwB;QACxB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,KAAK,CAAC,KAAK,QAAQ,KAAK,CAAC,MAAM,CAAC,MAAM,OAAO,CAAC,CAAC;YAC1E,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gBAAgB,KAAK,CAAC,UAAU,eAAe,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;QACrF,CAAC;QAED,wBAAwB;QACxB,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,kBAAkB,EAAE,CAAC;QAC7D,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,sBAAsB,UAAU,EAAE,CAAC,CAAC;QAEtD,gBAAgB;QAChB,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,CACtC,OAAO,EACP,UAAU,EACV,aAAa,EACb,MAAM,EACN,IAAI,EACJ,KAAK,CACL,CAAC;QAEF,mCAAmC;QACnC,MAAM,IAAI,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC;QAEzC,oBAAoB;QACpB,MAAM,UAAU,GAAuB,EAAE,CAAC;QAC1C,MAAM,SAAS,GAAuC,EAAE,CAAC;QAEzD,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;YAClC,UAAU,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC;YAClC,SAAS,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;QACjC,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CACf,0BAA0B,UAAU,CAAC,MAAM,aAAa,SAAS,CAAC,MAAM,SAAS,CACjF,CAAC;QAEF,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IACnD,CAAC;IAED;;;;;;;;;OASG;IACK,aAAa,CACpB,OAAmB,EACnB,UAAkB,EAClB,aAAqB,EACrB,MAAsB,EACtB,IAGG,EACH,KAIG;QAEH,MAAM,YAAY,GAAkB,EAAE,CAAC;QAEvC,mCAAmC;QACnC,MAAM,cAAc,GAAoB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAC1D,EAAE,EAAE,GAAG,CAAC,YAAY,CAAC,EAAE;YACvB,MAAM,EAAE,eAAe,GAAG,CAAC,YAAY,CAAC,EAAE,KAAK;YAC/C,UAAU,EAAE,WAAW;YACvB,YAAY,EAAE;gBACb,IAAI,EAAE,GAAG,CAAC,YAAY,CAAC,IAAI;gBAC3B,OAAO,EAAE,GAAG,CAAC,YAAY,CAAC,OAAO;gBACjC,IAAI,EAAE,GAAG,CAAC,YAAY,CAAC,IAAI;aAC3B;SACD,CAAC,CAAC,CAAC;QAEJ,oCAAoC;QACpC,MAAM,eAAe,GAAqB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3D,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM;YACrB,MAAM,EAAE,gBAAgB,CAAC,CAAC,IAAI,CAAC,MAAM,KAAK;YAC1C,UAAU,EAAE,YAAY;SACxB,CAAC,CAAC,CAAC;QAEJ,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,cAAc,GAAG,OAAO,CAC7B,aAAa,EACb,qBAAqB,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,OAAO,CAChE,CAAC;YAEF,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YACrD,MAAM,KAAK,GAAgB;gBAC1B,MAAM;gBACN,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,KAAK;gBACL,cAAc;gBACd,SAAS,EAAE,KAAK;gBAChB,OAAO,EAAE,EAAE;gBACX,MAAM,EAAE,EAAE;aACV,CAAC;YAEF,yBAAyB;YACzB,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,IAAyB,EAAE,EAAE;gBAClD,IAAI,CAAC,oBAAoB,CAAC,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;YAChD,CAAC,CAAC,CAAC;YAEH,uBAAuB;YACvB,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAY,EAAE,EAAE;gBACnC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,KAAK,WAAW,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;gBAClE,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC;gBACvB,uCAAuC;gBACvC,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;oBAClC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;gBACpD,CAAC;YACF,CAAC,CAAC,CAAC;YAEH,gEAAgE;YAChE,MAAM,aAAa,GAAkB;gBACpC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;oBAC9B,KAAK,EAAE,GAAG,CAAC,KAAK;oBAChB,KAAK,EAAE,GAAG,CAAC,KAAK;oBAChB,MAAM,EAAE,GAAG,CAAC,MAAM;oBAClB,UAAU,EAAE,GAAG,CAAC,UAAU;oBAC1B,MAAM,EAAE,GAAG,CAAC,MAAiB;iBAC7B,CAAC,CAAC;gBACH,MAAM,EAAE;oBACP,WAAW,EAAE,MAAM,CAAC,WAAW;oBAC/B,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,eAAe,EAAE,MAAM,CAAC,eAAe;oBACvC,SAAS,EAAE,MAAM,CAAC,SAAS;oBAC3B,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;iBAC3C;gBACD,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,IAAI,EAAE,cAAc;gBACpB,KAAK,EAAE,eAAe;aACtB,CAAC;YAEF,0DAA0D;YAC1D,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAC9B,MAAM,gBAAgB,GAAqB;oBAC1C,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;wBACxB,EAAE,EAAE,GAAG,CAAC,YAAY,CAAC,EAAE;wBACvB,IAAI,EAAE,GAAG,CAAC,YAAY,CAAC,IAAI;wBAC3B,OAAO,EAAE,GAAG,CAAC,YAAY,CAAC,OAAO;wBACjC,IAAI,EAAE,GAAG,CAAC,YAAY,CAAC,IAAI;wBAC3B,MAAM,EAAE,EAA6B;wBACrC,IAAI,EAAE,EAAc;qBACpB,CAAC,CAAC;oBACH,UAAU,EAAE,EAAE,EAAE,mDAAmD;oBACnE,UAAU,EAAE,MAAM,CAAC,WAAW,CAC7B,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,EAAE,eAAe,GAAG,CAAC,YAAY,CAAC,EAAE,KAAK,CAAC,CAAC,CACjF;oBACD,UAAU,EAAE,WAAW;iBACvB,CAAC;gBACF,aAAa,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;YACnD,CAAC;YAED,MAAM,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;YAElC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC1B,CAAC;QAED,OAAO,YAAY,CAAC;IACrB,CAAC;IAED;;;;;OAKG;IACK,oBAAoB,CAC3B,KAAkB,EAClB,IAAyB,EACzB,MAAsB;QAEtB,iDAAiD;QACjD,MAAM,OAAO,GAAG,CAAC,GAAwB,EAA6B,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,OAAO,CAAC;QAE9F,MAAM,SAAS,GAAG,CAAC,GAAwB,EAA+B,EAAE,CAC3E,GAAG,CAAC,IAAI,KAAK,MAAM,CAAC;QAErB,4BAA4B;QAC5B,IAAI,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YACnB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,KAAK,WAAW,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;YAC/D,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC;YACvB,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;gBACxC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;YACjD,CAAC;YACD,OAAO;QACR,CAAC;QAED,8BAA8B;QAC9B,IAAI,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;YACrB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC;YACpC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;YAClC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC;YACvB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,KAAK,CAAC,KAAK,eAAe,IAAI,CAAC,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;YAErF,yCAAyC;YACzC,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;gBACrB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;oBACnC,IAAI,CAAC;wBACJ,KAAK,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;oBAC9B,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBAChB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,+BAA+B,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;oBAClE,CAAC;gBACF,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,eAAe,CAAC,YAA2B;QACxD,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC5C,IAAI,cAAc,GAAG,CAAC,CAAC;YACvB,MAAM,YAAY,GAAG,YAAY,CAAC,MAAM,CAAC;YAEzC,MAAM,eAAe,GAAG,GAAG,EAAE;gBAC5B,cAAc,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC;gBAChE,IAAI,cAAc,KAAK,YAAY,EAAE,CAAC;oBACrC,aAAa,CAAC,QAAQ,CAAC,CAAC;oBACxB,YAAY,CAAC,OAAO,CAAC,CAAC;oBACtB,OAAO,EAAE,CAAC;gBACX,CAAC;YACF,CAAC,CAAC;YAEF,sBAAsB;YACtB,MAAM,QAAQ,GAAG,WAAW,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;YAEnD,uBAAuB;YACvB,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC/B,aAAa,CAAC,QAAQ,CAAC,CAAC;gBACxB,MAAM,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;gBAChF,MAAM,CACL,IAAI,KAAK,CAAC,oDAAoD,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CACtF,CAAC;YACH,CAAC,EAAE,SAAS,CAAC,CAAC;QACf,CAAC,CAAC,CAAC;IACJ,CAAC;CACD;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,KAAK,EAC5C,IAAkB,EAClB,IAGG,EACH,KAAiG,EACjG,MAA0E,EAC1E,UAAkE,EAAE,EACmB,EAAE;IACzF,MAAM,QAAQ,GAAG,IAAI,qBAAqB,CAAC,OAAO,CAAC,CAAC;IACpD,OAAO,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;AAC7D,CAAC,CAAC"}

package/dist/index.d.ts

@@ -9,6 +9,7 @@ export * from "./executor/index.js";
 export * from "./collector/index.js";
 export * from "./statistical/index.js";
 export * from "./aggregation/index.js";
+export * from "./evaluators/index.js";
 export * from "./claims/index.js";
 export * from "./robustness/index.js";
 export * from "./renderers/index.js";

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,cAAc,kBAAkB,CAAC;AAGjC,cAAc,qBAAqB,CAAC;AAGpC,cAAc,qBAAqB,CAAC;AAGpC,cAAc,sBAAsB,CAAC;AAGrC,cAAc,wBAAwB,CAAC;AAGvC,cAAc,wBAAwB,CAAC;AAGvC,cAAc,mBAAmB,CAAC;AAGlC,cAAc,uBAAuB,CAAC;AAGtC,cAAc,sBAAsB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,cAAc,kBAAkB,CAAC;AAGjC,cAAc,qBAAqB,CAAC;AAGpC,cAAc,qBAAqB,CAAC;AAGpC,cAAc,sBAAsB,CAAC;AAGrC,cAAc,wBAAwB,CAAC;AAGvC,cAAc,wBAAwB,CAAC;AAGvC,cAAc,uBAAuB,CAAC;AAGtC,cAAc,mBAAmB,CAAC;AAGlC,cAAc,uBAAuB,CAAC;AAGtC,cAAc,sBAAsB,CAAC"}

package/dist/index.js

@@ -15,9 +15,11 @@ export * from "./collector/index.js";
 export * from "./statistical/index.js";
 // Aggregation
 export * from "./aggregation/index.js";
-// Claims
+// Evaluators (new extensible evaluation system)
+export * from "./evaluators/index.js";
+// Claims (types only)
 export * from "./claims/index.js";
-// Robustness
+// Robustness (types and perturbations only)
 export * from "./robustness/index.js";
 // Renderers
 export * from "./renderers/index.js";

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,QAAQ;AACR,cAAc,kBAAkB,CAAC;AAEjC,WAAW;AACX,cAAc,qBAAqB,CAAC;AAEpC,WAAW;AACX,cAAc,qBAAqB,CAAC;AAEpC,YAAY;AACZ,cAAc,sBAAsB,CAAC;AAErC,cAAc;AACd,cAAc,wBAAwB,CAAC;AAEvC,cAAc;AACd,cAAc,wBAAwB,CAAC;AAEvC,SAAS;AACT,cAAc,mBAAmB,CAAC;AAElC,aAAa;AACb,cAAc,uBAAuB,CAAC;AAEtC,YAAY;AACZ,cAAc,sBAAsB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,QAAQ;AACR,cAAc,kBAAkB,CAAC;AAEjC,WAAW;AACX,cAAc,qBAAqB,CAAC;AAEpC,WAAW;AACX,cAAc,qBAAqB,CAAC;AAEpC,YAAY;AACZ,cAAc,sBAAsB,CAAC;AAErC,cAAc;AACd,cAAc,wBAAwB,CAAC;AAEvC,cAAc;AACd,cAAc,wBAAwB,CAAC;AAEvC,gDAAgD;AAChD,cAAc,uBAAuB,CAAC;AAEtC,sBAAsB;AACtB,cAAc,mBAAmB,CAAC;AAElC,4CAA4C;AAC5C,cAAc,uBAAuB,CAAC;AAEtC,YAAY;AACZ,cAAc,sBAAsB,CAAC"}

package/dist/renderers/latex-renderer.d.ts

@@ -6,6 +6,7 @@
  */
 import type { AggregatedResult } from "../types/aggregate.js";
 import type { ClaimEvaluation } from "../types/claims.js";
+import type { EvaluationOutput } from "../types/evaluator.js";
 import type { ClaimStatusDisplay, Renderer, RenderOutput, TableRenderSpec } from "./types.js";
 /**
  * Options for LaTeX rendering.
@@ -36,6 +37,65 @@ export declare class LaTeXRenderer implements Renderer {
      * @param specs
      */
     renderAll(aggregates: AggregatedResult[], specs: TableRenderSpec[]): RenderOutput[];
+    /**
+     * Render evaluation output (generic for all evaluation types).
+     * Dispatches to type-specific rendering methods.
+     *
+     * @param evaluation - Evaluation output from any evaluator
+     * @returns Rendered output
+     */
+    renderEvaluation<T>(evaluation: EvaluationOutput<T>): RenderOutput;
+    /**
+     * Render claims evaluation output.
+     *
+     * @param evaluation - Claims evaluation output
+     * @returns Rendered output
+     */
+    private renderClaimsEvaluation;
+    /**
+     * Render robustness evaluation output.
+     *
+     * @param evaluation - Robustness evaluation output
+     * @returns Rendered output
+     */
+    private renderRobustnessEvaluation;
+    /**
+     * Render metrics evaluation output.
+     *
+     * @param evaluation - Metrics evaluation output
+     * @returns Rendered output
+     */
+    private renderMetricsEvaluation;
+    /**
+     * Render exploratory evaluation output.
+     *
+     * @param evaluation - Exploratory evaluation output
+     * @returns Rendered output
+     */
+    private renderExploratoryEvaluation;
+    /**
+     * Render SUT rankings table.
+     */
+    private renderSutRankings;
+    /**
+     * Render pairwise comparisons table.
+     */
+    private renderPairwiseComparisons;
+    /**
+     * Render case-class effects table.
+     */
+    private renderCaseClassEffects;
+    /**
+     * Render metric correlations table.
+     */
+    private renderMetricCorrelations;
+    /**
+     * Render custom evaluation output (generic fallback).
+     *
+     * @param evaluation - Custom evaluation output
+     * @returns Rendered output
+     */
+    private renderCustomEvaluation;
     /**
      * Render claim evaluation summary.
      * @param evaluations

package/dist/renderers/latex-renderer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"latex-renderer.d.ts","sourceRoot":"","sources":["../../src/renderers/latex-renderer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,eAAe,EAAe,MAAM,oBAAoB,CAAC;AACvE,OAAO,KAAK,EACX,kBAAkB,EAElB,QAAQ,EACR,YAAY,EACZ,eAAe,EACf,MAAM,YAAY,CAAC;AAGpB;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACpC,2BAA2B;IAC3B,WAAW,CAAC,EAAE,kBAAkB,CAAC;IAEjC,sCAAsC;IACtC,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB,qCAAqC;IACrC,eAAe,CAAC,EAAE,MAAM,CAAC;CACzB;AAWD;;GAEG;AACH,qBAAa,aAAc,YAAW,QAAQ;IAC7C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAuB;gBAEnC,OAAO,GAAE,OAAO,CAAC,oBAAoB,CAAM;IAIvD;;;;OAIG;IACH,WAAW,CAAC,UAAU,EAAE,gBAAgB,EAAE,EAAE,IAAI,EAAE,eAAe,GAAG,YAAY;IAgChF;;;;OAIG;IACH,SAAS,CAAC,UAAU,EAAE,gBAAgB,EAAE,EAAE,KAAK,EAAE,eAAe,EAAE,GAAG,YAAY,EAAE;IAInF;;;OAGG;IACH,kBAAkB,CAAC,WAAW,EAAE,eAAe,EAAE,GAAG,YAAY;IAmDhE;;;;;;OAMG;IACH,OAAO,CAAC,UAAU;IAwClB;;;OAGG;IACH,OAAO,CAAC,WAAW;IAanB;;;;OAIG;IACH,YAAY,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM;IAOlD;;;OAGG;IACH,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAOpC;;;OAGG;IACH,gBAAgB,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM;CAMnC;AAED;;;GAGG;AACH,eAAO,MAAM,WAAW,GAAI,MAAM,MAAM,KAAG,MAKO,CAAC;AAEnD;;;GAGG;AACH,eAAO,MAAM,mBAAmB,GAAI,UAAU,OAAO,CAAC,oBAAoB,CAAC,KAAG,aACnD,CAAC"}
+{"version":3,"file":"latex-renderer.d.ts","sourceRoot":"","sources":["../../src/renderers/latex-renderer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,KAAK,EAAE,eAAe,EAAe,MAAM,oBAAoB,CAAC;AACvE,OAAO,KAAK,EACX,gBAAgB,EAKhB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,KAAK,EACX,kBAAkB,EAElB,QAAQ,EACR,YAAY,EACZ,eAAe,EACf,MAAM,YAAY,CAAC;AAGpB;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACpC,2BAA2B;IAC3B,WAAW,CAAC,EAAE,kBAAkB,CAAC;IAEjC,sCAAsC;IACtC,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB,qCAAqC;IACrC,eAAe,CAAC,EAAE,MAAM,CAAC;CACzB;AAWD;;GAEG;AACH,qBAAa,aAAc,YAAW,QAAQ;IAC7C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAuB;gBAEnC,OAAO,GAAE,OAAO,CAAC,oBAAoB,CAAM;IAIvD;;;;OAIG;IACH,WAAW,CAAC,UAAU,EAAE,gBAAgB,EAAE,EAAE,IAAI,EAAE,eAAe,GAAG,YAAY;IAgChF;;;;OAIG;IACH,SAAS,CAAC,UAAU,EAAE,gBAAgB,EAAE,EAAE,KAAK,EAAE,eAAe,EAAE,GAAG,YAAY,EAAE;IAInF;;;;;;OAMG;IACH,gBAAgB,CAAC,CAAC,EAAE,UAAU,EAAE,gBAAgB,CAAC,CAAC,CAAC,GAAG,YAAY;IAoBlE;;;;;OAKG;IACH,OAAO,CAAC,sBAAsB;IAM9B;;;;;OAKG;IACH,OAAO,CAAC,0BAA0B;IA+ClC;;;;;OAKG;IACH,OAAO,CAAC,uBAAuB;IAmE/B;;;;;OAKG;IACH,OAAO,CAAC,2BAA2B;IAwCnC;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAyCzB;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAuCjC;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAuC9B;;OAEG;IACH,OAAO,CAAC,wBAAwB;IAqChC;;;;;OAKG;IACH,OAAO,CAAC,sBAAsB;IAe9B;;;OAGG;IACH,kBAAkB,CAAC,WAAW,EAAE,eAAe,EAAE,GAAG,YAAY;IAmDhE;;;;;;OAMG;IACH,OAAO,CAAC,UAAU;IAwClB;;;OAGG;IACH,OAAO,CAAC,WAAW;IAanB;;;;OAIG;IACH,YAAY,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM;IAOlD;;;OAGG;IACH,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAOpC;;;OAGG;IACH,gBAAgB,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM;CAMnC;AAED;;;GAGG;AACH,eAAO,MAAM,WAAW,GAAI,MAAM,MAAM,KAAG,MAKO,CAAC;AAEnD;;;GAGG;AACH,eAAO,MAAM,mBAAmB,GAAI,UAAU,OAAO,CAAC,oBAAoB,CAAC,KAAG,aACnD,CAAC"}