ppef 1.0.0 → 1.1.0

package/dist/collector/result-collector.d.ts

@@ -0,0 +1,159 @@
+/**
+ * Result Collector
+ *
+ * Collects, validates, and stores evaluation results.
+ * Replaces the simple MetricsCollector with a schema-aware implementation.
+ */
+import type { Primitive } from "../types/case.js";
+import type { EvaluationResult, ResultBatch } from "../types/result.js";
+import type { SutRole } from "../types/sut.js";
+/**
+ * Filter criteria for querying results.
+ */
+export interface ResultFilter {
+    /** Filter by SUT ID */
+    sut?: string;
+    /** Filter by SUT role */
+    sutRole?: SutRole;
+    /** Filter by case ID */
+    caseId?: string;
+    /** Filter by case class */
+    caseClass?: string;
+    /** Filter by validity */
+    valid?: boolean;
+    /** Filter by metric presence */
+    hasMetric?: string;
+    /** Custom predicate */
+    predicate?: (result: EvaluationResult) => boolean;
+}
+/**
+ * Aggregation options.
+ */
+export interface AggregationOptions {
+    /** Group by SUT */
+    groupBySut?: boolean;
+    /** Group by case class */
+    groupByCaseClass?: boolean;
+    /** Metrics to aggregate */
+    metrics?: string[];
+}
+/**
+ * Schema validation error.
+ */
+export interface ValidationError {
+    field: string;
+    message: string;
+}
+/**
+ * Result collector with schema validation and querying.
+ */
+export declare class ResultCollector {
+    private results;
+    private readonly schemaVersion;
+    /**
+     * Record a single result with validation.
+     *
+     * @param result - Result to record
+     * @throws Error if result fails validation
+     */
+    record(result: EvaluationResult): void;
+    /**
+     * Record multiple results.
+     *
+     * @param results - Results to record
+     */
+    recordBatch(results: EvaluationResult[]): void;
+    /**
+     * Validate a result against the schema.
+     *
+     * @param result - Result to validate
+     * @returns Array of validation errors (empty if valid)
+     */
+    validate(result: EvaluationResult): ValidationError[];
+    /**
+     * Query results with filters.
+     *
+     * @param filter - Filter criteria
+     * @returns Matching results
+     */
+    query(filter?: ResultFilter): EvaluationResult[];
+    /**
+     * Get all results for a specific SUT.
+     *
+     * @param sutId - SUT identifier
+     * @returns Results for that SUT
+     */
+    getBySut(sutId: string): EvaluationResult[];
+    /**
+     * Get all results for a specific case class.
+     *
+     * @param caseClass - Case class
+     * @returns Results for that case class
+     */
+    getByCaseClass(caseClass: string): EvaluationResult[];
+    /**
+     * Get unique SUT IDs in the collection.
+     */
+    getUniqueSuts(): string[];
+    /**
+     * Get unique case classes in the collection.
+     */
+    getUniqueCaseClasses(): string[];
+    /**
+     * Get unique metric names in the collection.
+     */
+    getUniqueMetrics(): string[];
+    /**
+     * Get all results.
+     */
+    getAll(): EvaluationResult[];
+    /**
+     * Get result count.
+     */
+    get count(): number;
+    /**
+     * Check if empty.
+     */
+    get isEmpty(): boolean;
+    /**
+     * Clear all results.
+     */
+    clear(): void;
+    /**
+     * Serialize to ResultBatch format.
+     *
+     * @param metadata - Optional batch metadata
+     * @returns Serializable batch
+     */
+    serialize(metadata?: Record<string, Primitive>): ResultBatch;
+    /**
+     * Load from a ResultBatch.
+     *
+     * @param batch - Batch to load
+     * @param append - Whether to append to existing results
+     */
+    load(batch: ResultBatch, append?: boolean): void;
+    /**
+     * Extract a specific metric across all results.
+     *
+     * @param metricName - Metric to extract
+     * @returns Array of { runId, value } pairs
+     */
+    extractMetric(metricName: string): {
+        runId: string;
+        value: number;
+    }[];
+    /**
+     * Get metric values for a specific SUT.
+     *
+     * @param sutId - SUT identifier
+     * @param metricName - Metric name
+     * @returns Array of metric values
+     */
+    getMetricValues(sutId: string, metricName: string): number[];
+}
+/**
+ * Global result collector instance.
+ */
+export declare const resultCollector: ResultCollector;
+//# sourceMappingURL=result-collector.d.ts.map

package/dist/collector/result-collector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"result-collector.d.ts","sourceRoot":"","sources":["../../src/collector/result-collector.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,KAAK,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACxE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAE/C;;GAEG;AACH,MAAM,WAAW,YAAY;IAC5B,uBAAuB;IACvB,GAAG,CAAC,EAAE,MAAM,CAAC;IAEb,yBAAyB;IACzB,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB,wBAAwB;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,2BAA2B;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,yBAAyB;IACzB,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB,gCAAgC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,uBAAuB;IACvB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,gBAAgB,KAAK,OAAO,CAAC;CAClD;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC,mBAAmB;IACnB,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB,0BAA0B;IAC1B,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAE3B,2BAA2B;IAC3B,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,qBAAa,eAAe;IAC3B,OAAO,CAAC,OAAO,CAA0B;IACzC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAW;IAEzC;;;;;OAKG;IACH,MAAM,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI;IAUtC;;;;OAIG;IACH,WAAW,CAAC,OAAO,EAAE,gBAAgB,EAAE,GAAG,IAAI;IAM9C;;;;;OAKG;IACH,QAAQ,CAAC,MAAM,EAAE,gBAAgB,GAAG,eAAe,EAAE;IAyBrD;;;;;OAKG;IACH,KAAK,CAAC,MAAM,GAAE,YAAiB,GAAG,gBAAgB,EAAE;IAapD;;;;;OAKG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,gBAAgB,EAAE;IAI3C;;;;;OAKG;IACH,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,gBAAgB,EAAE;IAIrD;;OAEG;IACH,aAAa,IAAI,MAAM,EAAE;IAIzB;;OAEG;IACH,oBAAoB,IAAI,MAAM,EAAE;IAOhC;;OAEG;IACH,gBAAgB,IAAI,MAAM,EAAE;IAU5B;;OAEG;IACH,MAAM,IAAI,gBAAgB,EAAE;IAI5B;;OAEG;IACH,IAAI,KAAK,IAAI,MAAM,CAElB;IAED;;OAEG;IACH,IAAI,OAAO,IAAI,OAAO,CAErB;IAED;;OAEG;IACH,KAAK,IAAI,IAAI;IAIb;;;;;OAKG;IACH,SAAS,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,GAAG,WAAW;IAS5D;;;;;OAKG;IACH,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,MAAM,UAAQ,GAAG,IAAI;IAO9C;;;;;OAKG;IACH,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE;IASrE;;;;;;OAMG;IACH,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE;CAK5D;AAED;;GAEG;AACH,eAAO,MAAM,eAAe,iBAAwB,CAAC"}

package/dist/collector/result-collector.js

@@ -0,0 +1,213 @@
+/**
+ * Result Collector
+ *
+ * Collects, validates, and stores evaluation results.
+ * Replaces the simple MetricsCollector with a schema-aware implementation.
+ */
+/**
+ * Result collector with schema validation and querying.
+ */
+export class ResultCollector {
+    results = [];
+    schemaVersion = "1.0.0";
+    /**
+     * Record a single result with validation.
+     *
+     * @param result - Result to record
+     * @throws Error if result fails validation
+     */
+    record(result) {
+        const errors = this.validate(result);
+        if (errors.length > 0) {
+            throw new Error(`Invalid result: ${errors.map((e) => `${e.field}: ${e.message}`).join(", ")}`);
+        }
+        this.results.push(result);
+    }
+    /**
+     * Record multiple results.
+     *
+     * @param results - Results to record
+     */
+    recordBatch(results) {
+        for (const result of results) {
+            this.record(result);
+        }
+    }
+    /**
+     * Validate a result against the schema.
+     *
+     * @param result - Result to validate
+     * @returns Array of validation errors (empty if valid)
+     */
+    validate(result) {
+        const errors = [];
+        // Validate required nested properties have non-empty values
+        if (result.run.runId === "") {
+            errors.push({ field: "run.runId", message: "Missing run ID" });
+        }
+        if (result.run.sut === "") {
+            errors.push({ field: "run.sut", message: "Missing SUT identifier" });
+        }
+        if (result.run.caseId === "") {
+            errors.push({ field: "run.caseId", message: "Missing case ID" });
+        }
+        if (Object.keys(result.metrics.numeric).length === 0) {
+            errors.push({ field: "metrics.numeric", message: "Missing numeric metrics" });
+        }
+        if (result.provenance.runtime.platform === "") {
+            errors.push({ field: "provenance.runtime.platform", message: "Missing platform" });
+        }
+        return errors;
+    }
+    /**
+     * Query results with filters.
+     *
+     * @param filter - Filter criteria
+     * @returns Matching results
+     */
+    query(filter = {}) {
+        return this.results.filter((result) => {
+            if (filter.sut && result.run.sut !== filter.sut)
+                return false;
+            if (filter.sutRole && result.run.sutRole !== filter.sutRole)
+                return false;
+            if (filter.caseId && result.run.caseId !== filter.caseId)
+                return false;
+            if (filter.caseClass && result.run.caseClass !== filter.caseClass)
+                return false;
+            if (filter.valid !== undefined && result.correctness.valid !== filter.valid)
+                return false;
+            if (filter.hasMetric && !(filter.hasMetric in result.metrics.numeric))
+                return false;
+            if (filter.predicate && !filter.predicate(result))
+                return false;
+            return true;
+        });
+    }
+    /**
+     * Get all results for a specific SUT.
+     *
+     * @param sutId - SUT identifier
+     * @returns Results for that SUT
+     */
+    getBySut(sutId) {
+        return this.query({ sut: sutId });
+    }
+    /**
+     * Get all results for a specific case class.
+     *
+     * @param caseClass - Case class
+     * @returns Results for that case class
+     */
+    getByCaseClass(caseClass) {
+        return this.query({ caseClass });
+    }
+    /**
+     * Get unique SUT IDs in the collection.
+     */
+    getUniqueSuts() {
+        return [...new Set(this.results.map((r) => r.run.sut))];
+    }
+    /**
+     * Get unique case classes in the collection.
+     */
+    getUniqueCaseClasses() {
+        const classes = this.results
+            .map((r) => r.run.caseClass)
+            .filter((c) => c !== undefined);
+        return [...new Set(classes)];
+    }
+    /**
+     * Get unique metric names in the collection.
+     */
+    getUniqueMetrics() {
+        const metrics = new Set();
+        for (const result of this.results) {
+            for (const metric of Object.keys(result.metrics.numeric)) {
+                metrics.add(metric);
+            }
+        }
+        return [...metrics];
+    }
+    /**
+     * Get all results.
+     */
+    getAll() {
+        return [...this.results];
+    }
+    /**
+     * Get result count.
+     */
+    get count() {
+        return this.results.length;
+    }
+    /**
+     * Check if empty.
+     */
+    get isEmpty() {
+        return this.results.length === 0;
+    }
+    /**
+     * Clear all results.
+     */
+    clear() {
+        this.results = [];
+    }
+    /**
+     * Serialize to ResultBatch format.
+     *
+     * @param metadata - Optional batch metadata
+     * @returns Serializable batch
+     */
+    serialize(metadata) {
+        return {
+            version: this.schemaVersion,
+            timestamp: new Date().toISOString(),
+            results: this.results,
+            metadata,
+        };
+    }
+    /**
+     * Load from a ResultBatch.
+     *
+     * @param batch - Batch to load
+     * @param append - Whether to append to existing results
+     */
+    load(batch, append = false) {
+        if (!append) {
+            this.results = [];
+        }
+        this.recordBatch(batch.results);
+    }
+    /**
+     * Extract a specific metric across all results.
+     *
+     * @param metricName - Metric to extract
+     * @returns Array of { runId, value } pairs
+     */
+    extractMetric(metricName) {
+        return this.results
+            .filter((r) => metricName in r.metrics.numeric)
+            .map((r) => ({
+            runId: r.run.runId,
+            value: r.metrics.numeric[metricName],
+        }));
+    }
+    /**
+     * Get metric values for a specific SUT.
+     *
+     * @param sutId - SUT identifier
+     * @param metricName - Metric name
+     * @returns Array of metric values
+     */
+    getMetricValues(sutId, metricName) {
+        return this.results
+            .filter((r) => r.run.sut === sutId && metricName in r.metrics.numeric)
+            .map((r) => r.metrics.numeric[metricName]);
+    }
+}
+/**
+ * Global result collector instance.
+ */
+export const resultCollector = new ResultCollector();
+//# sourceMappingURL=result-collector.js.map

package/dist/collector/result-collector.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"result-collector.js","sourceRoot":"","sources":["../../src/collector/result-collector.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAsDH;;GAEG;AACH,MAAM,OAAO,eAAe;IACnB,OAAO,GAAuB,EAAE,CAAC;IACxB,aAAa,GAAG,OAAO,CAAC;IAEzC;;;;;OAKG;IACH,MAAM,CAAC,MAAwB;QAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACrC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CACd,mBAAmB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC7E,CAAC;QACH,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC3B,CAAC;IAED;;;;OAIG;IACH,WAAW,CAAC,OAA2B;QACtC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC9B,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACrB,CAAC;IACF,CAAC;IAED;;;;;OAKG;IACH,QAAQ,CAAC,MAAwB;QAChC,MAAM,MAAM,GAAsB,EAAE,CAAC;QAErC,4DAA4D;QAC5D,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,KAAK,EAAE,EAAE,CAAC;YAC7B,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC,CAAC;QAChE,CAAC;QACD,IAAI,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,EAAE,EAAE,CAAC;YAC3B,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,wBAAwB,EAAE,CAAC,CAAC;QACtE,CAAC;QACD,IAAI,MAAM,CAAC,GAAG,CAAC,MAAM,KAAK,EAAE,EAAE,CAAC;YAC9B,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,OAAO,EAAE,iBAAiB,EAAE,CAAC,CAAC;QAClE,CAAC;QAED,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtD,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,iBAAiB,EAAE,OAAO,EAAE,yBAAyB,EAAE,CAAC,CAAC;QAC/E,CAAC;QAED,IAAI,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,KAAK,EAAE,EAAE,CAAC;YAC/C,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,6BAA6B,EAAE,OAAO,EAAE,kBAAkB,EAAE,CAAC,CAAC;QACpF,CAAC;QAED,OAAO,MAAM,CAAC;IACf,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,SAAuB,EAAE;QAC9B,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE;YACrC,IAAI,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,MAAM,CAAC,GAAG;gBAAE,OAAO,KAAK,CAAC;YAC9D,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,OAAO,KAAK,MAAM,CAAC,OAAO;gBAAE,OAAO,KAAK,CAAC;YAC1E,IAAI,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,GAAG,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM;gBAAE,OAAO,KAAK,CAAC;YACvE,IAAI,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,GAAG,CAAC,SAAS,KAAK,MAAM,CAAC,SAAS;gBAAE,OAAO,KAAK,CAAC;YAChF,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,IAAI,MAAM,CAAC,WAAW,CAAC,KAAK,KAAK,MAAM,CAAC,KAAK;gBAAE,OAAO,KAAK,CAAC;YAC1F,IAAI,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;gBAAE,OAAO,KAAK,CAAC;YACpF,IAAI,MAAM,CAAC,SAAS,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC;gBAAE,OAAO,KAAK,CAAC;YAChE,OAAO,IAAI,CAAC;QACb,CAAC,CAAC,CAAC;IACJ,CAAC;IAED;;;;;OAKG;IACH,QAAQ,CAAC,KAAa;QACrB,OAAO,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;IACnC,CAAC;IAED;;;;;OAKG;IACH,cAAc,CAAC,SAAiB;QAC/B,OAAO,IAAI,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,aAAa;QACZ,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACzD,CAAC;IAED;;OAEG;IACH,oBAAoB;QACnB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO;aAC1B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC;aAC3B,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC;QAC9C,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;IAC9B,CAAC;IAED;;OAEG;IACH,gBAAgB;QACf,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;QAClC,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACnC,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC1D,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACrB,CAAC;QACF,CAAC;QACD,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,MAAM;QACL,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC;IAC1B,CAAC;IAED;;OAEG;IACH,IAAI,KAAK;QACR,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,IAAI,OAAO;QACV,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,KAAK;QACJ,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;IACnB,CAAC;IAED;;;;;OAKG;IACH,SAAS,CAAC,QAAoC;QAC7C,OAAO;YACN,OAAO,EAAE,IAAI,CAAC,aAAa;YAC3B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,QAAQ;SACR,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,IAAI,CAAC,KAAkB,EAAE,MAAM,GAAG,KAAK;QACtC,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QACnB,CAAC;QACD,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACjC,CAAC;IAED;;;;;OAKG;IACH,aAAa,CAAC,UAAkB;QAC/B,OAAO,IAAI,CAAC,OAAO;aACjB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,IAAI,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;aAC9C,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACZ,KAAK,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK;YAClB,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC;SACpC,CAAC,CAAC,CAAC;IACN,CAAC;IAED;;;;;;OAMG;IACH,eAAe,CAAC,KAAa,EAAE,UAAkB;QAChD,OAAO,IAAI,CAAC,OAAO;aACjB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,KAAK,KAAK,IAAI,UAAU,IAAI,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;aACrE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;IAC7C,CAAC;CACD;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC"}

package/dist/collector/schema.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Schema Definitions and Validators
+ *
+ * Provides schema validation utilities for the evaluation framework.
+ */
+/**
+ * Schema validation result.
+ */
+export interface SchemaValidation {
+    valid: boolean;
+    errors: string[];
+}
+/**
+ * Validate an EvaluationResult against the schema.
+ * @param result
+ */
+export declare const validateResult: (result: unknown) => SchemaValidation;
+/**
+ * Validate an EvaluationCase against the schema.
+ * @param evaluationCase
+ */
+export declare const validateCase: (evaluationCase: unknown) => SchemaValidation;
+/**
+ * Validate a SutRegistration against the schema.
+ * @param registration
+ */
+export declare const validateSutRegistration: (registration: unknown) => SchemaValidation;
+/**
+ * Deep freeze an object to ensure immutability.
+ * @param obj
+ * @param object
+ */
+export declare const deepFreeze: <T extends object>(object: T) => Readonly<T>;
+//# sourceMappingURL=schema.d.ts.map

package/dist/collector/schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/collector/schema.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAMH;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,EAAE,MAAM,EAAE,CAAC;CACjB;AAED;;;GAGG;AACH,eAAO,MAAM,cAAc,GAAI,QAAQ,OAAO,KAAG,gBAoEhD,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,YAAY,GAAI,gBAAgB,OAAO,KAAG,gBAqBtD,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,uBAAuB,GAAI,cAAc,OAAO,KAAG,gBAqC/D,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,UAAU,GAAI,CAAC,SAAS,MAAM,EAAE,QAAQ,CAAC,KAAG,QAAQ,CAAC,CAAC,CAQlE,CAAC"}

package/dist/collector/schema.js

@@ -0,0 +1,145 @@
+/**
+ * Schema Definitions and Validators
+ *
+ * Provides schema validation utilities for the evaluation framework.
+ */
+/**
+ * Validate an EvaluationResult against the schema.
+ * @param result
+ */
+export const validateResult = (result) => {
+    const errors = [];
+    if (!result || typeof result !== "object") {
+        return { valid: false, errors: ["Result must be an object"] };
+    }
+    const r = result;
+    // Validate run context
+    if (!r.run || typeof r.run !== "object") {
+        errors.push("Missing or invalid run context");
+    }
+    else {
+        if (typeof r.run.runId !== "string" || r.run.runId.length === 0) {
+            errors.push("run.runId must be a non-empty string");
+        }
+        if (typeof r.run.sut !== "string" || r.run.sut.length === 0) {
+            errors.push("run.sut must be a non-empty string");
+        }
+        if (!["primary", "baseline", "oracle"].includes(r.run.sutRole)) {
+            errors.push("run.sutRole must be 'primary', 'baseline', or 'oracle'");
+        }
+        if (typeof r.run.caseId !== "string" || r.run.caseId.length === 0) {
+            errors.push("run.caseId must be a non-empty string");
+        }
+    }
+    // Validate correctness
+    if (!r.correctness || typeof r.correctness !== "object") {
+        errors.push("Missing or invalid correctness assessment");
+    }
+    else {
+        if (typeof r.correctness.expectedExists !== "boolean") {
+            errors.push("correctness.expectedExists must be a boolean");
+        }
+        if (typeof r.correctness.producedOutput !== "boolean") {
+            errors.push("correctness.producedOutput must be a boolean");
+        }
+        if (typeof r.correctness.valid !== "boolean") {
+            errors.push("correctness.valid must be a boolean");
+        }
+    }
+    // Validate metrics
+    if (!r.metrics || typeof r.metrics !== "object") {
+        errors.push("Missing or invalid metrics");
+    }
+    else {
+        if (typeof r.metrics.numeric !== "object") {
+            errors.push("metrics.numeric must be an object");
+        }
+        else {
+            for (const [key, value] of Object.entries(r.metrics.numeric)) {
+                if (typeof value !== "number" || !Number.isFinite(value)) {
+                    errors.push(`metrics.numeric.${key} must be a finite number`);
+                }
+            }
+        }
+    }
+    // Validate provenance
+    if (!r.provenance || typeof r.provenance !== "object") {
+        errors.push("Missing or invalid provenance");
+    }
+    else if (typeof r.provenance.runtime !== "object") {
+        errors.push("provenance.runtime must be an object");
+    }
+    return {
+        valid: errors.length === 0,
+        errors,
+    };
+};
+/**
+ * Validate an EvaluationCase against the schema.
+ * @param evaluationCase
+ */
+export const validateCase = (evaluationCase) => {
+    const errors = [];
+    if (!evaluationCase || typeof evaluationCase !== "object") {
+        return { valid: false, errors: ["Case must be an object"] };
+    }
+    const c = evaluationCase;
+    if (typeof c.caseId !== "string" || c.caseId.length === 0) {
+        errors.push("caseId must be a non-empty string");
+    }
+    if (!c.inputs || typeof c.inputs !== "object") {
+        errors.push("inputs must be an object");
+    }
+    return {
+        valid: errors.length === 0,
+        errors,
+    };
+};
+/**
+ * Validate a SutRegistration against the schema.
+ * @param registration
+ */
+export const validateSutRegistration = (registration) => {
+    const errors = [];
+    if (!registration || typeof registration !== "object") {
+        return { valid: false, errors: ["Registration must be an object"] };
+    }
+    const r = registration;
+    if (typeof r.id !== "string" || r.id.length === 0) {
+        errors.push("id must be a non-empty string");
+    }
+    if (typeof r.name !== "string" || r.name.length === 0) {
+        errors.push("name must be a non-empty string");
+    }
+    if (typeof r.version !== "string" || r.version.length === 0) {
+        errors.push("version must be a non-empty string");
+    }
+    if (!["primary", "baseline", "oracle"].includes(r.role ?? "")) {
+        errors.push("role must be 'primary', 'baseline', or 'oracle'");
+    }
+    if (!r.config || typeof r.config !== "object") {
+        errors.push("config must be an object");
+    }
+    if (!Array.isArray(r.tags)) {
+        errors.push("tags must be an array");
+    }
+    return {
+        valid: errors.length === 0,
+        errors,
+    };
+};
+/**
+ * Deep freeze an object to ensure immutability.
+ * @param obj
+ * @param object
+ */
+export const deepFreeze = (object) => {
+    Object.freeze(object);
+    for (const value of Object.values(object)) {
+        if (value && typeof value === "object" && !Object.isFrozen(value)) {
+            deepFreeze(value);
+        }
+    }
+    return object;
+};
+//# sourceMappingURL=schema.js.map

package/dist/collector/schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/collector/schema.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAcH;;;GAGG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,MAAe,EAAoB,EAAE;IACnE,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC3C,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,0BAA0B,CAAC,EAAE,CAAC;IAC/D,CAAC;IAED,MAAM,CAAC,GAAG,MAAmC,CAAC;IAE9C,uBAAuB;IACvB,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,OAAO,CAAC,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC;QACzC,MAAM,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;IAC/C,CAAC;SAAM,CAAC;QACP,IAAI,OAAO,CAAC,CAAC,GAAG,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjE,MAAM,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;QACrD,CAAC;QACD,IAAI,OAAO,CAAC,CAAC,GAAG,CAAC,GAAG,KAAK,QAAQ,IAAI,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7D,MAAM,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;QACnD,CAAC;QACD,IAAI,CAAC,CAAC,SAAS,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAChE,MAAM,CAAC,IAAI,CAAC,wDAAwD,CAAC,CAAC;QACvE,CAAC;QACD,IAAI,OAAO,CAAC,CAAC,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnE,MAAM,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;QACtD,CAAC;IACF,CAAC;IAED,uBAAuB;IACvB,IAAI,CAAC,CAAC,CAAC,WAAW,IAAI,OAAO,CAAC,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;QACzD,MAAM,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAC;IAC1D,CAAC;SAAM,CAAC;QACP,IAAI,OAAO,CAAC,CAAC,WAAW,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;YACvD,MAAM,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;QAC7D,CAAC;QACD,IAAI,OAAO,CAAC,CAAC,WAAW,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;YACvD,MAAM,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;QAC7D,CAAC;QACD,IAAI,OAAO,CAAC,CAAC,WAAW,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC9C,MAAM,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC;QACpD,CAAC;IACF,CAAC;IAED,mBAAmB;IACnB,IAAI,CAAC,CAAC,CAAC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QACjD,MAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;IAC3C,CAAC;SAAM,CAAC;QACP,IAAI,OAAO,CAAC,CAAC,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YAC3C,MAAM,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;QAClD,CAAC;aAAM,CAAC;YACP,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC9D,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC1D,MAAM,CAAC,IAAI,CAAC,mBAAmB,GAAG,0BAA0B,CAAC,CAAC;gBAC/D,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;IAED,sBAAsB;IACtB,IAAI,CAAC,CAAC,CAAC,UAAU,IAAI,OAAO,CAAC,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;QACvD,MAAM,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;IAC9C,CAAC;SAAM,IAAI,OAAO,CAAC,CAAC,UAAU,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QACrD,MAAM,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;IACrD,CAAC;IAED,OAAO;QACN,KAAK,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC;QAC1B,MAAM;KACN,CAAC;AACH,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,cAAuB,EAAoB,EAAE;IACzE,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,IAAI,CAAC,cAAc,IAAI,OAAO,cAAc,KAAK,QAAQ,EAAE,CAAC;QAC3D,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,wBAAwB,CAAC,EAAE,CAAC;IAC7D,CAAC;IAED,MAAM,CAAC,GAAG,cAAyC,CAAC;IAEpD,IAAI,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3D,MAAM,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;IAClD,CAAC;IAED,IAAI,CAAC,CAAC,CAAC,MAAM,IAAI,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/C,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IACzC,CAAC;IAED,OAAO;QACN,KAAK,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC;QAC1B,MAAM;KACN,CAAC;AACH,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,YAAqB,EAAoB,EAAE;IAClF,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,IAAI,CAAC,YAAY,IAAI,OAAO,YAAY,KAAK,QAAQ,EAAE,CAAC;QACvD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,gCAAgC,CAAC,EAAE,CAAC;IACrE,CAAC;IAED,MAAM,CAAC,GAAG,YAAwC,CAAC;IAEnD,IAAI,OAAO,CAAC,CAAC,EAAE,KAAK,QAAQ,IAAI,CAAC,CAAC,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnD,MAAM,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;IAC9C,CAAC;IAED,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvD,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;IAChD,CAAC;IAED,IAAI,OAAO,CAAC,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7D,MAAM,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;IACnD,CAAC;IAED,IAAI,CAAC,CAAC,SAAS,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,EAAE,CAAC;QAC/D,MAAM,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;IAChE,CAAC;IAED,IAAI,CAAC,CAAC,CAAC,MAAM,IAAI,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/C,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IACzC,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;QAC5B,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IACtC,CAAC;IAED,OAAO;QACN,KAAK,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC;QAC1B,MAAM;KACN,CAAC;AACH,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAmB,MAAS,EAAe,EAAE;IACtE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACtB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3C,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACnE,UAAU,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;IACF,CAAC;IACD,OAAO,MAAM,CAAC;AACf,CAAC,CAAC"}

package/dist/executor/__tests__/checkpoint-hash-bug.diagnostic.test.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Diagnostic Tests for Checkpoint Config Hash Bug
+ *
+ * Root cause: Config hash calculation includes properties that vary between
+ * main process and workers (e.g., 'concurrency'), causing checkpoint invalidation.
+ *
+ * This test diagnoses and will help fix the hash matching issue.
+ */
+export {};
+//# sourceMappingURL=checkpoint-hash-bug.diagnostic.test.d.ts.map

package/dist/executor/__tests__/checkpoint-hash-bug.diagnostic.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"checkpoint-hash-bug.diagnostic.test.d.ts","sourceRoot":"","sources":["../../../src/executor/__tests__/checkpoint-hash-bug.diagnostic.test.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG"}