pcl-mcp 0.2.4 → 0.2.5

package/dist/benchmarks/evaluators/context-retrieval-quality.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Context Retrieval Quality — measures if PCL retrieves the RIGHT documents.
+ * Disentangles "did PCL find the right docs" from "did the LLM use them well."
+ */
+export interface ContextMetrics {
+    recall: number;
+    precision: number;
+    f1: number;
+    retrieved: string[];
+    required: string[];
+    hits: string[];
+    misses: string[];
+    noise: string[];
+}
+/**
+ * Compute context retrieval quality metrics.
+ *
+ * @param retrievedIds - Document IDs returned by PCL search
+ * @param requiredIds - Document IDs that the task actually needs
+ */
+export declare function measureContextRetrieval(retrievedIds: string[], requiredIds: string[]): ContextMetrics;
+/**
+ * Aggregate context metrics across multiple tasks.
+ */
+export declare function averageContextMetrics(metrics: ContextMetrics[]): {
+    recall: number;
+    precision: number;
+    f1: number;
+};
+//# sourceMappingURL=context-retrieval-quality.d.ts.map

package/dist/benchmarks/evaluators/context-retrieval-quality.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"context-retrieval-quality.d.ts","sourceRoot":"","sources":["../../../benchmarks/evaluators/context-retrieval-quality.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB;AAED;;;;;GAKG;AACH,wBAAgB,uBAAuB,CACrC,YAAY,EAAE,MAAM,EAAE,EACtB,WAAW,EAAE,MAAM,EAAE,GACpB,cAAc,CA0BhB;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,cAAc,EAAE,GACxB;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAA;CAAE,CAenD"}

package/dist/benchmarks/evaluators/context-retrieval-quality.js

@@ -0,0 +1,50 @@
+/**
+ * Context Retrieval Quality — measures if PCL retrieves the RIGHT documents.
+ * Disentangles "did PCL find the right docs" from "did the LLM use them well."
+ */
+/**
+ * Compute context retrieval quality metrics.
+ *
+ * @param retrievedIds - Document IDs returned by PCL search
+ * @param requiredIds - Document IDs that the task actually needs
+ */
+export function measureContextRetrieval(retrievedIds, requiredIds) {
+    const retrievedSet = new Set(retrievedIds);
+    const requiredSet = new Set(requiredIds);
+    const hits = requiredIds.filter((id) => retrievedSet.has(id));
+    const misses = requiredIds.filter((id) => !retrievedSet.has(id));
+    const noise = retrievedIds.filter((id) => !requiredSet.has(id));
+    const recall = requiredIds.length > 0 ? hits.length / requiredIds.length : 1;
+    const precision = retrievedIds.length > 0 ? hits.length / retrievedIds.length : 1;
+    const f1 = recall + precision > 0
+        ? (2 * recall * precision) / (recall + precision)
+        : 0;
+    return {
+        recall,
+        precision,
+        f1,
+        retrieved: retrievedIds,
+        required: requiredIds,
+        hits,
+        misses,
+        noise,
+    };
+}
+/**
+ * Aggregate context metrics across multiple tasks.
+ */
+export function averageContextMetrics(metrics) {
+    if (metrics.length === 0)
+        return { recall: 0, precision: 0, f1: 0 };
+    const sum = metrics.reduce((acc, m) => ({
+        recall: acc.recall + m.recall,
+        precision: acc.precision + m.precision,
+        f1: acc.f1 + m.f1,
+    }), { recall: 0, precision: 0, f1: 0 });
+    return {
+        recall: sum.recall / metrics.length,
+        precision: sum.precision / metrics.length,
+        f1: sum.f1 / metrics.length,
+    };
+}
+//# sourceMappingURL=context-retrieval-quality.js.map

package/dist/benchmarks/evaluators/context-retrieval-quality.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"context-retrieval-quality.js","sourceRoot":"","sources":["../../../benchmarks/evaluators/context-retrieval-quality.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAaH;;;;;GAKG;AACH,MAAM,UAAU,uBAAuB,CACrC,YAAsB,EACtB,WAAqB;IAErB,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC;IAC3C,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC;IAEzC,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9D,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACjE,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IAEhE,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7E,MAAM,SAAS,GACb,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAClE,MAAM,EAAE,GACN,MAAM,GAAG,SAAS,GAAG,CAAC;QACpB,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC;QACjD,CAAC,CAAC,CAAC,CAAC;IAER,OAAO;QACL,MAAM;QACN,SAAS;QACT,EAAE;QACF,SAAS,EAAE,YAAY;QACvB,QAAQ,EAAE,WAAW;QACrB,IAAI;QACJ,MAAM;QACN,KAAK;KACN,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CACnC,OAAyB;IAEzB,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC;IACpE,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,CACxB,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QACX,MAAM,EAAE,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM;QAC7B,SAAS,EAAE,GAAG,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS;QACtC,EAAE,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE;KAClB,CAAC,EACF,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CACnC,CAAC;IACF,OAAO;QACL,MAAM,EAAE,GAAG,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM;QACnC,SAAS,EAAE,GAAG,CAAC,SAAS,GAAG,OAAO,CAAC,MAAM;QACzC,EAAE,EAAE,GAAG,CAAC,EAAE,GAAG,OAAO,CAAC,MAAM;KAC5B,CAAC;AACJ,CAAC"}

package/dist/benchmarks/evaluators/ir-metrics.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Precision@K: fraction of top-K results that are relevant.
+ */
+export declare function precisionAtK(retrieved: string[], relevant: Set<string>, k: number): number;
+/**
+ * Recall@K: fraction of relevant docs found in top-K.
+ */
+export declare function recallAtK(retrieved: string[], relevant: Set<string>, k: number): number;
+/**
+ * MRR (Mean Reciprocal Rank): 1 / rank of first relevant result.
+ */
+export declare function reciprocalRank(retrieved: string[], relevant: Set<string>): number;
+/**
+ * NDCG@K: normalized DCG using ideal ranking.
+ */
+export declare function ndcgAtK(retrieved: string[], relevanceScores: Map<string, number>, k: number): number;
+/**
+ * Compute all metrics for a single query.
+ */
+export declare function computeMetrics(retrieved: string[], relevanceScores: Map<string, number>, relevantThreshold?: number): {
+    precisionAt1: number;
+    precisionAt3: number;
+    precisionAt5: number;
+    recallAt5: number;
+    mrr: number;
+    ndcgAt5: number;
+};
+/**
+ * Average metrics across multiple queries.
+ */
+export declare function averageMetrics(results: Array<ReturnType<typeof computeMetrics>>): ReturnType<typeof computeMetrics>;
+//# sourceMappingURL=ir-metrics.d.ts.map

package/dist/benchmarks/evaluators/ir-metrics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ir-metrics.d.ts","sourceRoot":"","sources":["../../../benchmarks/evaluators/ir-metrics.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,wBAAgB,YAAY,CAC1B,SAAS,EAAE,MAAM,EAAE,EACnB,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,EACrB,CAAC,EAAE,MAAM,GACR,MAAM,CAKR;AAED;;GAEG;AACH,wBAAgB,SAAS,CACvB,SAAS,EAAE,MAAM,EAAE,EACnB,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,EACrB,CAAC,EAAE,MAAM,GACR,MAAM,CAKR;AAED;;GAEG;AACH,wBAAgB,cAAc,CAC5B,SAAS,EAAE,MAAM,EAAE,EACnB,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,GACpB,MAAM,CAKR;AAmBD;;GAEG;AACH,wBAAgB,OAAO,CACrB,SAAS,EAAE,MAAM,EAAE,EACnB,eAAe,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EACpC,CAAC,EAAE,MAAM,GACR,MAAM,CASR;AAED;;GAEG;AACH,wBAAgB,cAAc,CAC5B,SAAS,EAAE,MAAM,EAAE,EACnB,eAAe,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EACpC,iBAAiB,SAAI,GACpB;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;CACjB,CAeA;AAED;;GAEG;AACH,wBAAgB,cAAc,CAC5B,OAAO,EAAE,KAAK,CAAC,UAAU,CAAC,OAAO,cAAc,CAAC,CAAC,GAChD,UAAU,CAAC,OAAO,cAAc,CAAC,CA0BnC"}

package/dist/benchmarks/evaluators/ir-metrics.js

@@ -0,0 +1,98 @@
+/**
+ * Precision@K: fraction of top-K results that are relevant.
+ */
+export function precisionAtK(retrieved, relevant, k) {
+    const topK = retrieved.slice(0, k);
+    if (topK.length === 0)
+        return 0;
+    const hits = topK.filter(id => relevant.has(id)).length;
+    return hits / topK.length;
+}
+/**
+ * Recall@K: fraction of relevant docs found in top-K.
+ */
+export function recallAtK(retrieved, relevant, k) {
+    if (relevant.size === 0)
+        return 1; // no relevant docs = perfect recall vacuously
+    const topK = retrieved.slice(0, k);
+    const hits = topK.filter(id => relevant.has(id)).length;
+    return hits / relevant.size;
+}
+/**
+ * MRR (Mean Reciprocal Rank): 1 / rank of first relevant result.
+ */
+export function reciprocalRank(retrieved, relevant) {
+    for (let i = 0; i < retrieved.length; i++) {
+        if (relevant.has(retrieved[i]))
+            return 1 / (i + 1);
+    }
+    return 0;
+}
+/**
+ * DCG@K with graded relevance (relevance scores 0-3).
+ */
+function dcgAtK(retrieved, relevanceScores, k) {
+    let dcg = 0;
+    const topK = retrieved.slice(0, k);
+    for (let i = 0; i < topK.length; i++) {
+        const rel = relevanceScores.get(topK[i]) ?? 0;
+        dcg += (Math.pow(2, rel) - 1) / Math.log2(i + 2); // i+2 because log2(1)=0
+    }
+    return dcg;
+}
+/**
+ * NDCG@K: normalized DCG using ideal ranking.
+ */
+export function ndcgAtK(retrieved, relevanceScores, k) {
+    const dcg = dcgAtK(retrieved, relevanceScores, k);
+    // Ideal ranking: sort all docs by relevance descending
+    const idealOrder = [...relevanceScores.entries()]
+        .sort(([, a], [, b]) => b - a)
+        .map(([id]) => id);
+    const idcg = dcgAtK(idealOrder, relevanceScores, k);
+    if (idcg === 0)
+        return 0;
+    return dcg / idcg;
+}
+/**
+ * Compute all metrics for a single query.
+ */
+export function computeMetrics(retrieved, relevanceScores, relevantThreshold = 1) {
+    const relevant = new Set([...relevanceScores.entries()]
+        .filter(([, score]) => score >= relevantThreshold)
+        .map(([id]) => id));
+    return {
+        precisionAt1: precisionAtK(retrieved, relevant, 1),
+        precisionAt3: precisionAtK(retrieved, relevant, 3),
+        precisionAt5: precisionAtK(retrieved, relevant, 5),
+        recallAt5: recallAtK(retrieved, relevant, 5),
+        mrr: reciprocalRank(retrieved, relevant),
+        ndcgAt5: ndcgAtK(retrieved, relevanceScores, 5),
+    };
+}
+/**
+ * Average metrics across multiple queries.
+ */
+export function averageMetrics(results) {
+    const n = results.length;
+    if (n === 0) {
+        return { precisionAt1: 0, precisionAt3: 0, precisionAt5: 0, recallAt5: 0, mrr: 0, ndcgAt5: 0 };
+    }
+    const sum = results.reduce((acc, r) => ({
+        precisionAt1: acc.precisionAt1 + r.precisionAt1,
+        precisionAt3: acc.precisionAt3 + r.precisionAt3,
+        precisionAt5: acc.precisionAt5 + r.precisionAt5,
+        recallAt5: acc.recallAt5 + r.recallAt5,
+        mrr: acc.mrr + r.mrr,
+        ndcgAt5: acc.ndcgAt5 + r.ndcgAt5,
+    }), { precisionAt1: 0, precisionAt3: 0, precisionAt5: 0, recallAt5: 0, mrr: 0, ndcgAt5: 0 });
+    return {
+        precisionAt1: sum.precisionAt1 / n,
+        precisionAt3: sum.precisionAt3 / n,
+        precisionAt5: sum.precisionAt5 / n,
+        recallAt5: sum.recallAt5 / n,
+        mrr: sum.mrr / n,
+        ndcgAt5: sum.ndcgAt5 / n,
+    };
+}
+//# sourceMappingURL=ir-metrics.js.map

package/dist/benchmarks/evaluators/ir-metrics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ir-metrics.js","sourceRoot":"","sources":["../../../benchmarks/evaluators/ir-metrics.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,UAAU,YAAY,CAC1B,SAAmB,EACnB,QAAqB,EACrB,CAAS;IAET,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAChC,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;IACxD,OAAO,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC;AAC5B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,SAAS,CACvB,SAAmB,EACnB,QAAqB,EACrB,CAAS;IAET,IAAI,QAAQ,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC,CAAC,8CAA8C;IACjF,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;IACxD,OAAO,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;AAC9B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAC5B,SAAmB,EACnB,QAAqB;IAErB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,IAAI,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC;YAAE,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED;;GAEG;AACH,SAAS,MAAM,CACb,SAAmB,EACnB,eAAoC,EACpC,CAAS;IAET,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,IAAI,CAAC,CAAC;QAC/C,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,wBAAwB;IAC5E,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,OAAO,CACrB,SAAmB,EACnB,eAAoC,EACpC,CAAS;IAET,MAAM,GAAG,GAAG,MAAM,CAAC,SAAS,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC;IAClD,uDAAuD;IACvD,MAAM,UAAU,GAAG,CAAC,GAAG,eAAe,CAAC,OAAO,EAAE,CAAC;SAC9C,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;SAC7B,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;IACrB,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC;IACpD,IAAI,IAAI,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACzB,OAAO,GAAG,GAAG,IAAI,CAAC;AACpB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAC5B,SAAmB,EACnB,eAAoC,EACpC,iBAAiB,GAAG,CAAC;IASrB,MAAM,QAAQ,GAAG,IAAI,GAAG,CACtB,CAAC,GAAG,eAAe,CAAC,OAAO,EAAE,CAAC;SAC3B,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,IAAI,iBAAiB,CAAC;SACjD,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CACrB,CAAC;IAEF,OAAO;QACL,YAAY,EAAE,YAAY,CAAC,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;QAClD,YAAY,EAAE,YAAY,CAAC,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;QAClD,YAAY,EAAE,YAAY,CAAC,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;QAClD,SAAS,EAAE,SAAS,CAAC,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC5C,GAAG,EAAE,cAAc,CAAC,SAAS,EAAE,QAAQ,CAAC;QACxC,OAAO,EAAE,OAAO,CAAC,SAAS,EAAE,eAAe,EAAE,CAAC,CAAC;KAChD,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAC5B,OAAiD;IAEjD,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IACzB,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACZ,OAAO,EAAE,YAAY,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IACjG,CAAC;IAED,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,CACxB,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QACX,YAAY,EAAE,GAAG,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY;QAC/C,YAAY,EAAE,GAAG,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY;QAC/C,YAAY,EAAE,GAAG,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY;QAC/C,SAAS,EAAE,GAAG,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS;QACtC,GAAG,EAAE,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG;QACpB,OAAO,EAAE,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO;KACjC,CAAC,EACF,EAAE,YAAY,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CACxF,CAAC;IAEF,OAAO;QACL,YAAY,EAAE,GAAG,CAAC,YAAY,GAAG,CAAC;QAClC,YAAY,EAAE,GAAG,CAAC,YAAY,GAAG,CAAC;QAClC,YAAY,EAAE,GAAG,CAAC,YAAY,GAAG,CAAC;QAClC,SAAS,EAAE,GAAG,CAAC,SAAS,GAAG,CAAC;QAC5B,GAAG,EAAE,GAAG,CAAC,GAAG,GAAG,CAAC;QAChB,OAAO,EAAE,GAAG,CAAC,OAAO,GAAG,CAAC;KACzB,CAAC;AACJ,CAAC"}

package/dist/benchmarks/evaluators/structured-judge.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Structured Judge — evaluates AI output with specific yes/no questions per criterion.
+ *
+ * Instead of "rate 0-10", asks: "Does the code implement X? YES/NO"
+ * Then scores = (yes_count / total_questions) * 10.
+ */
+import Anthropic from "@anthropic-ai/sdk";
+import type { CodingTask } from "../lib/types.js";
+export interface JudgmentResult {
+    taskId: string;
+    totalQuestions: number;
+    yesCount: number;
+    score: number;
+    details: Array<{
+        criterion: string;
+        answer: "YES" | "NO" | "PARTIAL";
+        reasoning: string;
+    }>;
+}
+/**
+ * Evaluate an AI-generated output against structured criteria.
+ */
+export declare function structuredJudge(client: Anthropic, task: CodingTask, output: string, contextDocs: string): Promise<JudgmentResult>;
+/**
+ * Check if generated code contains valid TypeScript/JSX syntax.
+ * Uses the TypeScript compiler API in syntax-only mode for accurate parsing
+ * of template literals, JSX, and other complex syntax.
+ */
+export declare function checkTypeScriptSyntax(output: string): {
+    valid: boolean;
+    errorCount: number;
+    codeBlockCount: number;
+};
+//# sourceMappingURL=structured-judge.d.ts.map

package/dist/benchmarks/evaluators/structured-judge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"structured-judge.d.ts","sourceRoot":"","sources":["../../../benchmarks/evaluators/structured-judge.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,SAAS,MAAM,mBAAmB,CAAC;AAC1C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAIlD,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,KAAK,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,KAAK,GAAG,IAAI,GAAG,SAAS,CAAC;QACjC,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC,CAAC;CACJ;AAiCD;;GAEG;AACH,wBAAsB,eAAe,CACnC,MAAM,EAAE,SAAS,EACjB,IAAI,EAAE,UAAU,EAChB,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,cAAc,CAAC,CAsGzB;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,MAAM,GACb;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,cAAc,EAAE,MAAM,CAAA;CAAE,CAgDhE"}

package/dist/benchmarks/evaluators/structured-judge.js

@@ -0,0 +1,153 @@
+const JUDGE_MODEL = "claude-sonnet-4-20250514";
+/**
+ * Build structured yes/no questions from task criteria and context.
+ */
+function buildQuestions(task, contextDocs) {
+    const questions = [];
+    for (const criterion of task.evaluationCriteria) {
+        if (criterion.prompt) {
+            questions.push(criterion.prompt);
+        }
+        else if (criterion.pattern) {
+            questions.push(`Does the code contain or implement: ${criterion.criterion}?`);
+        }
+        else {
+            questions.push(`Does the code satisfy: ${criterion.criterion}?`);
+        }
+    }
+    // Add forbidden pattern checks
+    for (const pattern of task.forbiddenPatterns) {
+        questions.push(`Does the code AVOID the forbidden pattern: ${pattern}? (YES means it correctly avoids it)`);
+    }
+    return questions;
+}
+/**
+ * Evaluate an AI-generated output against structured criteria.
+ */
+export async function structuredJudge(client, task, output, contextDocs) {
+    const questions = buildQuestions(task, contextDocs);
+    const questionsBlock = questions
+        .map((q, i) => `Q${i + 1}: ${q}`)
+        .join("\n");
+    const prompt = `You are evaluating an AI coding agent's output against specific product requirements.
+
+TASK DESCRIPTION:
+${task.description}
+
+RELEVANT PRODUCT CONTEXT:
+${contextDocs.slice(0, 8000)}
+
+AI AGENT'S CODE OUTPUT:
+${output.slice(0, 12000)}
+
+EVALUATION QUESTIONS:
+${questionsBlock}
+
+For each question, answer with:
+- "YES" if the code clearly satisfies the criterion
+- "NO" if the code clearly fails the criterion
+- "PARTIAL" if partially satisfied
+
+Return a JSON array with one object per question:
+[{"answer": "YES"|"NO"|"PARTIAL", "reasoning": "brief explanation"}]
+
+Return ONLY the JSON array, no other text.`;
+    const response = await client.messages.create({
+        model: JUDGE_MODEL,
+        max_tokens: 2048,
+        temperature: 0,
+        system: "You are a precise code evaluation judge. Return only valid JSON arrays.",
+        messages: [{ role: "user", content: prompt }],
+    });
+    const text = response.content[0]?.type === "text" ? response.content[0].text : "[]";
+    let answers;
+    try {
+        answers = JSON.parse(text);
+    }
+    catch {
+        // Fallback: try to extract JSON from text
+        const match = text.match(/\[[\s\S]*\]/);
+        if (match) {
+            try {
+                answers = JSON.parse(match[0]);
+            }
+            catch {
+                answers = [];
+            }
+        }
+        else {
+            answers = [];
+        }
+    }
+    const details = questions.map((q, i) => {
+        const a = answers[i];
+        const answer = (a?.answer?.toUpperCase() ?? "NO");
+        return {
+            criterion: q,
+            answer: answer === "YES" || answer === "NO" || answer === "PARTIAL"
+                ? answer
+                : "NO",
+            reasoning: a?.reasoning ?? "No response from judge",
+        };
+    });
+    const yesCount = details.reduce((sum, d) => {
+        if (d.answer === "YES")
+            return sum + 1;
+        if (d.answer === "PARTIAL")
+            return sum + 0.5;
+        return sum;
+    }, 0);
+    const totalQuestions = details.length;
+    const score = totalQuestions > 0 ? (yesCount / totalQuestions) * 10 : 0;
+    return {
+        taskId: task.id,
+        totalQuestions,
+        yesCount,
+        score: Math.round(score * 10) / 10,
+        details,
+    };
+}
+/**
+ * Check if generated code contains valid TypeScript/JSX syntax.
+ * Uses the TypeScript compiler API in syntax-only mode for accurate parsing
+ * of template literals, JSX, and other complex syntax.
+ */
+export function checkTypeScriptSyntax(output) {
+    // Extract fenced code blocks
+    const codeBlockRegex = /```(?:typescript|tsx?|jsx?|js)?\s*\n([\s\S]*?)```/g;
+    const blocks = [];
+    let match;
+    while ((match = codeBlockRegex.exec(output)) !== null) {
+        if (match[1]?.trim())
+            blocks.push(match[1]);
+    }
+    if (blocks.length === 0) {
+        return { valid: true, errorCount: 0, codeBlockCount: 0 };
+    }
+    let totalErrors = 0;
+    // Dynamic import of typescript — may not be available in all environments
+    let ts;
+    try {
+        // eslint-disable-next-line @typescript-eslint/no-require-imports
+        ts = require("typescript");
+    }
+    catch {
+        // TypeScript not available — fall back to permissive (assume valid)
+        return { valid: true, errorCount: 0, codeBlockCount: blocks.length };
+    }
+    for (const block of blocks) {
+        // Parse as TSX to handle both TypeScript and JSX syntax
+        const sourceFile = ts.createSourceFile("check.tsx", block, ts.ScriptTarget.Latest, false, ts.ScriptKind.TSX);
+        // Count syntax-level diagnostics only (not semantic/type errors)
+        // parseDiagnostics is internal; access via type assertion
+        const syntaxDiags = (sourceFile
+            .parseDiagnostics?.length) ?? 0;
+        totalErrors += syntaxDiags;
+    }
+    return {
+        valid: totalErrors === 0,
+        errorCount: totalErrors,
+        codeBlockCount: blocks.length,
+    };
+}
+//# sourceMappingURL=structured-judge.js.map

package/dist/benchmarks/evaluators/structured-judge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"structured-judge.js","sourceRoot":"","sources":["../../../benchmarks/evaluators/structured-judge.ts"],"names":[],"mappings":"AASA,MAAM,WAAW,GAAG,0BAA0B,CAAC;AAc/C;;GAEG;AACH,SAAS,cAAc,CACrB,IAAgB,EAChB,WAAmB;IAEnB,MAAM,SAAS,GAAa,EAAE,CAAC;IAE/B,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAChD,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC;YACrB,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACnC,CAAC;aAAM,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;YAC7B,SAAS,CAAC,IAAI,CACZ,uCAAuC,SAAS,CAAC,SAAS,GAAG,CAC9D,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,SAAS,CAAC,IAAI,CAAC,0BAA0B,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC;QACnE,CAAC;IACH,CAAC;IAED,+BAA+B;IAC/B,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC7C,SAAS,CAAC,IAAI,CACZ,8CAA8C,OAAO,sCAAsC,CAC5F,CAAC;IACJ,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAiB,EACjB,IAAgB,EAChB,MAAc,EACd,WAAmB;IAEnB,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;IAEpD,MAAM,cAAc,GAAG,SAAS;SAC7B,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;SAChC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,MAAM,MAAM,GAAG;;;EAGf,IAAI,CAAC,WAAW;;;EAGhB,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC;;;EAG1B,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;;;EAGtB,cAAc;;;;;;;;;;2CAU2B,CAAC;IAE1C,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;QAC5C,KAAK,EAAE,WAAW;QAClB,UAAU,EAAE,IAAI;QAChB,WAAW,EAAE,CAAC;QACd,MAAM,EACJ,yEAAyE;QAC3E,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;KAC9C,CAAC,CAAC;IAEH,MAAM,IAAI,GACR,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IAEzE,IAAI,OAGF,CAAC;IAEH,IAAI,CAAC;QACH,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAGvB,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,0CAA0C;QAC1C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QACxC,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,CAAC;gBACH,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAG3B,CAAC;YACL,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,GAAG,EAAE,CAAC;YACf,CAAC;QACH,CAAC;aAAM,CAAC;YACN,OAAO,GAAG,EAAE,CAAC;QACf,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACrC,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,MAAM,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,IAAI,CAGnC,CAAC;QACd,OAAO;YACL,SAAS,EAAE,CAAC;YACZ,MAAM,EAAE,MAAM,KAAK,KAAK,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,KAAK,SAAS;gBACjE,CAAC,CAAC,MAAM;gBACR,CAAC,CAAE,IAAc;YACnB,SAAS,EAAE,CAAC,EAAE,SAAS,IAAI,wBAAwB;SACpD,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;QACzC,IAAI,CAAC,CAAC,MAAM,KAAK,KAAK;YAAE,OAAO,GAAG,GAAG,CAAC,CAAC;QACvC,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS;YAAE,OAAO,GAAG,GAAG,GAAG,CAAC;QAC7C,OAAO,GAAG,CAAC;IACb,CAAC,EAAE,CAAC,CAAC,CAAC;IAEN,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC;IACtC,MAAM,KAAK,GACT,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,cAAc,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAE5D,OAAO;QACL,MAAM,EAAE,IAAI,CAAC,EAAE;QACf,cAAc;QACd,QAAQ;QACR,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,EAAE,CAAC,GAAG,EAAE;QAClC,OAAO;KACR,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CACnC,MAAc;IAEd,6BAA6B;IAC7B,MAAM,cAAc,GAAG,oDAAoD,CAAC;IAC5E,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,KAAK,CAAC;IACV,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACtD,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9C,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,CAAC;IAC3D,CAAC;IAED,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,0EAA0E;IAC1E,IAAI,EAA2C,CAAC;IAChD,IAAI,CAAC;QACH,iEAAiE;QACjE,EAAE,GAAG,OAAO,CAAC,YAAY,CAAgC,CAAC;IAC5D,CAAC;IAAC,MAAM,CAAC;QACP,oEAAoE;QACpE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;IACvE,CAAC;IAED,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,wDAAwD;QACxD,MAAM,UAAU,GAAG,EAAE,CAAC,gBAAgB,CACpC,WAAW,EACX,KAAK,EACL,EAAE,CAAC,YAAY,CAAC,MAAM,EACtB,KAAK,EACL,EAAE,CAAC,UAAU,CAAC,GAAG,CAClB,CAAC;QAEF,iEAAiE;QACjE,0DAA0D;QAC1D,MAAM,WAAW,GACf,CAAE,UAA0D;aACzD,gBAAgB,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;QACpC,WAAW,IAAI,WAAW,CAAC;IAC7B,CAAC;IAED,OAAO;QACL,KAAK,EAAE,WAAW,KAAK,CAAC;QACxB,UAAU,EAAE,WAAW;QACvB,cAAc,EAAE,MAAM,CAAC,MAAM;KAC9B,CAAC;AACJ,CAAC"}

package/dist/benchmarks/evaluators/token-counter.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Count tokens in a string using cl100k_base (Claude/GPT-4 tokenizer).
+ */
+export declare function countTokens(text: string): number;
+/**
+ * Dispose the encoder to free memory.
+ */
+export declare function disposeEncoder(): void;
+//# sourceMappingURL=token-counter.d.ts.map

package/dist/benchmarks/evaluators/token-counter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"token-counter.d.ts","sourceRoot":"","sources":["../../../benchmarks/evaluators/token-counter.ts"],"names":[],"mappings":"AAWA;;GAEG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEhD;AAED;;GAEG;AACH,wBAAgB,cAAc,IAAI,IAAI,CAKrC"}

package/dist/benchmarks/evaluators/token-counter.js

@@ -0,0 +1,24 @@
+import { get_encoding } from "tiktoken";
+let _encoder = null;
+function getEncoder() {
+    if (!_encoder) {
+        _encoder = get_encoding("cl100k_base");
+    }
+    return _encoder;
+}
+/**
+ * Count tokens in a string using cl100k_base (Claude/GPT-4 tokenizer).
+ */
+export function countTokens(text) {
+    return getEncoder().encode(text).length;
+}
+/**
+ * Dispose the encoder to free memory.
+ */
+export function disposeEncoder() {
+    if (_encoder) {
+        _encoder.free();
+        _encoder = null;
+    }
+}
+//# sourceMappingURL=token-counter.js.map

package/dist/benchmarks/evaluators/token-counter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"token-counter.js","sourceRoot":"","sources":["../../../benchmarks/evaluators/token-counter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAiB,MAAM,UAAU,CAAC;AAEvD,IAAI,QAAQ,GAAoB,IAAI,CAAC;AAErC,SAAS,UAAU;IACjB,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,QAAQ,GAAG,YAAY,CAAC,aAAa,CAAC,CAAC;IACzC,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,IAAY;IACtC,OAAO,UAAU,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;AAC1C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc;IAC5B,IAAI,QAAQ,EAAE,CAAC;QACb,QAAQ,CAAC,IAAI,EAAE,CAAC;QAChB,QAAQ,GAAG,IAAI,CAAC;IAClB,CAAC;AACH,CAAC"}

package/dist/benchmarks/generators/generate-corpus.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=generate-corpus.d.ts.map

package/dist/benchmarks/generators/generate-corpus.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"generate-corpus.d.ts","sourceRoot":"","sources":["../../../benchmarks/generators/generate-corpus.ts"],"names":[],"mappings":""}