peerbench 0.0.5 → 0.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{abstract-Dec9Sc5O.d.ts → abstract-BdgLjkNC.d.ts} +7 -1
- package/dist/aggregators/index.d.ts +3 -3
- package/dist/benchmarks/index.d.ts +641 -16
- package/dist/benchmarks/index.js +15 -15
- package/dist/{chunk-DNGT4SJC.js → chunk-3JHDJEY3.js} +16 -7
- package/dist/chunk-3JHDJEY3.js.map +1 -0
- package/dist/{chunk-QY5MPNNB.js → chunk-RTEAK4II.js} +11 -2
- package/dist/chunk-RTEAK4II.js.map +1 -0
- package/dist/{chunk-HPPCDSJ3.js → chunk-SMLNDQFX.js} +17 -8
- package/dist/chunk-SMLNDQFX.js.map +1 -0
- package/dist/{chunk-Q6GSOHOP.js → chunk-ZXTQJFGL.js} +4 -4
- package/dist/{index-BAioQhp2.d.ts → index-Cn20kPrz.d.ts} +2 -2
- package/dist/index.d.ts +25 -6
- package/dist/index.js +4 -4
- package/dist/{llm-BND163ns.d.ts → llm-8ecJmwKJ.d.ts} +1 -1
- package/dist/{llm-judge-BS_oNYUK.d.ts → llm-judge-BuF80-5-.d.ts} +12 -4
- package/dist/provider-DnEBdl1n.d.ts +16 -0
- package/dist/providers/index.d.ts +18 -6
- package/dist/providers/index.js +1 -1
- package/dist/schemas/index.d.ts +7 -7
- package/dist/schemas/llm/index.d.ts +5 -5
- package/dist/schemas/llm/index.js +2 -2
- package/dist/scorers/index.d.ts +11 -7
- package/dist/scorers/index.js +1 -1
- package/package.json +1 -1
- package/dist/chunk-DNGT4SJC.js.map +0 -1
- package/dist/chunk-HPPCDSJ3.js.map +0 -1
- package/dist/chunk-QY5MPNNB.js.map +0 -1
- package/dist/provider-BDjGp2y-.d.ts +0 -10
- /package/dist/{chunk-Q6GSOHOP.js.map → chunk-ZXTQJFGL.js.map} +0 -0
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { I as IdGenerator } from '../index-
|
|
1
|
+
import { I as IdGenerator } from '../index-Cn20kPrz.js';
|
|
2
2
|
import z__default, { z } from 'zod';
|
|
3
|
-
import { A as AbstractLLMProvider } from '../llm-
|
|
4
|
-
import { a as MCQScorer, L as LLMAsAJudgeScorer } from '../llm-judge-
|
|
3
|
+
import { A as AbstractLLMProvider } from '../llm-8ecJmwKJ.js';
|
|
4
|
+
import { a as MCQScorer, L as LLMAsAJudgeScorer } from '../llm-judge-BuF80-5-.js';
|
|
5
5
|
import { J as JSONFileStorage } from '../json-file-Bgv9TLcX.js';
|
|
6
|
-
import '../provider-
|
|
7
|
-
import '../abstract-
|
|
6
|
+
import '../provider-DnEBdl1n.js';
|
|
7
|
+
import '../abstract-BdgLjkNC.js';
|
|
8
8
|
import 'openai/resources/shared';
|
|
9
9
|
import 'openai/resources/chat/completions';
|
|
10
10
|
import '../rate-limiter-CSmVIRsM.js';
|
|
@@ -54,7 +54,7 @@ declare const MCQTestCaseSchemaV1: z.ZodObject<Omit<{
|
|
|
54
54
|
kind: "llm/mcq.tc";
|
|
55
55
|
schemaVersion: 1;
|
|
56
56
|
metadata?: Record<string, unknown> | undefined;
|
|
57
|
-
}, "
|
|
57
|
+
}, "kind" | "id" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
|
|
58
58
|
id: string;
|
|
59
59
|
question: string;
|
|
60
60
|
options: Record<string, string>;
|
|
@@ -141,7 +141,7 @@ declare const MCQResponseSchemaV1: z.ZodObject<Omit<{
|
|
|
141
141
|
outputTokensUsed?: number | undefined;
|
|
142
142
|
inputCost?: string | undefined;
|
|
143
143
|
outputCost?: string | undefined;
|
|
144
|
-
}, "
|
|
144
|
+
}, "kind" | "id" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
|
|
145
145
|
startedAt: number;
|
|
146
146
|
completedAt: number;
|
|
147
147
|
id: string;
|
|
@@ -248,7 +248,7 @@ declare const MCQScoreSchemaV1: z.ZodObject<Omit<{
|
|
|
248
248
|
scorerAIOutputTokensUsed?: number | undefined;
|
|
249
249
|
scorerAIInputCost?: string | undefined;
|
|
250
250
|
scorerAIOutputCost?: string | undefined;
|
|
251
|
-
}, "
|
|
251
|
+
}, "kind" | "id" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
|
|
252
252
|
id: string;
|
|
253
253
|
value: number;
|
|
254
254
|
responseId: string;
|
|
@@ -335,7 +335,7 @@ declare const MultiTurnTestCaseSchemaV1: z.ZodObject<Omit<{
|
|
|
335
335
|
metadata?: Record<string, unknown> | undefined;
|
|
336
336
|
maxTurns?: number | undefined;
|
|
337
337
|
expectedOutcome?: string | undefined;
|
|
338
|
-
}, "
|
|
338
|
+
}, "kind" | "id" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
|
|
339
339
|
id: string;
|
|
340
340
|
messages: {
|
|
341
341
|
role: string;
|
|
@@ -467,7 +467,7 @@ declare const MultiTurnResponseSchemaV1: z.ZodObject<Omit<{
|
|
|
467
467
|
outputTokensUsed?: number | undefined;
|
|
468
468
|
inputCost?: string | undefined;
|
|
469
469
|
outputCost?: string | undefined;
|
|
470
|
-
}, "
|
|
470
|
+
}, "kind" | "id" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
|
|
471
471
|
startedAt: number;
|
|
472
472
|
completedAt: number;
|
|
473
473
|
id: string;
|
|
@@ -596,7 +596,7 @@ declare const MultiTurnScoreSchemaV1: z.ZodObject<Omit<{
|
|
|
596
596
|
scorerAIOutputTokensUsed?: number | undefined;
|
|
597
597
|
scorerAIInputCost?: string | undefined;
|
|
598
598
|
scorerAIOutputCost?: string | undefined;
|
|
599
|
-
}, "
|
|
599
|
+
}, "kind" | "id" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
|
|
600
600
|
id: string;
|
|
601
601
|
value: number;
|
|
602
602
|
responseId: string;
|
|
@@ -666,7 +666,7 @@ declare const QATestCaseSchemaV1: z.ZodObject<Omit<{
|
|
|
666
666
|
kind: "llm/qa.tc";
|
|
667
667
|
schemaVersion: 1;
|
|
668
668
|
metadata?: Record<string, unknown> | undefined;
|
|
669
|
-
}, "
|
|
669
|
+
}, "kind" | "id" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
|
|
670
670
|
id: string;
|
|
671
671
|
question: string;
|
|
672
672
|
goodAnswers: string[];
|
|
@@ -753,7 +753,7 @@ declare const QAResponseSchemaV1: z.ZodObject<Omit<{
|
|
|
753
753
|
outputTokensUsed?: number | undefined;
|
|
754
754
|
inputCost?: string | undefined;
|
|
755
755
|
outputCost?: string | undefined;
|
|
756
|
-
}, "
|
|
756
|
+
}, "kind" | "id" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
|
|
757
757
|
startedAt: number;
|
|
758
758
|
completedAt: number;
|
|
759
759
|
id: string;
|
|
@@ -856,7 +856,7 @@ declare const QAScoreSchemaV1: z.ZodObject<Omit<{
|
|
|
856
856
|
scorerAIOutputTokensUsed?: number | undefined;
|
|
857
857
|
scorerAIInputCost?: string | undefined;
|
|
858
858
|
scorerAIOutputCost?: string | undefined;
|
|
859
|
-
}, "
|
|
859
|
+
}, "kind" | "id" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
|
|
860
860
|
id: string;
|
|
861
861
|
value: number;
|
|
862
862
|
responseId: string;
|
|
@@ -878,7 +878,7 @@ declare const QAScoreSchemaV1: z.ZodObject<Omit<{
|
|
|
878
878
|
};
|
|
879
879
|
type QAScoreV1 = z.infer<typeof QAScoreSchemaV1>;
|
|
880
880
|
|
|
881
|
-
declare const peerbenchRunner: (params: {
|
|
881
|
+
declare const peerbenchRunner: ((params: {
|
|
882
882
|
testCase: {
|
|
883
883
|
id: string;
|
|
884
884
|
question: string;
|
|
@@ -1002,7 +1002,632 @@ declare const peerbenchRunner: (params: {
|
|
|
1002
1002
|
scorerAIInputCost?: string | undefined;
|
|
1003
1003
|
scorerAIOutputCost?: string | undefined;
|
|
1004
1004
|
} | undefined;
|
|
1005
|
-
}
|
|
1005
|
+
}>) & {
|
|
1006
|
+
config: {
|
|
1007
|
+
runConfigSchema: z__default.ZodObject<{
|
|
1008
|
+
model: z__default.ZodString;
|
|
1009
|
+
llmJudgeModel: z__default.ZodOptional<z__default.ZodString>;
|
|
1010
|
+
llmJudgeSystemPrompt: z__default.ZodOptional<z__default.ZodObject<Omit<{
|
|
1011
|
+
id: z__default.ZodString;
|
|
1012
|
+
namespace: z__default.ZodString;
|
|
1013
|
+
kind: z__default.ZodString;
|
|
1014
|
+
schemaVersion: z__default.ZodNumber;
|
|
1015
|
+
version: z__default.ZodNumber;
|
|
1016
|
+
metadata: z__default.ZodOptional<z__default.ZodRecord<z__default.ZodString, z__default.ZodUnknown>>;
|
|
1017
|
+
}, "kind" | "namespace" | "schemaVersion"> & {
|
|
1018
|
+
content: z__default.ZodString;
|
|
1019
|
+
} & {
|
|
1020
|
+
namespace: z__default.ZodLiteral<"peerbench.ai">;
|
|
1021
|
+
kind: z__default.ZodLiteral<`${string}/simple.sys-prompt`>;
|
|
1022
|
+
schemaVersion: z__default.ZodLiteral<1>;
|
|
1023
|
+
}, z__default.core.$strip> & {
|
|
1024
|
+
new: (input: Omit<{
|
|
1025
|
+
id: string;
|
|
1026
|
+
version: number;
|
|
1027
|
+
content: string;
|
|
1028
|
+
namespace: "peerbench.ai";
|
|
1029
|
+
kind: `${string}/simple.sys-prompt`;
|
|
1030
|
+
schemaVersion: 1;
|
|
1031
|
+
metadata?: Record<string, unknown> | undefined;
|
|
1032
|
+
}, "kind" | "namespace" | "schemaVersion">) => {
|
|
1033
|
+
id: string;
|
|
1034
|
+
version: number;
|
|
1035
|
+
content: string;
|
|
1036
|
+
namespace: "peerbench.ai";
|
|
1037
|
+
kind: `${string}/simple.sys-prompt`;
|
|
1038
|
+
schemaVersion: 1;
|
|
1039
|
+
metadata?: Record<string, unknown> | undefined;
|
|
1040
|
+
};
|
|
1041
|
+
newWithId(input: Omit<{
|
|
1042
|
+
id: string;
|
|
1043
|
+
version: number;
|
|
1044
|
+
content: string;
|
|
1045
|
+
namespace: "peerbench.ai";
|
|
1046
|
+
kind: `${string}/simple.sys-prompt`;
|
|
1047
|
+
schemaVersion: 1;
|
|
1048
|
+
metadata?: Record<string, unknown> | undefined;
|
|
1049
|
+
}, "kind" | "id" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
|
|
1050
|
+
id: string;
|
|
1051
|
+
version: number;
|
|
1052
|
+
content: string;
|
|
1053
|
+
namespace: "peerbench.ai";
|
|
1054
|
+
kind: `${string}/simple.sys-prompt`;
|
|
1055
|
+
schemaVersion: 1;
|
|
1056
|
+
metadata?: Record<string, unknown> | undefined;
|
|
1057
|
+
}>;
|
|
1058
|
+
}>;
|
|
1059
|
+
llmJudgeFieldsToExtract: z__default.ZodOptional<z__default.ZodRecord<z__default.ZodString, z__default.ZodCustom<z__default.ZodType<unknown, unknown, z__default.core.$ZodTypeInternals<unknown, unknown>>, z__default.ZodType<unknown, unknown, z__default.core.$ZodTypeInternals<unknown, unknown>>>>>;
|
|
1060
|
+
systemPrompt: z__default.ZodOptional<z__default.ZodObject<Omit<{
|
|
1061
|
+
id: z__default.ZodString;
|
|
1062
|
+
namespace: z__default.ZodString;
|
|
1063
|
+
kind: z__default.ZodString;
|
|
1064
|
+
schemaVersion: z__default.ZodNumber;
|
|
1065
|
+
version: z__default.ZodNumber;
|
|
1066
|
+
metadata: z__default.ZodOptional<z__default.ZodRecord<z__default.ZodString, z__default.ZodUnknown>>;
|
|
1067
|
+
}, "kind" | "namespace" | "schemaVersion"> & {
|
|
1068
|
+
content: z__default.ZodString;
|
|
1069
|
+
} & {
|
|
1070
|
+
namespace: z__default.ZodLiteral<"peerbench.ai">;
|
|
1071
|
+
kind: z__default.ZodLiteral<`${string}/simple.sys-prompt`>;
|
|
1072
|
+
schemaVersion: z__default.ZodLiteral<1>;
|
|
1073
|
+
}, z__default.core.$strip> & {
|
|
1074
|
+
new: (input: Omit<{
|
|
1075
|
+
id: string;
|
|
1076
|
+
version: number;
|
|
1077
|
+
content: string;
|
|
1078
|
+
namespace: "peerbench.ai";
|
|
1079
|
+
kind: `${string}/simple.sys-prompt`;
|
|
1080
|
+
schemaVersion: 1;
|
|
1081
|
+
metadata?: Record<string, unknown> | undefined;
|
|
1082
|
+
}, "kind" | "namespace" | "schemaVersion">) => {
|
|
1083
|
+
id: string;
|
|
1084
|
+
version: number;
|
|
1085
|
+
content: string;
|
|
1086
|
+
namespace: "peerbench.ai";
|
|
1087
|
+
kind: `${string}/simple.sys-prompt`;
|
|
1088
|
+
schemaVersion: 1;
|
|
1089
|
+
metadata?: Record<string, unknown> | undefined;
|
|
1090
|
+
};
|
|
1091
|
+
newWithId(input: Omit<{
|
|
1092
|
+
id: string;
|
|
1093
|
+
version: number;
|
|
1094
|
+
content: string;
|
|
1095
|
+
namespace: "peerbench.ai";
|
|
1096
|
+
kind: `${string}/simple.sys-prompt`;
|
|
1097
|
+
schemaVersion: 1;
|
|
1098
|
+
metadata?: Record<string, unknown> | undefined;
|
|
1099
|
+
}, "kind" | "id" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
|
|
1100
|
+
id: string;
|
|
1101
|
+
version: number;
|
|
1102
|
+
content: string;
|
|
1103
|
+
namespace: "peerbench.ai";
|
|
1104
|
+
kind: `${string}/simple.sys-prompt`;
|
|
1105
|
+
schemaVersion: 1;
|
|
1106
|
+
metadata?: Record<string, unknown> | undefined;
|
|
1107
|
+
}>;
|
|
1108
|
+
}>;
|
|
1109
|
+
templateVariables: z__default.ZodOptional<z__default.ZodRecord<z__default.ZodString, z__default.ZodString>>;
|
|
1110
|
+
}, z__default.core.$strip>;
|
|
1111
|
+
schemaSets: [{
|
|
1112
|
+
readonly testCase: z__default.ZodObject<Omit<{
|
|
1113
|
+
id: z__default.ZodString;
|
|
1114
|
+
namespace: z__default.ZodString;
|
|
1115
|
+
schemaVersion: z__default.ZodNumber;
|
|
1116
|
+
kind: z__default.ZodString;
|
|
1117
|
+
metadata: z__default.ZodOptional<z__default.ZodRecord<z__default.ZodString, z__default.ZodUnknown>>;
|
|
1118
|
+
}, "kind" | "namespace" | "schemaVersion"> & {
|
|
1119
|
+
question: z__default.ZodString;
|
|
1120
|
+
options: z__default.ZodRecord<z__default.ZodString, z__default.ZodString>;
|
|
1121
|
+
correctAnswerKeys: z__default.ZodArray<z__default.ZodString>;
|
|
1122
|
+
} & {
|
|
1123
|
+
namespace: z__default.ZodLiteral<"peerbench.ai">;
|
|
1124
|
+
kind: z__default.ZodLiteral<"llm/mcq.tc">;
|
|
1125
|
+
schemaVersion: z__default.ZodLiteral<1>;
|
|
1126
|
+
}, z__default.core.$strip> & {
|
|
1127
|
+
new: (input: Omit<{
|
|
1128
|
+
id: string;
|
|
1129
|
+
question: string;
|
|
1130
|
+
options: Record<string, string>;
|
|
1131
|
+
correctAnswerKeys: string[];
|
|
1132
|
+
namespace: "peerbench.ai";
|
|
1133
|
+
kind: "llm/mcq.tc";
|
|
1134
|
+
schemaVersion: 1;
|
|
1135
|
+
metadata?: Record<string, unknown> | undefined;
|
|
1136
|
+
}, "kind" | "namespace" | "schemaVersion">) => {
|
|
1137
|
+
id: string;
|
|
1138
|
+
question: string;
|
|
1139
|
+
options: Record<string, string>;
|
|
1140
|
+
correctAnswerKeys: string[];
|
|
1141
|
+
namespace: "peerbench.ai";
|
|
1142
|
+
kind: "llm/mcq.tc";
|
|
1143
|
+
schemaVersion: 1;
|
|
1144
|
+
metadata?: Record<string, unknown> | undefined;
|
|
1145
|
+
};
|
|
1146
|
+
newWithId(input: Omit<{
|
|
1147
|
+
id: string;
|
|
1148
|
+
question: string;
|
|
1149
|
+
options: Record<string, string>;
|
|
1150
|
+
correctAnswerKeys: string[];
|
|
1151
|
+
namespace: "peerbench.ai";
|
|
1152
|
+
kind: "llm/mcq.tc";
|
|
1153
|
+
schemaVersion: 1;
|
|
1154
|
+
metadata?: Record<string, unknown> | undefined;
|
|
1155
|
+
}, "kind" | "id" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
|
|
1156
|
+
id: string;
|
|
1157
|
+
question: string;
|
|
1158
|
+
options: Record<string, string>;
|
|
1159
|
+
correctAnswerKeys: string[];
|
|
1160
|
+
namespace: "peerbench.ai";
|
|
1161
|
+
kind: "llm/mcq.tc";
|
|
1162
|
+
schemaVersion: 1;
|
|
1163
|
+
metadata?: Record<string, unknown> | undefined;
|
|
1164
|
+
}>;
|
|
1165
|
+
};
|
|
1166
|
+
readonly response: z__default.ZodObject<Omit<{
|
|
1167
|
+
id: z__default.ZodString;
|
|
1168
|
+
namespace: z__default.ZodString;
|
|
1169
|
+
schemaVersion: z__default.ZodNumber;
|
|
1170
|
+
kind: z__default.ZodString;
|
|
1171
|
+
startedAt: z__default.ZodNumber;
|
|
1172
|
+
completedAt: z__default.ZodNumber;
|
|
1173
|
+
testCaseId: z__default.ZodString;
|
|
1174
|
+
metadata: z__default.ZodOptional<z__default.ZodRecord<z__default.ZodString, z__default.ZodUnknown>>;
|
|
1175
|
+
}, "kind" | "namespace" | "schemaVersion"> & {
|
|
1176
|
+
data: z__default.ZodString;
|
|
1177
|
+
modelSlug: z__default.ZodString;
|
|
1178
|
+
provider: z__default.ZodString;
|
|
1179
|
+
systemPromptId: z__default.ZodOptional<z__default.ZodString>;
|
|
1180
|
+
inputTokensUsed: z__default.ZodOptional<z__default.ZodNumber>;
|
|
1181
|
+
outputTokensUsed: z__default.ZodOptional<z__default.ZodNumber>;
|
|
1182
|
+
inputCost: z__default.ZodOptional<z__default.ZodString>;
|
|
1183
|
+
outputCost: z__default.ZodOptional<z__default.ZodString>;
|
|
1184
|
+
} & {
|
|
1185
|
+
namespace: z__default.ZodLiteral<"peerbench.ai">;
|
|
1186
|
+
kind: z__default.ZodLiteral<"llm/mcq.rs">;
|
|
1187
|
+
schemaVersion: z__default.ZodLiteral<1>;
|
|
1188
|
+
}, z__default.core.$strip> & {
|
|
1189
|
+
new: (input: Omit<{
|
|
1190
|
+
startedAt: number;
|
|
1191
|
+
completedAt: number;
|
|
1192
|
+
id: string;
|
|
1193
|
+
testCaseId: string;
|
|
1194
|
+
data: string;
|
|
1195
|
+
modelSlug: string;
|
|
1196
|
+
provider: string;
|
|
1197
|
+
namespace: "peerbench.ai";
|
|
1198
|
+
kind: "llm/mcq.rs";
|
|
1199
|
+
schemaVersion: 1;
|
|
1200
|
+
metadata?: Record<string, unknown> | undefined;
|
|
1201
|
+
systemPromptId?: string | undefined;
|
|
1202
|
+
inputTokensUsed?: number | undefined;
|
|
1203
|
+
outputTokensUsed?: number | undefined;
|
|
1204
|
+
inputCost?: string | undefined;
|
|
1205
|
+
outputCost?: string | undefined;
|
|
1206
|
+
}, "kind" | "namespace" | "schemaVersion">) => {
|
|
1207
|
+
startedAt: number;
|
|
1208
|
+
completedAt: number;
|
|
1209
|
+
id: string;
|
|
1210
|
+
testCaseId: string;
|
|
1211
|
+
data: string;
|
|
1212
|
+
modelSlug: string;
|
|
1213
|
+
provider: string;
|
|
1214
|
+
namespace: "peerbench.ai";
|
|
1215
|
+
kind: "llm/mcq.rs";
|
|
1216
|
+
schemaVersion: 1;
|
|
1217
|
+
metadata?: Record<string, unknown> | undefined;
|
|
1218
|
+
systemPromptId?: string | undefined;
|
|
1219
|
+
inputTokensUsed?: number | undefined;
|
|
1220
|
+
outputTokensUsed?: number | undefined;
|
|
1221
|
+
inputCost?: string | undefined;
|
|
1222
|
+
outputCost?: string | undefined;
|
|
1223
|
+
};
|
|
1224
|
+
newWithId(input: Omit<{
|
|
1225
|
+
startedAt: number;
|
|
1226
|
+
completedAt: number;
|
|
1227
|
+
id: string;
|
|
1228
|
+
testCaseId: string;
|
|
1229
|
+
data: string;
|
|
1230
|
+
modelSlug: string;
|
|
1231
|
+
provider: string;
|
|
1232
|
+
namespace: "peerbench.ai";
|
|
1233
|
+
kind: "llm/mcq.rs";
|
|
1234
|
+
schemaVersion: 1;
|
|
1235
|
+
metadata?: Record<string, unknown> | undefined;
|
|
1236
|
+
systemPromptId?: string | undefined;
|
|
1237
|
+
inputTokensUsed?: number | undefined;
|
|
1238
|
+
outputTokensUsed?: number | undefined;
|
|
1239
|
+
inputCost?: string | undefined;
|
|
1240
|
+
outputCost?: string | undefined;
|
|
1241
|
+
}, "kind" | "id" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
|
|
1242
|
+
startedAt: number;
|
|
1243
|
+
completedAt: number;
|
|
1244
|
+
id: string;
|
|
1245
|
+
testCaseId: string;
|
|
1246
|
+
data: string;
|
|
1247
|
+
modelSlug: string;
|
|
1248
|
+
provider: string;
|
|
1249
|
+
namespace: "peerbench.ai";
|
|
1250
|
+
kind: "llm/mcq.rs";
|
|
1251
|
+
schemaVersion: 1;
|
|
1252
|
+
metadata?: Record<string, unknown> | undefined;
|
|
1253
|
+
systemPromptId?: string | undefined;
|
|
1254
|
+
inputTokensUsed?: number | undefined;
|
|
1255
|
+
outputTokensUsed?: number | undefined;
|
|
1256
|
+
inputCost?: string | undefined;
|
|
1257
|
+
outputCost?: string | undefined;
|
|
1258
|
+
}>;
|
|
1259
|
+
};
|
|
1260
|
+
readonly score: z__default.ZodObject<Omit<{
|
|
1261
|
+
id: z__default.ZodString;
|
|
1262
|
+
namespace: z__default.ZodString;
|
|
1263
|
+
kind: z__default.ZodString;
|
|
1264
|
+
schemaVersion: z__default.ZodNumber;
|
|
1265
|
+
value: z__default.ZodNumber;
|
|
1266
|
+
responseId: z__default.ZodString;
|
|
1267
|
+
explanation: z__default.ZodOptional<z__default.ZodString>;
|
|
1268
|
+
metadata: z__default.ZodOptional<z__default.ZodRecord<z__default.ZodString, z__default.ZodUnknown>>;
|
|
1269
|
+
scoringMethod: z__default.ZodEnum<{
|
|
1270
|
+
readonly ai: "ai";
|
|
1271
|
+
readonly human: "human";
|
|
1272
|
+
readonly algo: "algo";
|
|
1273
|
+
}>;
|
|
1274
|
+
}, "kind" | "namespace" | "schemaVersion"> & {
|
|
1275
|
+
extractedAnswers: z__default.ZodArray<z__default.ZodString>;
|
|
1276
|
+
scorerAISystemPrompt: z__default.ZodOptional<z__default.ZodString>;
|
|
1277
|
+
scorerAISystemPromptId: z__default.ZodOptional<z__default.ZodString>;
|
|
1278
|
+
scorerAIProvider: z__default.ZodOptional<z__default.ZodString>;
|
|
1279
|
+
scorerAIModelSlug: z__default.ZodOptional<z__default.ZodString>;
|
|
1280
|
+
scorerAIInputTokensUsed: z__default.ZodOptional<z__default.ZodNumber>;
|
|
1281
|
+
scorerAIOutputTokensUsed: z__default.ZodOptional<z__default.ZodNumber>;
|
|
1282
|
+
scorerAIInputCost: z__default.ZodOptional<z__default.ZodString>;
|
|
1283
|
+
scorerAIOutputCost: z__default.ZodOptional<z__default.ZodString>;
|
|
1284
|
+
} & {
|
|
1285
|
+
namespace: z__default.ZodLiteral<"peerbench.ai">;
|
|
1286
|
+
kind: z__default.ZodLiteral<"llm/mcq.sc">;
|
|
1287
|
+
schemaVersion: z__default.ZodLiteral<1>;
|
|
1288
|
+
}, z__default.core.$strip> & {
|
|
1289
|
+
new: (input: Omit<{
|
|
1290
|
+
id: string;
|
|
1291
|
+
value: number;
|
|
1292
|
+
responseId: string;
|
|
1293
|
+
scoringMethod: "ai" | "human" | "algo";
|
|
1294
|
+
extractedAnswers: string[];
|
|
1295
|
+
namespace: "peerbench.ai";
|
|
1296
|
+
kind: "llm/mcq.sc";
|
|
1297
|
+
schemaVersion: 1;
|
|
1298
|
+
metadata?: Record<string, unknown> | undefined;
|
|
1299
|
+
explanation?: string | undefined;
|
|
1300
|
+
scorerAISystemPrompt?: string | undefined;
|
|
1301
|
+
scorerAISystemPromptId?: string | undefined;
|
|
1302
|
+
scorerAIProvider?: string | undefined;
|
|
1303
|
+
scorerAIModelSlug?: string | undefined;
|
|
1304
|
+
scorerAIInputTokensUsed?: number | undefined;
|
|
1305
|
+
scorerAIOutputTokensUsed?: number | undefined;
|
|
1306
|
+
scorerAIInputCost?: string | undefined;
|
|
1307
|
+
scorerAIOutputCost?: string | undefined;
|
|
1308
|
+
}, "kind" | "namespace" | "schemaVersion">) => {
|
|
1309
|
+
id: string;
|
|
1310
|
+
value: number;
|
|
1311
|
+
responseId: string;
|
|
1312
|
+
scoringMethod: "ai" | "human" | "algo";
|
|
1313
|
+
extractedAnswers: string[];
|
|
1314
|
+
namespace: "peerbench.ai";
|
|
1315
|
+
kind: "llm/mcq.sc";
|
|
1316
|
+
schemaVersion: 1;
|
|
1317
|
+
metadata?: Record<string, unknown> | undefined;
|
|
1318
|
+
explanation?: string | undefined;
|
|
1319
|
+
scorerAISystemPrompt?: string | undefined;
|
|
1320
|
+
scorerAISystemPromptId?: string | undefined;
|
|
1321
|
+
scorerAIProvider?: string | undefined;
|
|
1322
|
+
scorerAIModelSlug?: string | undefined;
|
|
1323
|
+
scorerAIInputTokensUsed?: number | undefined;
|
|
1324
|
+
scorerAIOutputTokensUsed?: number | undefined;
|
|
1325
|
+
scorerAIInputCost?: string | undefined;
|
|
1326
|
+
scorerAIOutputCost?: string | undefined;
|
|
1327
|
+
};
|
|
1328
|
+
newWithId(input: Omit<{
|
|
1329
|
+
id: string;
|
|
1330
|
+
value: number;
|
|
1331
|
+
responseId: string;
|
|
1332
|
+
scoringMethod: "ai" | "human" | "algo";
|
|
1333
|
+
extractedAnswers: string[];
|
|
1334
|
+
namespace: "peerbench.ai";
|
|
1335
|
+
kind: "llm/mcq.sc";
|
|
1336
|
+
schemaVersion: 1;
|
|
1337
|
+
metadata?: Record<string, unknown> | undefined;
|
|
1338
|
+
explanation?: string | undefined;
|
|
1339
|
+
scorerAISystemPrompt?: string | undefined;
|
|
1340
|
+
scorerAISystemPromptId?: string | undefined;
|
|
1341
|
+
scorerAIProvider?: string | undefined;
|
|
1342
|
+
scorerAIModelSlug?: string | undefined;
|
|
1343
|
+
scorerAIInputTokensUsed?: number | undefined;
|
|
1344
|
+
scorerAIOutputTokensUsed?: number | undefined;
|
|
1345
|
+
scorerAIInputCost?: string | undefined;
|
|
1346
|
+
scorerAIOutputCost?: string | undefined;
|
|
1347
|
+
}, "kind" | "id" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
|
|
1348
|
+
id: string;
|
|
1349
|
+
value: number;
|
|
1350
|
+
responseId: string;
|
|
1351
|
+
scoringMethod: "ai" | "human" | "algo";
|
|
1352
|
+
extractedAnswers: string[];
|
|
1353
|
+
namespace: "peerbench.ai";
|
|
1354
|
+
kind: "llm/mcq.sc";
|
|
1355
|
+
schemaVersion: 1;
|
|
1356
|
+
metadata?: Record<string, unknown> | undefined;
|
|
1357
|
+
explanation?: string | undefined;
|
|
1358
|
+
scorerAISystemPrompt?: string | undefined;
|
|
1359
|
+
scorerAISystemPromptId?: string | undefined;
|
|
1360
|
+
scorerAIProvider?: string | undefined;
|
|
1361
|
+
scorerAIModelSlug?: string | undefined;
|
|
1362
|
+
scorerAIInputTokensUsed?: number | undefined;
|
|
1363
|
+
scorerAIOutputTokensUsed?: number | undefined;
|
|
1364
|
+
scorerAIInputCost?: string | undefined;
|
|
1365
|
+
scorerAIOutputCost?: string | undefined;
|
|
1366
|
+
}>;
|
|
1367
|
+
};
|
|
1368
|
+
}, {
|
|
1369
|
+
readonly testCase: z__default.ZodObject<Omit<{
|
|
1370
|
+
id: z__default.ZodString;
|
|
1371
|
+
namespace: z__default.ZodString;
|
|
1372
|
+
schemaVersion: z__default.ZodNumber;
|
|
1373
|
+
kind: z__default.ZodString;
|
|
1374
|
+
metadata: z__default.ZodOptional<z__default.ZodRecord<z__default.ZodString, z__default.ZodUnknown>>;
|
|
1375
|
+
}, "kind" | "namespace" | "schemaVersion"> & {
|
|
1376
|
+
question: z__default.ZodString;
|
|
1377
|
+
goodAnswers: z__default.ZodArray<z__default.ZodString>;
|
|
1378
|
+
badAnswers: z__default.ZodArray<z__default.ZodString>;
|
|
1379
|
+
} & {
|
|
1380
|
+
namespace: z__default.ZodLiteral<"peerbench.ai">;
|
|
1381
|
+
kind: z__default.ZodLiteral<"llm/qa.tc">;
|
|
1382
|
+
schemaVersion: z__default.ZodLiteral<1>;
|
|
1383
|
+
}, z__default.core.$strip> & {
|
|
1384
|
+
new: (input: Omit<{
|
|
1385
|
+
id: string;
|
|
1386
|
+
question: string;
|
|
1387
|
+
goodAnswers: string[];
|
|
1388
|
+
badAnswers: string[];
|
|
1389
|
+
namespace: "peerbench.ai";
|
|
1390
|
+
kind: "llm/qa.tc";
|
|
1391
|
+
schemaVersion: 1;
|
|
1392
|
+
metadata?: Record<string, unknown> | undefined;
|
|
1393
|
+
}, "kind" | "namespace" | "schemaVersion">) => {
|
|
1394
|
+
id: string;
|
|
1395
|
+
question: string;
|
|
1396
|
+
goodAnswers: string[];
|
|
1397
|
+
badAnswers: string[];
|
|
1398
|
+
namespace: "peerbench.ai";
|
|
1399
|
+
kind: "llm/qa.tc";
|
|
1400
|
+
schemaVersion: 1;
|
|
1401
|
+
metadata?: Record<string, unknown> | undefined;
|
|
1402
|
+
};
|
|
1403
|
+
newWithId(input: Omit<{
|
|
1404
|
+
id: string;
|
|
1405
|
+
question: string;
|
|
1406
|
+
goodAnswers: string[];
|
|
1407
|
+
badAnswers: string[];
|
|
1408
|
+
namespace: "peerbench.ai";
|
|
1409
|
+
kind: "llm/qa.tc";
|
|
1410
|
+
schemaVersion: 1;
|
|
1411
|
+
metadata?: Record<string, unknown> | undefined;
|
|
1412
|
+
}, "kind" | "id" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
|
|
1413
|
+
id: string;
|
|
1414
|
+
question: string;
|
|
1415
|
+
goodAnswers: string[];
|
|
1416
|
+
badAnswers: string[];
|
|
1417
|
+
namespace: "peerbench.ai";
|
|
1418
|
+
kind: "llm/qa.tc";
|
|
1419
|
+
schemaVersion: 1;
|
|
1420
|
+
metadata?: Record<string, unknown> | undefined;
|
|
1421
|
+
}>;
|
|
1422
|
+
};
|
|
1423
|
+
readonly response: z__default.ZodObject<Omit<{
|
|
1424
|
+
id: z__default.ZodString;
|
|
1425
|
+
namespace: z__default.ZodString;
|
|
1426
|
+
schemaVersion: z__default.ZodNumber;
|
|
1427
|
+
kind: z__default.ZodString;
|
|
1428
|
+
startedAt: z__default.ZodNumber;
|
|
1429
|
+
completedAt: z__default.ZodNumber;
|
|
1430
|
+
testCaseId: z__default.ZodString;
|
|
1431
|
+
metadata: z__default.ZodOptional<z__default.ZodRecord<z__default.ZodString, z__default.ZodUnknown>>;
|
|
1432
|
+
}, "kind" | "namespace" | "schemaVersion"> & {
|
|
1433
|
+
data: z__default.ZodString;
|
|
1434
|
+
modelSlug: z__default.ZodString;
|
|
1435
|
+
provider: z__default.ZodString;
|
|
1436
|
+
systemPromptId: z__default.ZodOptional<z__default.ZodString>;
|
|
1437
|
+
inputTokensUsed: z__default.ZodOptional<z__default.ZodNumber>;
|
|
1438
|
+
outputTokensUsed: z__default.ZodOptional<z__default.ZodNumber>;
|
|
1439
|
+
inputCost: z__default.ZodOptional<z__default.ZodString>;
|
|
1440
|
+
outputCost: z__default.ZodOptional<z__default.ZodString>;
|
|
1441
|
+
} & {
|
|
1442
|
+
namespace: z__default.ZodLiteral<"peerbench.ai">;
|
|
1443
|
+
kind: z__default.ZodLiteral<"llm/qa.rs">;
|
|
1444
|
+
schemaVersion: z__default.ZodLiteral<1>;
|
|
1445
|
+
}, z__default.core.$strip> & {
|
|
1446
|
+
new: (input: Omit<{
|
|
1447
|
+
startedAt: number;
|
|
1448
|
+
completedAt: number;
|
|
1449
|
+
id: string;
|
|
1450
|
+
testCaseId: string;
|
|
1451
|
+
data: string;
|
|
1452
|
+
modelSlug: string;
|
|
1453
|
+
provider: string;
|
|
1454
|
+
namespace: "peerbench.ai";
|
|
1455
|
+
kind: "llm/qa.rs";
|
|
1456
|
+
schemaVersion: 1;
|
|
1457
|
+
metadata?: Record<string, unknown> | undefined;
|
|
1458
|
+
systemPromptId?: string | undefined;
|
|
1459
|
+
inputTokensUsed?: number | undefined;
|
|
1460
|
+
outputTokensUsed?: number | undefined;
|
|
1461
|
+
inputCost?: string | undefined;
|
|
1462
|
+
outputCost?: string | undefined;
|
|
1463
|
+
}, "kind" | "namespace" | "schemaVersion">) => {
|
|
1464
|
+
startedAt: number;
|
|
1465
|
+
completedAt: number;
|
|
1466
|
+
id: string;
|
|
1467
|
+
testCaseId: string;
|
|
1468
|
+
data: string;
|
|
1469
|
+
modelSlug: string;
|
|
1470
|
+
provider: string;
|
|
1471
|
+
namespace: "peerbench.ai";
|
|
1472
|
+
kind: "llm/qa.rs";
|
|
1473
|
+
schemaVersion: 1;
|
|
1474
|
+
metadata?: Record<string, unknown> | undefined;
|
|
1475
|
+
systemPromptId?: string | undefined;
|
|
1476
|
+
inputTokensUsed?: number | undefined;
|
|
1477
|
+
outputTokensUsed?: number | undefined;
|
|
1478
|
+
inputCost?: string | undefined;
|
|
1479
|
+
outputCost?: string | undefined;
|
|
1480
|
+
};
|
|
1481
|
+
newWithId(input: Omit<{
|
|
1482
|
+
startedAt: number;
|
|
1483
|
+
completedAt: number;
|
|
1484
|
+
id: string;
|
|
1485
|
+
testCaseId: string;
|
|
1486
|
+
data: string;
|
|
1487
|
+
modelSlug: string;
|
|
1488
|
+
provider: string;
|
|
1489
|
+
namespace: "peerbench.ai";
|
|
1490
|
+
kind: "llm/qa.rs";
|
|
1491
|
+
schemaVersion: 1;
|
|
1492
|
+
metadata?: Record<string, unknown> | undefined;
|
|
1493
|
+
systemPromptId?: string | undefined;
|
|
1494
|
+
inputTokensUsed?: number | undefined;
|
|
1495
|
+
outputTokensUsed?: number | undefined;
|
|
1496
|
+
inputCost?: string | undefined;
|
|
1497
|
+
outputCost?: string | undefined;
|
|
1498
|
+
}, "kind" | "id" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
|
|
1499
|
+
startedAt: number;
|
|
1500
|
+
completedAt: number;
|
|
1501
|
+
id: string;
|
|
1502
|
+
testCaseId: string;
|
|
1503
|
+
data: string;
|
|
1504
|
+
modelSlug: string;
|
|
1505
|
+
provider: string;
|
|
1506
|
+
namespace: "peerbench.ai";
|
|
1507
|
+
kind: "llm/qa.rs";
|
|
1508
|
+
schemaVersion: 1;
|
|
1509
|
+
metadata?: Record<string, unknown> | undefined;
|
|
1510
|
+
systemPromptId?: string | undefined;
|
|
1511
|
+
inputTokensUsed?: number | undefined;
|
|
1512
|
+
outputTokensUsed?: number | undefined;
|
|
1513
|
+
inputCost?: string | undefined;
|
|
1514
|
+
outputCost?: string | undefined;
|
|
1515
|
+
}>;
|
|
1516
|
+
};
|
|
1517
|
+
readonly score: z__default.ZodObject<Omit<{
|
|
1518
|
+
id: z__default.ZodString;
|
|
1519
|
+
namespace: z__default.ZodString;
|
|
1520
|
+
kind: z__default.ZodString;
|
|
1521
|
+
schemaVersion: z__default.ZodNumber;
|
|
1522
|
+
value: z__default.ZodNumber;
|
|
1523
|
+
responseId: z__default.ZodString;
|
|
1524
|
+
explanation: z__default.ZodOptional<z__default.ZodString>;
|
|
1525
|
+
metadata: z__default.ZodOptional<z__default.ZodRecord<z__default.ZodString, z__default.ZodUnknown>>;
|
|
1526
|
+
scoringMethod: z__default.ZodEnum<{
|
|
1527
|
+
readonly ai: "ai";
|
|
1528
|
+
readonly human: "human";
|
|
1529
|
+
readonly algo: "algo";
|
|
1530
|
+
}>;
|
|
1531
|
+
}, "kind" | "namespace" | "schemaVersion"> & {
|
|
1532
|
+
scorerAISystemPrompt: z__default.ZodOptional<z__default.ZodString>;
|
|
1533
|
+
scorerAISystemPromptId: z__default.ZodOptional<z__default.ZodString>;
|
|
1534
|
+
scorerAIProvider: z__default.ZodOptional<z__default.ZodString>;
|
|
1535
|
+
scorerAIModelSlug: z__default.ZodOptional<z__default.ZodString>;
|
|
1536
|
+
scorerAIInputTokensUsed: z__default.ZodOptional<z__default.ZodNumber>;
|
|
1537
|
+
scorerAIOutputTokensUsed: z__default.ZodOptional<z__default.ZodNumber>;
|
|
1538
|
+
scorerAIInputCost: z__default.ZodOptional<z__default.ZodString>;
|
|
1539
|
+
scorerAIOutputCost: z__default.ZodOptional<z__default.ZodString>;
|
|
1540
|
+
} & {
|
|
1541
|
+
namespace: z__default.ZodLiteral<"peerbench.ai">;
|
|
1542
|
+
kind: z__default.ZodLiteral<"llm/qa.sc">;
|
|
1543
|
+
schemaVersion: z__default.ZodLiteral<1>;
|
|
1544
|
+
}, z__default.core.$strip> & {
|
|
1545
|
+
new: (input: Omit<{
|
|
1546
|
+
id: string;
|
|
1547
|
+
value: number;
|
|
1548
|
+
responseId: string;
|
|
1549
|
+
scoringMethod: "ai" | "human" | "algo";
|
|
1550
|
+
namespace: "peerbench.ai";
|
|
1551
|
+
kind: "llm/qa.sc";
|
|
1552
|
+
schemaVersion: 1;
|
|
1553
|
+
metadata?: Record<string, unknown> | undefined;
|
|
1554
|
+
explanation?: string | undefined;
|
|
1555
|
+
scorerAISystemPrompt?: string | undefined;
|
|
1556
|
+
scorerAISystemPromptId?: string | undefined;
|
|
1557
|
+
scorerAIProvider?: string | undefined;
|
|
1558
|
+
scorerAIModelSlug?: string | undefined;
|
|
1559
|
+
scorerAIInputTokensUsed?: number | undefined;
|
|
1560
|
+
scorerAIOutputTokensUsed?: number | undefined;
|
|
1561
|
+
scorerAIInputCost?: string | undefined;
|
|
1562
|
+
scorerAIOutputCost?: string | undefined;
|
|
1563
|
+
}, "kind" | "namespace" | "schemaVersion">) => {
|
|
1564
|
+
id: string;
|
|
1565
|
+
value: number;
|
|
1566
|
+
responseId: string;
|
|
1567
|
+
scoringMethod: "ai" | "human" | "algo";
|
|
1568
|
+
namespace: "peerbench.ai";
|
|
1569
|
+
kind: "llm/qa.sc";
|
|
1570
|
+
schemaVersion: 1;
|
|
1571
|
+
metadata?: Record<string, unknown> | undefined;
|
|
1572
|
+
explanation?: string | undefined;
|
|
1573
|
+
scorerAISystemPrompt?: string | undefined;
|
|
1574
|
+
scorerAISystemPromptId?: string | undefined;
|
|
1575
|
+
scorerAIProvider?: string | undefined;
|
|
1576
|
+
scorerAIModelSlug?: string | undefined;
|
|
1577
|
+
scorerAIInputTokensUsed?: number | undefined;
|
|
1578
|
+
scorerAIOutputTokensUsed?: number | undefined;
|
|
1579
|
+
scorerAIInputCost?: string | undefined;
|
|
1580
|
+
scorerAIOutputCost?: string | undefined;
|
|
1581
|
+
};
|
|
1582
|
+
newWithId(input: Omit<{
|
|
1583
|
+
id: string;
|
|
1584
|
+
value: number;
|
|
1585
|
+
responseId: string;
|
|
1586
|
+
scoringMethod: "ai" | "human" | "algo";
|
|
1587
|
+
namespace: "peerbench.ai";
|
|
1588
|
+
kind: "llm/qa.sc";
|
|
1589
|
+
schemaVersion: 1;
|
|
1590
|
+
metadata?: Record<string, unknown> | undefined;
|
|
1591
|
+
explanation?: string | undefined;
|
|
1592
|
+
scorerAISystemPrompt?: string | undefined;
|
|
1593
|
+
scorerAISystemPromptId?: string | undefined;
|
|
1594
|
+
scorerAIProvider?: string | undefined;
|
|
1595
|
+
scorerAIModelSlug?: string | undefined;
|
|
1596
|
+
scorerAIInputTokensUsed?: number | undefined;
|
|
1597
|
+
scorerAIOutputTokensUsed?: number | undefined;
|
|
1598
|
+
scorerAIInputCost?: string | undefined;
|
|
1599
|
+
scorerAIOutputCost?: string | undefined;
|
|
1600
|
+
}, "kind" | "id" | "namespace" | "schemaVersion">, generator: IdGenerator): Promise<{
|
|
1601
|
+
id: string;
|
|
1602
|
+
value: number;
|
|
1603
|
+
responseId: string;
|
|
1604
|
+
scoringMethod: "ai" | "human" | "algo";
|
|
1605
|
+
namespace: "peerbench.ai";
|
|
1606
|
+
kind: "llm/qa.sc";
|
|
1607
|
+
schemaVersion: 1;
|
|
1608
|
+
metadata?: Record<string, unknown> | undefined;
|
|
1609
|
+
explanation?: string | undefined;
|
|
1610
|
+
scorerAISystemPrompt?: string | undefined;
|
|
1611
|
+
scorerAISystemPromptId?: string | undefined;
|
|
1612
|
+
scorerAIProvider?: string | undefined;
|
|
1613
|
+
scorerAIModelSlug?: string | undefined;
|
|
1614
|
+
scorerAIInputTokensUsed?: number | undefined;
|
|
1615
|
+
scorerAIOutputTokensUsed?: number | undefined;
|
|
1616
|
+
scorerAIInputCost?: string | undefined;
|
|
1617
|
+
scorerAIOutputCost?: string | undefined;
|
|
1618
|
+
}>;
|
|
1619
|
+
};
|
|
1620
|
+
}];
|
|
1621
|
+
providers: [typeof AbstractLLMProvider];
|
|
1622
|
+
scorers: [typeof LLMAsAJudgeScorer, typeof MCQScorer];
|
|
1623
|
+
parseRunConfig?: boolean;
|
|
1624
|
+
defaults?: {
|
|
1625
|
+
scorer?: MCQScorer | LLMAsAJudgeScorer | undefined;
|
|
1626
|
+
responseIdGenerator?: IdGenerator;
|
|
1627
|
+
scoreIdGenerator?: IdGenerator;
|
|
1628
|
+
} | undefined;
|
|
1629
|
+
};
|
|
1630
|
+
};
|
|
1006
1631
|
|
|
1007
1632
|
declare class PeerbenchJSONStorage extends JSONFileStorage<MCQTestCaseV1 | MCQResponseV1 | MCQScoreV1 | QATestCaseV1 | QAResponseV1 | QAScoreV1 | MultiTurnTestCaseV1 | MultiTurnResponseV1 | MultiTurnScoreV1> {
|
|
1008
1633
|
constructor(config: {
|