peerbench 0.0.10 → 0.0.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/README.md +71 -58
  2. package/dist/benchmarks/examples/echo-basic/runner.d.ts +11 -254
  3. package/dist/benchmarks/examples/echo-basic/schema-sets/echo.v1.d.ts +25 -25
  4. package/dist/benchmarks/examples/exact-match-scorer/runner.d.ts +38 -386
  5. package/dist/benchmarks/examples/exact-match-scorer/schema-sets/exact-match.v1.d.ts +25 -25
  6. package/dist/benchmarks/examples/text-transform/runner.d.ts +32 -480
  7. package/dist/benchmarks/examples/text-transform/schema-sets/echo.v1.d.ts +25 -25
  8. package/dist/benchmarks/examples/text-transform/schema-sets/reverse.v1.d.ts +25 -25
  9. package/dist/benchmarks/index.js +180 -248
  10. package/dist/benchmarks/index.js.map +1 -1
  11. package/dist/benchmarks/peerbench/index.d.ts +2 -1
  12. package/dist/benchmarks/peerbench/mcq-runner.d.ts +78 -0
  13. package/dist/benchmarks/peerbench/qa-runner.d.ts +77 -0
  14. package/dist/benchmarks/peerbench/schema-sets/mcq.v1.d.ts +25 -25
  15. package/dist/benchmarks/peerbench/schema-sets/multi-turn.v1.d.ts +25 -25
  16. package/dist/benchmarks/peerbench/schema-sets/qa.v1.d.ts +25 -25
  17. package/dist/chunk-6WDCU5BP.js +9 -0
  18. package/dist/chunk-6WDCU5BP.js.map +1 -0
  19. package/dist/{chunk-YY33MNMV.js → chunk-7KMGLEYP.js} +2 -2
  20. package/dist/{chunk-TRNCF2BG.js → chunk-HBGC6BDW.js} +1 -1
  21. package/dist/chunk-HBGC6BDW.js.map +1 -0
  22. package/dist/{chunk-HMQYGCKI.js → chunk-ZJWSK4VO.js} +1 -1
  23. package/dist/chunk-ZJWSK4VO.js.map +1 -0
  24. package/dist/dev.d.ts +22 -0
  25. package/dist/helpers/define-runner.d.ts +2 -45
  26. package/dist/index.js +2 -2
  27. package/dist/providers/ai-sdk.d.ts +24 -0
  28. package/dist/providers/callables/callable.d.ts +4 -0
  29. package/dist/providers/callables/llm.d.ts +41 -0
  30. package/dist/providers/example/echo.d.ts +12 -11
  31. package/dist/providers/example/restapi.d.ts +11 -18
  32. package/dist/providers/index.d.ts +4 -2
  33. package/dist/providers/index.js +380 -9
  34. package/dist/providers/index.js.map +1 -1
  35. package/dist/providers/mastra.d.ts +16 -21
  36. package/dist/providers/openai.d.ts +25 -10
  37. package/dist/providers/openrouter.d.ts +6 -8
  38. package/dist/schemas/extensions/index.js +1 -1
  39. package/dist/schemas/extensions/response/llm.d.ts +17 -0
  40. package/dist/schemas/index.js +2 -2
  41. package/dist/schemas/llm/index.js +36 -7
  42. package/dist/schemas/llm/index.js.map +1 -1
  43. package/dist/schemas/llm/simple-system-prompt.d.ts +3 -3
  44. package/dist/schemas/llm/system-prompt.d.ts +7 -7
  45. package/dist/schemas/response.d.ts +7 -7
  46. package/dist/schemas/schema-definer.d.ts +5 -5
  47. package/dist/schemas/score.d.ts +7 -7
  48. package/dist/schemas/test-case.d.ts +7 -7
  49. package/dist/scorers/abstract.d.ts +1 -1
  50. package/dist/scorers/index.js +377 -7
  51. package/dist/scorers/index.js.map +1 -1
  52. package/dist/scorers/llm-judge.d.ts +6 -6
  53. package/dist/types/index.d.ts +0 -5
  54. package/dist/types/runner.d.ts +13 -17
  55. package/package.json +8 -7
  56. package/dist/benchmarks/peerbench/runner.d.ts +0 -754
  57. package/dist/chunk-3JHDJEY3.js +0 -374
  58. package/dist/chunk-3JHDJEY3.js.map +0 -1
  59. package/dist/chunk-HMQYGCKI.js.map +0 -1
  60. package/dist/chunk-Q6GSOHOP.js +0 -44
  61. package/dist/chunk-Q6GSOHOP.js.map +0 -1
  62. package/dist/chunk-RTEAK4II.js +0 -37
  63. package/dist/chunk-RTEAK4II.js.map +0 -1
  64. package/dist/chunk-SMLNDQFX.js +0 -244
  65. package/dist/chunk-SMLNDQFX.js.map +0 -1
  66. package/dist/chunk-TRNCF2BG.js.map +0 -1
  67. package/dist/providers/abstract/llm.d.ts +0 -20
  68. /package/dist/{chunk-YY33MNMV.js.map → chunk-7KMGLEYP.js.map} +0 -0
  69. /package/dist/providers/{abstract/provider.d.ts → abstract.d.ts} +0 -0
package/dist/benchmarks/examples/text-transform/runner.d.ts CHANGED
@@ -1,37 +1,21 @@
1
- import { AbstractLLMProvider } from "../../../providers/index.js";
2
- import z from "zod";
3
- export declare const textTransformRunner: ((params: {
4
- testCase: {
5
- id: string;
6
- input: string;
7
- namespace: "example.peerbench.ai";
8
- kind: "llm/text-transform-echo.tc";
9
- schemaVersion: 1;
10
- metadata?: Record<string, unknown> | undefined;
11
- } | {
12
- id: string;
13
- input: string;
14
- namespace: "example.peerbench.ai";
15
- kind: "llm/text-transform-reverse.tc";
16
- schemaVersion: 1;
17
- metadata?: Record<string, unknown> | undefined;
18
- };
19
- provider: AbstractLLMProvider;
20
- scorer?: undefined;
21
- runConfig: {
22
- model: string;
23
- temperature?: number | undefined;
24
- };
1
+ import { CallableLLM } from "../../../providers/index.js";
2
+ import { IdGenerator } from "../../../types";
3
+ import { TextTransformEchoTestCaseV1 } from "./schema-sets/echo.v1";
4
+ import { TextTransformReverseTestCaseV1 } from "./schema-sets/reverse.v1";
5
+ export declare const textTransformRunner: (params: {
6
+ testCase: TextTransformEchoTestCaseV1 | TextTransformReverseTestCaseV1;
7
+ target: CallableLLM;
8
+ temperature?: number;
25
9
  idGenerators?: {
26
- response?: import("../../../types").IdGenerator;
27
- score?: import("../../../types").IdGenerator;
10
+ response?: IdGenerator;
11
+ score?: IdGenerator;
28
12
  };
29
13
  }) => Promise<{
30
14
  response: {
31
- startedAt: number;
32
- completedAt: number;
33
15
  id: string;
34
16
  testCaseId: string;
17
+ startedAt: number;
18
+ completedAt: number;
35
19
  data: string;
36
20
  modelSlug: string;
37
21
  provider: string;
@@ -44,11 +28,25 @@ export declare const textTransformRunner: ((params: {
44
28
  outputTokensUsed?: number | undefined;
45
29
  inputCost?: string | undefined;
46
30
  outputCost?: string | undefined;
47
- } | {
48
- startedAt: number;
49
- completedAt: number;
31
+ };
32
+ score: {
33
+ id: string;
34
+ responseId: string;
35
+ value: number;
36
+ scoringMethod: "ai" | "human" | "algo";
37
+ match: boolean;
38
+ namespace: "example.peerbench.ai";
39
+ kind: "llm/text-transform-echo.sc";
40
+ schemaVersion: 1;
41
+ explanation?: string | undefined;
42
+ metadata?: Record<string, unknown> | undefined;
43
+ };
44
+ } | {
45
+ response: {
50
46
  id: string;
51
47
  testCaseId: string;
48
+ startedAt: number;
49
+ completedAt: number;
52
50
  data: string;
53
51
  modelSlug: string;
54
52
  provider: string;
@@ -62,463 +60,17 @@ export declare const textTransformRunner: ((params: {
62
60
  inputCost?: string | undefined;
63
61
  outputCost?: string | undefined;
64
62
  };
65
- score?: {
63
+ score: {
66
64
  id: string;
67
- value: number;
68
65
  responseId: string;
69
- scoringMethod: "ai" | "human" | "algo";
70
- match: boolean;
71
- namespace: "example.peerbench.ai";
72
- kind: "llm/text-transform-echo.sc";
73
- schemaVersion: 1;
74
- metadata?: Record<string, unknown> | undefined;
75
- explanation?: string | undefined;
76
- } | {
77
- id: string;
78
66
  value: number;
79
- responseId: string;
80
67
  scoringMethod: "ai" | "human" | "algo";
81
68
  match: boolean;
82
69
  expected: string;
83
70
  namespace: "example.peerbench.ai";
84
71
  kind: "llm/text-transform-reverse.sc";
85
72
  schemaVersion: 1;
86
- metadata?: Record<string, unknown> | undefined;
87
73
  explanation?: string | undefined;
88
- } | undefined;
89
- }>) & {
90
- config: {
91
- runConfigSchema: z.ZodObject<{
92
- model: z.ZodString;
93
- temperature: z.ZodOptional<z.ZodNumber>;
94
- }, z.core.$strip>;
95
- schemaSets: [{
96
- readonly testCase: z.ZodObject<Omit<{
97
- id: z.ZodString;
98
- namespace: z.ZodString;
99
- schemaVersion: z.ZodNumber;
100
- kind: z.ZodString;
101
- metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
102
- }, "kind" | "namespace" | "schemaVersion"> & {
103
- input: z.ZodString;
104
- } & {
105
- namespace: z.ZodLiteral<"example.peerbench.ai">;
106
- kind: z.ZodLiteral<"llm/text-transform-echo.tc">;
107
- schemaVersion: z.ZodLiteral<1>;
108
- }, z.core.$strip> & {
109
- new: (input: Omit<{
110
- id: string;
111
- input: string;
112
- namespace: "example.peerbench.ai";
113
- kind: "llm/text-transform-echo.tc";
114
- schemaVersion: 1;
115
- metadata?: Record<string, unknown> | undefined;
116
- }, "kind" | "namespace" | "schemaVersion">) => {
117
- id: string;
118
- input: string;
119
- namespace: "example.peerbench.ai";
120
- kind: "llm/text-transform-echo.tc";
121
- schemaVersion: 1;
122
- metadata?: Record<string, unknown> | undefined;
123
- };
124
- newWithId(input: Omit<{
125
- id: string;
126
- input: string;
127
- namespace: "example.peerbench.ai";
128
- kind: "llm/text-transform-echo.tc";
129
- schemaVersion: 1;
130
- metadata?: Record<string, unknown> | undefined;
131
- }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../types").IdGenerator): Promise<{
132
- id: string;
133
- input: string;
134
- namespace: "example.peerbench.ai";
135
- kind: "llm/text-transform-echo.tc";
136
- schemaVersion: 1;
137
- metadata?: Record<string, unknown> | undefined;
138
- }>;
139
- };
140
- readonly response: z.ZodObject<Omit<{
141
- id: z.ZodString;
142
- namespace: z.ZodString;
143
- schemaVersion: z.ZodNumber;
144
- kind: z.ZodString;
145
- startedAt: z.ZodNumber;
146
- completedAt: z.ZodNumber;
147
- testCaseId: z.ZodString;
148
- metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
149
- }, "kind" | "namespace" | "schemaVersion"> & {
150
- data: z.ZodString;
151
- modelSlug: z.ZodString;
152
- provider: z.ZodString;
153
- systemPromptId: z.ZodOptional<z.ZodString>;
154
- inputTokensUsed: z.ZodOptional<z.ZodNumber>;
155
- outputTokensUsed: z.ZodOptional<z.ZodNumber>;
156
- inputCost: z.ZodOptional<z.ZodString>;
157
- outputCost: z.ZodOptional<z.ZodString>;
158
- } & {
159
- namespace: z.ZodLiteral<"example.peerbench.ai">;
160
- kind: z.ZodLiteral<"llm/text-transform-echo.rs">;
161
- schemaVersion: z.ZodLiteral<1>;
162
- }, z.core.$strip> & {
163
- new: (input: Omit<{
164
- startedAt: number;
165
- completedAt: number;
166
- id: string;
167
- testCaseId: string;
168
- data: string;
169
- modelSlug: string;
170
- provider: string;
171
- namespace: "example.peerbench.ai";
172
- kind: "llm/text-transform-echo.rs";
173
- schemaVersion: 1;
174
- metadata?: Record<string, unknown> | undefined;
175
- systemPromptId?: string | undefined;
176
- inputTokensUsed?: number | undefined;
177
- outputTokensUsed?: number | undefined;
178
- inputCost?: string | undefined;
179
- outputCost?: string | undefined;
180
- }, "kind" | "namespace" | "schemaVersion">) => {
181
- startedAt: number;
182
- completedAt: number;
183
- id: string;
184
- testCaseId: string;
185
- data: string;
186
- modelSlug: string;
187
- provider: string;
188
- namespace: "example.peerbench.ai";
189
- kind: "llm/text-transform-echo.rs";
190
- schemaVersion: 1;
191
- metadata?: Record<string, unknown> | undefined;
192
- systemPromptId?: string | undefined;
193
- inputTokensUsed?: number | undefined;
194
- outputTokensUsed?: number | undefined;
195
- inputCost?: string | undefined;
196
- outputCost?: string | undefined;
197
- };
198
- newWithId(input: Omit<{
199
- startedAt: number;
200
- completedAt: number;
201
- id: string;
202
- testCaseId: string;
203
- data: string;
204
- modelSlug: string;
205
- provider: string;
206
- namespace: "example.peerbench.ai";
207
- kind: "llm/text-transform-echo.rs";
208
- schemaVersion: 1;
209
- metadata?: Record<string, unknown> | undefined;
210
- systemPromptId?: string | undefined;
211
- inputTokensUsed?: number | undefined;
212
- outputTokensUsed?: number | undefined;
213
- inputCost?: string | undefined;
214
- outputCost?: string | undefined;
215
- }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../types").IdGenerator): Promise<{
216
- startedAt: number;
217
- completedAt: number;
218
- id: string;
219
- testCaseId: string;
220
- data: string;
221
- modelSlug: string;
222
- provider: string;
223
- namespace: "example.peerbench.ai";
224
- kind: "llm/text-transform-echo.rs";
225
- schemaVersion: 1;
226
- metadata?: Record<string, unknown> | undefined;
227
- systemPromptId?: string | undefined;
228
- inputTokensUsed?: number | undefined;
229
- outputTokensUsed?: number | undefined;
230
- inputCost?: string | undefined;
231
- outputCost?: string | undefined;
232
- }>;
233
- };
234
- readonly score: z.ZodObject<Omit<{
235
- id: z.ZodString;
236
- namespace: z.ZodString;
237
- kind: z.ZodString;
238
- schemaVersion: z.ZodNumber;
239
- value: z.ZodNumber;
240
- responseId: z.ZodString;
241
- explanation: z.ZodOptional<z.ZodString>;
242
- metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
243
- scoringMethod: z.ZodEnum<{
244
- readonly ai: "ai";
245
- readonly human: "human";
246
- readonly algo: "algo";
247
- }>;
248
- }, "kind" | "namespace" | "schemaVersion"> & {
249
- match: z.ZodBoolean;
250
- } & {
251
- namespace: z.ZodLiteral<"example.peerbench.ai">;
252
- kind: z.ZodLiteral<"llm/text-transform-echo.sc">;
253
- schemaVersion: z.ZodLiteral<1>;
254
- }, z.core.$strip> & {
255
- new: (input: Omit<{
256
- id: string;
257
- value: number;
258
- responseId: string;
259
- scoringMethod: "ai" | "human" | "algo";
260
- match: boolean;
261
- namespace: "example.peerbench.ai";
262
- kind: "llm/text-transform-echo.sc";
263
- schemaVersion: 1;
264
- metadata?: Record<string, unknown> | undefined;
265
- explanation?: string | undefined;
266
- }, "kind" | "namespace" | "schemaVersion">) => {
267
- id: string;
268
- value: number;
269
- responseId: string;
270
- scoringMethod: "ai" | "human" | "algo";
271
- match: boolean;
272
- namespace: "example.peerbench.ai";
273
- kind: "llm/text-transform-echo.sc";
274
- schemaVersion: 1;
275
- metadata?: Record<string, unknown> | undefined;
276
- explanation?: string | undefined;
277
- };
278
- newWithId(input: Omit<{
279
- id: string;
280
- value: number;
281
- responseId: string;
282
- scoringMethod: "ai" | "human" | "algo";
283
- match: boolean;
284
- namespace: "example.peerbench.ai";
285
- kind: "llm/text-transform-echo.sc";
286
- schemaVersion: 1;
287
- metadata?: Record<string, unknown> | undefined;
288
- explanation?: string | undefined;
289
- }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../types").IdGenerator): Promise<{
290
- id: string;
291
- value: number;
292
- responseId: string;
293
- scoringMethod: "ai" | "human" | "algo";
294
- match: boolean;
295
- namespace: "example.peerbench.ai";
296
- kind: "llm/text-transform-echo.sc";
297
- schemaVersion: 1;
298
- metadata?: Record<string, unknown> | undefined;
299
- explanation?: string | undefined;
300
- }>;
301
- };
302
- }, {
303
- readonly testCase: z.ZodObject<Omit<{
304
- id: z.ZodString;
305
- namespace: z.ZodString;
306
- schemaVersion: z.ZodNumber;
307
- kind: z.ZodString;
308
- metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
309
- }, "kind" | "namespace" | "schemaVersion"> & {
310
- input: z.ZodString;
311
- } & {
312
- namespace: z.ZodLiteral<"example.peerbench.ai">;
313
- kind: z.ZodLiteral<"llm/text-transform-reverse.tc">;
314
- schemaVersion: z.ZodLiteral<1>;
315
- }, z.core.$strip> & {
316
- new: (input: Omit<{
317
- id: string;
318
- input: string;
319
- namespace: "example.peerbench.ai";
320
- kind: "llm/text-transform-reverse.tc";
321
- schemaVersion: 1;
322
- metadata?: Record<string, unknown> | undefined;
323
- }, "kind" | "namespace" | "schemaVersion">) => {
324
- id: string;
325
- input: string;
326
- namespace: "example.peerbench.ai";
327
- kind: "llm/text-transform-reverse.tc";
328
- schemaVersion: 1;
329
- metadata?: Record<string, unknown> | undefined;
330
- };
331
- newWithId(input: Omit<{
332
- id: string;
333
- input: string;
334
- namespace: "example.peerbench.ai";
335
- kind: "llm/text-transform-reverse.tc";
336
- schemaVersion: 1;
337
- metadata?: Record<string, unknown> | undefined;
338
- }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../types").IdGenerator): Promise<{
339
- id: string;
340
- input: string;
341
- namespace: "example.peerbench.ai";
342
- kind: "llm/text-transform-reverse.tc";
343
- schemaVersion: 1;
344
- metadata?: Record<string, unknown> | undefined;
345
- }>;
346
- };
347
- readonly response: z.ZodObject<Omit<{
348
- id: z.ZodString;
349
- namespace: z.ZodString;
350
- schemaVersion: z.ZodNumber;
351
- kind: z.ZodString;
352
- startedAt: z.ZodNumber;
353
- completedAt: z.ZodNumber;
354
- testCaseId: z.ZodString;
355
- metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
356
- }, "kind" | "namespace" | "schemaVersion"> & {
357
- data: z.ZodString;
358
- modelSlug: z.ZodString;
359
- provider: z.ZodString;
360
- systemPromptId: z.ZodOptional<z.ZodString>;
361
- inputTokensUsed: z.ZodOptional<z.ZodNumber>;
362
- outputTokensUsed: z.ZodOptional<z.ZodNumber>;
363
- inputCost: z.ZodOptional<z.ZodString>;
364
- outputCost: z.ZodOptional<z.ZodString>;
365
- } & {
366
- namespace: z.ZodLiteral<"example.peerbench.ai">;
367
- kind: z.ZodLiteral<"llm/text-transform-reverse.rs">;
368
- schemaVersion: z.ZodLiteral<1>;
369
- }, z.core.$strip> & {
370
- new: (input: Omit<{
371
- startedAt: number;
372
- completedAt: number;
373
- id: string;
374
- testCaseId: string;
375
- data: string;
376
- modelSlug: string;
377
- provider: string;
378
- namespace: "example.peerbench.ai";
379
- kind: "llm/text-transform-reverse.rs";
380
- schemaVersion: 1;
381
- metadata?: Record<string, unknown> | undefined;
382
- systemPromptId?: string | undefined;
383
- inputTokensUsed?: number | undefined;
384
- outputTokensUsed?: number | undefined;
385
- inputCost?: string | undefined;
386
- outputCost?: string | undefined;
387
- }, "kind" | "namespace" | "schemaVersion">) => {
388
- startedAt: number;
389
- completedAt: number;
390
- id: string;
391
- testCaseId: string;
392
- data: string;
393
- modelSlug: string;
394
- provider: string;
395
- namespace: "example.peerbench.ai";
396
- kind: "llm/text-transform-reverse.rs";
397
- schemaVersion: 1;
398
- metadata?: Record<string, unknown> | undefined;
399
- systemPromptId?: string | undefined;
400
- inputTokensUsed?: number | undefined;
401
- outputTokensUsed?: number | undefined;
402
- inputCost?: string | undefined;
403
- outputCost?: string | undefined;
404
- };
405
- newWithId(input: Omit<{
406
- startedAt: number;
407
- completedAt: number;
408
- id: string;
409
- testCaseId: string;
410
- data: string;
411
- modelSlug: string;
412
- provider: string;
413
- namespace: "example.peerbench.ai";
414
- kind: "llm/text-transform-reverse.rs";
415
- schemaVersion: 1;
416
- metadata?: Record<string, unknown> | undefined;
417
- systemPromptId?: string | undefined;
418
- inputTokensUsed?: number | undefined;
419
- outputTokensUsed?: number | undefined;
420
- inputCost?: string | undefined;
421
- outputCost?: string | undefined;
422
- }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../types").IdGenerator): Promise<{
423
- startedAt: number;
424
- completedAt: number;
425
- id: string;
426
- testCaseId: string;
427
- data: string;
428
- modelSlug: string;
429
- provider: string;
430
- namespace: "example.peerbench.ai";
431
- kind: "llm/text-transform-reverse.rs";
432
- schemaVersion: 1;
433
- metadata?: Record<string, unknown> | undefined;
434
- systemPromptId?: string | undefined;
435
- inputTokensUsed?: number | undefined;
436
- outputTokensUsed?: number | undefined;
437
- inputCost?: string | undefined;
438
- outputCost?: string | undefined;
439
- }>;
440
- };
441
- readonly score: z.ZodObject<Omit<{
442
- id: z.ZodString;
443
- namespace: z.ZodString;
444
- kind: z.ZodString;
445
- schemaVersion: z.ZodNumber;
446
- value: z.ZodNumber;
447
- responseId: z.ZodString;
448
- explanation: z.ZodOptional<z.ZodString>;
449
- metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
450
- scoringMethod: z.ZodEnum<{
451
- readonly ai: "ai";
452
- readonly human: "human";
453
- readonly algo: "algo";
454
- }>;
455
- }, "kind" | "namespace" | "schemaVersion"> & {
456
- match: z.ZodBoolean;
457
- expected: z.ZodString;
458
- } & {
459
- namespace: z.ZodLiteral<"example.peerbench.ai">;
460
- kind: z.ZodLiteral<"llm/text-transform-reverse.sc">;
461
- schemaVersion: z.ZodLiteral<1>;
462
- }, z.core.$strip> & {
463
- new: (input: Omit<{
464
- id: string;
465
- value: number;
466
- responseId: string;
467
- scoringMethod: "ai" | "human" | "algo";
468
- match: boolean;
469
- expected: string;
470
- namespace: "example.peerbench.ai";
471
- kind: "llm/text-transform-reverse.sc";
472
- schemaVersion: 1;
473
- metadata?: Record<string, unknown> | undefined;
474
- explanation?: string | undefined;
475
- }, "kind" | "namespace" | "schemaVersion">) => {
476
- id: string;
477
- value: number;
478
- responseId: string;
479
- scoringMethod: "ai" | "human" | "algo";
480
- match: boolean;
481
- expected: string;
482
- namespace: "example.peerbench.ai";
483
- kind: "llm/text-transform-reverse.sc";
484
- schemaVersion: 1;
485
- metadata?: Record<string, unknown> | undefined;
486
- explanation?: string | undefined;
487
- };
488
- newWithId(input: Omit<{
489
- id: string;
490
- value: number;
491
- responseId: string;
492
- scoringMethod: "ai" | "human" | "algo";
493
- match: boolean;
494
- expected: string;
495
- namespace: "example.peerbench.ai";
496
- kind: "llm/text-transform-reverse.sc";
497
- schemaVersion: 1;
498
- metadata?: Record<string, unknown> | undefined;
499
- explanation?: string | undefined;
500
- }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../types").IdGenerator): Promise<{
501
- id: string;
502
- value: number;
503
- responseId: string;
504
- scoringMethod: "ai" | "human" | "algo";
505
- match: boolean;
506
- expected: string;
507
- namespace: "example.peerbench.ai";
508
- kind: "llm/text-transform-reverse.sc";
509
- schemaVersion: 1;
510
- metadata?: Record<string, unknown> | undefined;
511
- explanation?: string | undefined;
512
- }>;
513
- };
514
- }];
515
- providers: [typeof AbstractLLMProvider];
516
- scorers: [];
517
- parseRunConfig?: boolean;
518
- defaults?: {
519
- scorer?: undefined;
520
- responseIdGenerator?: import("../../../types").IdGenerator;
521
- scoreIdGenerator?: import("../../../types").IdGenerator;
522
- } | undefined;
74
+ metadata?: Record<string, unknown> | undefined;
523
75
  };
524
- };
76
+ }>;