peerbench 0.0.10 → 0.0.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/README.md +71 -58
  2. package/dist/benchmarks/examples/echo-basic/runner.d.ts +11 -254
  3. package/dist/benchmarks/examples/echo-basic/schema-sets/echo.v1.d.ts +25 -25
  4. package/dist/benchmarks/examples/exact-match-scorer/runner.d.ts +38 -386
  5. package/dist/benchmarks/examples/exact-match-scorer/schema-sets/exact-match.v1.d.ts +25 -25
  6. package/dist/benchmarks/examples/text-transform/runner.d.ts +32 -480
  7. package/dist/benchmarks/examples/text-transform/schema-sets/echo.v1.d.ts +25 -25
  8. package/dist/benchmarks/examples/text-transform/schema-sets/reverse.v1.d.ts +25 -25
  9. package/dist/benchmarks/index.js +180 -248
  10. package/dist/benchmarks/index.js.map +1 -1
  11. package/dist/benchmarks/peerbench/index.d.ts +2 -1
  12. package/dist/benchmarks/peerbench/mcq-runner.d.ts +78 -0
  13. package/dist/benchmarks/peerbench/qa-runner.d.ts +77 -0
  14. package/dist/benchmarks/peerbench/schema-sets/mcq.v1.d.ts +25 -25
  15. package/dist/benchmarks/peerbench/schema-sets/multi-turn.v1.d.ts +25 -25
  16. package/dist/benchmarks/peerbench/schema-sets/qa.v1.d.ts +25 -25
  17. package/dist/chunk-6WDCU5BP.js +9 -0
  18. package/dist/chunk-6WDCU5BP.js.map +1 -0
  19. package/dist/{chunk-YY33MNMV.js → chunk-7KMGLEYP.js} +2 -2
  20. package/dist/{chunk-TRNCF2BG.js → chunk-HBGC6BDW.js} +1 -1
  21. package/dist/chunk-HBGC6BDW.js.map +1 -0
  22. package/dist/{chunk-HMQYGCKI.js → chunk-ZJWSK4VO.js} +1 -1
  23. package/dist/chunk-ZJWSK4VO.js.map +1 -0
  24. package/dist/dev.d.ts +22 -0
  25. package/dist/helpers/define-runner.d.ts +2 -45
  26. package/dist/index.js +2 -2
  27. package/dist/providers/ai-sdk.d.ts +24 -0
  28. package/dist/providers/callables/callable.d.ts +4 -0
  29. package/dist/providers/callables/llm.d.ts +41 -0
  30. package/dist/providers/example/echo.d.ts +12 -11
  31. package/dist/providers/example/restapi.d.ts +11 -18
  32. package/dist/providers/index.d.ts +4 -2
  33. package/dist/providers/index.js +380 -9
  34. package/dist/providers/index.js.map +1 -1
  35. package/dist/providers/mastra.d.ts +16 -21
  36. package/dist/providers/openai.d.ts +25 -10
  37. package/dist/providers/openrouter.d.ts +6 -8
  38. package/dist/schemas/extensions/index.js +1 -1
  39. package/dist/schemas/extensions/response/llm.d.ts +17 -0
  40. package/dist/schemas/index.js +2 -2
  41. package/dist/schemas/llm/index.js +36 -7
  42. package/dist/schemas/llm/index.js.map +1 -1
  43. package/dist/schemas/llm/simple-system-prompt.d.ts +3 -3
  44. package/dist/schemas/llm/system-prompt.d.ts +7 -7
  45. package/dist/schemas/response.d.ts +7 -7
  46. package/dist/schemas/schema-definer.d.ts +5 -5
  47. package/dist/schemas/score.d.ts +7 -7
  48. package/dist/schemas/test-case.d.ts +7 -7
  49. package/dist/scorers/abstract.d.ts +1 -1
  50. package/dist/scorers/index.js +377 -7
  51. package/dist/scorers/index.js.map +1 -1
  52. package/dist/scorers/llm-judge.d.ts +6 -6
  53. package/dist/types/index.d.ts +0 -5
  54. package/dist/types/runner.d.ts +13 -17
  55. package/package.json +8 -7
  56. package/dist/benchmarks/peerbench/runner.d.ts +0 -754
  57. package/dist/chunk-3JHDJEY3.js +0 -374
  58. package/dist/chunk-3JHDJEY3.js.map +0 -1
  59. package/dist/chunk-HMQYGCKI.js.map +0 -1
  60. package/dist/chunk-Q6GSOHOP.js +0 -44
  61. package/dist/chunk-Q6GSOHOP.js.map +0 -1
  62. package/dist/chunk-RTEAK4II.js +0 -37
  63. package/dist/chunk-RTEAK4II.js.map +0 -1
  64. package/dist/chunk-SMLNDQFX.js +0 -244
  65. package/dist/chunk-SMLNDQFX.js.map +0 -1
  66. package/dist/chunk-TRNCF2BG.js.map +0 -1
  67. package/dist/providers/abstract/llm.d.ts +0 -20
  68. /package/dist/{chunk-YY33MNMV.js.map → chunk-7KMGLEYP.js.map} +0 -0
  69. /package/dist/providers/{abstract/provider.d.ts → abstract.d.ts} +0 -0
package/dist/benchmarks/examples/text-transform/schema-sets/echo.v1.d.ts CHANGED
@@ -6,7 +6,7 @@ export declare const TextTransformEchoTestCaseSchemaV1: z.ZodObject<Omit<{
6
6
  schemaVersion: z.ZodNumber;
7
7
  kind: z.ZodString;
8
8
  metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
9
- }, "kind" | "namespace" | "schemaVersion"> & {
9
+ }, "kind" | "schemaVersion" | "namespace"> & {
10
10
  input: z.ZodString;
11
11
  } & {
12
12
  namespace: z.ZodLiteral<"example.peerbench.ai">;
@@ -20,7 +20,7 @@ export declare const TextTransformEchoTestCaseSchemaV1: z.ZodObject<Omit<{
20
20
  kind: "llm/text-transform-echo.tc";
21
21
  schemaVersion: 1;
22
22
  metadata?: Record<string, unknown> | undefined;
23
- }, "kind" | "namespace" | "schemaVersion">) => {
23
+ }, "kind" | "schemaVersion" | "namespace">) => {
24
24
  id: string;
25
25
  input: string;
26
26
  namespace: "example.peerbench.ai";
@@ -35,7 +35,7 @@ export declare const TextTransformEchoTestCaseSchemaV1: z.ZodObject<Omit<{
35
35
  kind: "llm/text-transform-echo.tc";
36
36
  schemaVersion: 1;
37
37
  metadata?: Record<string, unknown> | undefined;
38
- }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../../index.js").IdGenerator): Promise<{
38
+ }, "id" | "kind" | "schemaVersion" | "namespace">, generator: import("../../../../index.js").IdGenerator): Promise<{
39
39
  id: string;
40
40
  input: string;
41
41
  namespace: "example.peerbench.ai";
@@ -54,7 +54,7 @@ export declare const TextTransformEchoResponseSchemaV1: z.ZodObject<Omit<{
54
54
  completedAt: z.ZodNumber;
55
55
  testCaseId: z.ZodString;
56
56
  metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
57
- }, "kind" | "namespace" | "schemaVersion"> & {
57
+ }, "kind" | "schemaVersion" | "namespace"> & {
58
58
  data: z.ZodString;
59
59
  modelSlug: z.ZodString;
60
60
  provider: z.ZodString;
@@ -69,10 +69,10 @@ export declare const TextTransformEchoResponseSchemaV1: z.ZodObject<Omit<{
69
69
  schemaVersion: z.ZodLiteral<1>;
70
70
  }, z.core.$strip> & {
71
71
  new: (input: Omit<{
72
- startedAt: number;
73
- completedAt: number;
74
72
  id: string;
75
73
  testCaseId: string;
74
+ startedAt: number;
75
+ completedAt: number;
76
76
  data: string;
77
77
  modelSlug: string;
78
78
  provider: string;
@@ -85,11 +85,11 @@ export declare const TextTransformEchoResponseSchemaV1: z.ZodObject<Omit<{
85
85
  outputTokensUsed?: number | undefined;
86
86
  inputCost?: string | undefined;
87
87
  outputCost?: string | undefined;
88
- }, "kind" | "namespace" | "schemaVersion">) => {
89
- startedAt: number;
90
- completedAt: number;
88
+ }, "kind" | "schemaVersion" | "namespace">) => {
91
89
  id: string;
92
90
  testCaseId: string;
91
+ startedAt: number;
92
+ completedAt: number;
93
93
  data: string;
94
94
  modelSlug: string;
95
95
  provider: string;
@@ -104,10 +104,10 @@ export declare const TextTransformEchoResponseSchemaV1: z.ZodObject<Omit<{
104
104
  outputCost?: string | undefined;
105
105
  };
106
106
  newWithId(input: Omit<{
107
- startedAt: number;
108
- completedAt: number;
109
107
  id: string;
110
108
  testCaseId: string;
109
+ startedAt: number;
110
+ completedAt: number;
111
111
  data: string;
112
112
  modelSlug: string;
113
113
  provider: string;
@@ -120,11 +120,11 @@ export declare const TextTransformEchoResponseSchemaV1: z.ZodObject<Omit<{
120
120
  outputTokensUsed?: number | undefined;
121
121
  inputCost?: string | undefined;
122
122
  outputCost?: string | undefined;
123
- }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../../index.js").IdGenerator): Promise<{
124
- startedAt: number;
125
- completedAt: number;
123
+ }, "id" | "kind" | "schemaVersion" | "namespace">, generator: import("../../../../index.js").IdGenerator): Promise<{
126
124
  id: string;
127
125
  testCaseId: string;
126
+ startedAt: number;
127
+ completedAt: number;
128
128
  data: string;
129
129
  modelSlug: string;
130
130
  provider: string;
@@ -154,7 +154,7 @@ export declare const TextTransformEchoScoreSchemaV1: z.ZodObject<Omit<{
154
154
  readonly human: "human";
155
155
  readonly algo: "algo";
156
156
  }>;
157
- }, "kind" | "namespace" | "schemaVersion"> & {
157
+ }, "kind" | "schemaVersion" | "namespace"> & {
158
158
  match: z.ZodBoolean;
159
159
  } & {
160
160
  namespace: z.ZodLiteral<"example.peerbench.ai">;
@@ -163,49 +163,49 @@ export declare const TextTransformEchoScoreSchemaV1: z.ZodObject<Omit<{
163
163
  }, z.core.$strip> & {
164
164
  new: (input: Omit<{
165
165
  id: string;
166
- value: number;
167
166
  responseId: string;
167
+ value: number;
168
168
  scoringMethod: "ai" | "human" | "algo";
169
169
  match: boolean;
170
170
  namespace: "example.peerbench.ai";
171
171
  kind: "llm/text-transform-echo.sc";
172
172
  schemaVersion: 1;
173
- metadata?: Record<string, unknown> | undefined;
174
173
  explanation?: string | undefined;
175
- }, "kind" | "namespace" | "schemaVersion">) => {
174
+ metadata?: Record<string, unknown> | undefined;
175
+ }, "kind" | "schemaVersion" | "namespace">) => {
176
176
  id: string;
177
- value: number;
178
177
  responseId: string;
178
+ value: number;
179
179
  scoringMethod: "ai" | "human" | "algo";
180
180
  match: boolean;
181
181
  namespace: "example.peerbench.ai";
182
182
  kind: "llm/text-transform-echo.sc";
183
183
  schemaVersion: 1;
184
- metadata?: Record<string, unknown> | undefined;
185
184
  explanation?: string | undefined;
185
+ metadata?: Record<string, unknown> | undefined;
186
186
  };
187
187
  newWithId(input: Omit<{
188
188
  id: string;
189
- value: number;
190
189
  responseId: string;
190
+ value: number;
191
191
  scoringMethod: "ai" | "human" | "algo";
192
192
  match: boolean;
193
193
  namespace: "example.peerbench.ai";
194
194
  kind: "llm/text-transform-echo.sc";
195
195
  schemaVersion: 1;
196
- metadata?: Record<string, unknown> | undefined;
197
196
  explanation?: string | undefined;
198
- }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../../index.js").IdGenerator): Promise<{
197
+ metadata?: Record<string, unknown> | undefined;
198
+ }, "id" | "kind" | "schemaVersion" | "namespace">, generator: import("../../../../index.js").IdGenerator): Promise<{
199
199
  id: string;
200
- value: number;
201
200
  responseId: string;
201
+ value: number;
202
202
  scoringMethod: "ai" | "human" | "algo";
203
203
  match: boolean;
204
204
  namespace: "example.peerbench.ai";
205
205
  kind: "llm/text-transform-echo.sc";
206
206
  schemaVersion: 1;
207
- metadata?: Record<string, unknown> | undefined;
208
207
  explanation?: string | undefined;
208
+ metadata?: Record<string, unknown> | undefined;
209
209
  }>;
210
210
  };
211
211
  export type TextTransformEchoScoreV1 = z.infer<typeof TextTransformEchoScoreSchemaV1>;
package/dist/benchmarks/examples/text-transform/schema-sets/reverse.v1.d.ts CHANGED
@@ -6,7 +6,7 @@ export declare const TextTransformReverseTestCaseSchemaV1: z.ZodObject<Omit<{
6
6
  schemaVersion: z.ZodNumber;
7
7
  kind: z.ZodString;
8
8
  metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
9
- }, "kind" | "namespace" | "schemaVersion"> & {
9
+ }, "kind" | "schemaVersion" | "namespace"> & {
10
10
  input: z.ZodString;
11
11
  } & {
12
12
  namespace: z.ZodLiteral<"example.peerbench.ai">;
@@ -20,7 +20,7 @@ export declare const TextTransformReverseTestCaseSchemaV1: z.ZodObject<Omit<{
20
20
  kind: "llm/text-transform-reverse.tc";
21
21
  schemaVersion: 1;
22
22
  metadata?: Record<string, unknown> | undefined;
23
- }, "kind" | "namespace" | "schemaVersion">) => {
23
+ }, "kind" | "schemaVersion" | "namespace">) => {
24
24
  id: string;
25
25
  input: string;
26
26
  namespace: "example.peerbench.ai";
@@ -35,7 +35,7 @@ export declare const TextTransformReverseTestCaseSchemaV1: z.ZodObject<Omit<{
35
35
  kind: "llm/text-transform-reverse.tc";
36
36
  schemaVersion: 1;
37
37
  metadata?: Record<string, unknown> | undefined;
38
- }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../../index.js").IdGenerator): Promise<{
38
+ }, "id" | "kind" | "schemaVersion" | "namespace">, generator: import("../../../../index.js").IdGenerator): Promise<{
39
39
  id: string;
40
40
  input: string;
41
41
  namespace: "example.peerbench.ai";
@@ -54,7 +54,7 @@ export declare const TextTransformReverseResponseSchemaV1: z.ZodObject<Omit<{
54
54
  completedAt: z.ZodNumber;
55
55
  testCaseId: z.ZodString;
56
56
  metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
57
- }, "kind" | "namespace" | "schemaVersion"> & {
57
+ }, "kind" | "schemaVersion" | "namespace"> & {
58
58
  data: z.ZodString;
59
59
  modelSlug: z.ZodString;
60
60
  provider: z.ZodString;
@@ -69,10 +69,10 @@ export declare const TextTransformReverseResponseSchemaV1: z.ZodObject<Omit<{
69
69
  schemaVersion: z.ZodLiteral<1>;
70
70
  }, z.core.$strip> & {
71
71
  new: (input: Omit<{
72
- startedAt: number;
73
- completedAt: number;
74
72
  id: string;
75
73
  testCaseId: string;
74
+ startedAt: number;
75
+ completedAt: number;
76
76
  data: string;
77
77
  modelSlug: string;
78
78
  provider: string;
@@ -85,11 +85,11 @@ export declare const TextTransformReverseResponseSchemaV1: z.ZodObject<Omit<{
85
85
  outputTokensUsed?: number | undefined;
86
86
  inputCost?: string | undefined;
87
87
  outputCost?: string | undefined;
88
- }, "kind" | "namespace" | "schemaVersion">) => {
89
- startedAt: number;
90
- completedAt: number;
88
+ }, "kind" | "schemaVersion" | "namespace">) => {
91
89
  id: string;
92
90
  testCaseId: string;
91
+ startedAt: number;
92
+ completedAt: number;
93
93
  data: string;
94
94
  modelSlug: string;
95
95
  provider: string;
@@ -104,10 +104,10 @@ export declare const TextTransformReverseResponseSchemaV1: z.ZodObject<Omit<{
104
104
  outputCost?: string | undefined;
105
105
  };
106
106
  newWithId(input: Omit<{
107
- startedAt: number;
108
- completedAt: number;
109
107
  id: string;
110
108
  testCaseId: string;
109
+ startedAt: number;
110
+ completedAt: number;
111
111
  data: string;
112
112
  modelSlug: string;
113
113
  provider: string;
@@ -120,11 +120,11 @@ export declare const TextTransformReverseResponseSchemaV1: z.ZodObject<Omit<{
120
120
  outputTokensUsed?: number | undefined;
121
121
  inputCost?: string | undefined;
122
122
  outputCost?: string | undefined;
123
- }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../../index.js").IdGenerator): Promise<{
124
- startedAt: number;
125
- completedAt: number;
123
+ }, "id" | "kind" | "schemaVersion" | "namespace">, generator: import("../../../../index.js").IdGenerator): Promise<{
126
124
  id: string;
127
125
  testCaseId: string;
126
+ startedAt: number;
127
+ completedAt: number;
128
128
  data: string;
129
129
  modelSlug: string;
130
130
  provider: string;
@@ -154,7 +154,7 @@ export declare const TextTransformReverseScoreSchemaV1: z.ZodObject<Omit<{
154
154
  readonly human: "human";
155
155
  readonly algo: "algo";
156
156
  }>;
157
- }, "kind" | "namespace" | "schemaVersion"> & {
157
+ }, "kind" | "schemaVersion" | "namespace"> & {
158
158
  match: z.ZodBoolean;
159
159
  expected: z.ZodString;
160
160
  } & {
@@ -164,53 +164,53 @@ export declare const TextTransformReverseScoreSchemaV1: z.ZodObject<Omit<{
164
164
  }, z.core.$strip> & {
165
165
  new: (input: Omit<{
166
166
  id: string;
167
- value: number;
168
167
  responseId: string;
168
+ value: number;
169
169
  scoringMethod: "ai" | "human" | "algo";
170
170
  match: boolean;
171
171
  expected: string;
172
172
  namespace: "example.peerbench.ai";
173
173
  kind: "llm/text-transform-reverse.sc";
174
174
  schemaVersion: 1;
175
- metadata?: Record<string, unknown> | undefined;
176
175
  explanation?: string | undefined;
177
- }, "kind" | "namespace" | "schemaVersion">) => {
176
+ metadata?: Record<string, unknown> | undefined;
177
+ }, "kind" | "schemaVersion" | "namespace">) => {
178
178
  id: string;
179
- value: number;
180
179
  responseId: string;
180
+ value: number;
181
181
  scoringMethod: "ai" | "human" | "algo";
182
182
  match: boolean;
183
183
  expected: string;
184
184
  namespace: "example.peerbench.ai";
185
185
  kind: "llm/text-transform-reverse.sc";
186
186
  schemaVersion: 1;
187
- metadata?: Record<string, unknown> | undefined;
188
187
  explanation?: string | undefined;
188
+ metadata?: Record<string, unknown> | undefined;
189
189
  };
190
190
  newWithId(input: Omit<{
191
191
  id: string;
192
- value: number;
193
192
  responseId: string;
193
+ value: number;
194
194
  scoringMethod: "ai" | "human" | "algo";
195
195
  match: boolean;
196
196
  expected: string;
197
197
  namespace: "example.peerbench.ai";
198
198
  kind: "llm/text-transform-reverse.sc";
199
199
  schemaVersion: 1;
200
- metadata?: Record<string, unknown> | undefined;
201
200
  explanation?: string | undefined;
202
- }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../../index.js").IdGenerator): Promise<{
201
+ metadata?: Record<string, unknown> | undefined;
202
+ }, "id" | "kind" | "schemaVersion" | "namespace">, generator: import("../../../../index.js").IdGenerator): Promise<{
203
203
  id: string;
204
- value: number;
205
204
  responseId: string;
205
+ value: number;
206
206
  scoringMethod: "ai" | "human" | "algo";
207
207
  match: boolean;
208
208
  expected: string;
209
209
  namespace: "example.peerbench.ai";
210
210
  kind: "llm/text-transform-reverse.sc";
211
211
  schemaVersion: 1;
212
- metadata?: Record<string, unknown> | undefined;
213
212
  explanation?: string | undefined;
213
+ metadata?: Record<string, unknown> | undefined;
214
214
  }>;
215
215
  };
216
216
  export type TextTransformReverseScoreV1 = z.infer<typeof TextTransformReverseScoreSchemaV1>;