peerbench 0.0.10 → 0.0.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/README.md +71 -58
  2. package/dist/benchmarks/examples/echo-basic/runner.d.ts +11 -254
  3. package/dist/benchmarks/examples/echo-basic/schema-sets/echo.v1.d.ts +25 -25
  4. package/dist/benchmarks/examples/exact-match-scorer/runner.d.ts +38 -386
  5. package/dist/benchmarks/examples/exact-match-scorer/schema-sets/exact-match.v1.d.ts +25 -25
  6. package/dist/benchmarks/examples/text-transform/runner.d.ts +32 -480
  7. package/dist/benchmarks/examples/text-transform/schema-sets/echo.v1.d.ts +25 -25
  8. package/dist/benchmarks/examples/text-transform/schema-sets/reverse.v1.d.ts +25 -25
  9. package/dist/benchmarks/index.js +180 -248
  10. package/dist/benchmarks/index.js.map +1 -1
  11. package/dist/benchmarks/peerbench/index.d.ts +2 -1
  12. package/dist/benchmarks/peerbench/mcq-runner.d.ts +78 -0
  13. package/dist/benchmarks/peerbench/qa-runner.d.ts +77 -0
  14. package/dist/benchmarks/peerbench/schema-sets/mcq.v1.d.ts +25 -25
  15. package/dist/benchmarks/peerbench/schema-sets/multi-turn.v1.d.ts +25 -25
  16. package/dist/benchmarks/peerbench/schema-sets/qa.v1.d.ts +25 -25
  17. package/dist/chunk-6WDCU5BP.js +9 -0
  18. package/dist/chunk-6WDCU5BP.js.map +1 -0
  19. package/dist/{chunk-YY33MNMV.js → chunk-7KMGLEYP.js} +2 -2
  20. package/dist/{chunk-TRNCF2BG.js → chunk-HBGC6BDW.js} +1 -1
  21. package/dist/chunk-HBGC6BDW.js.map +1 -0
  22. package/dist/{chunk-HMQYGCKI.js → chunk-ZJWSK4VO.js} +1 -1
  23. package/dist/chunk-ZJWSK4VO.js.map +1 -0
  24. package/dist/dev.d.ts +22 -0
  25. package/dist/helpers/define-runner.d.ts +2 -45
  26. package/dist/index.js +2 -2
  27. package/dist/providers/ai-sdk.d.ts +24 -0
  28. package/dist/providers/callables/callable.d.ts +4 -0
  29. package/dist/providers/callables/llm.d.ts +41 -0
  30. package/dist/providers/example/echo.d.ts +12 -11
  31. package/dist/providers/example/restapi.d.ts +11 -18
  32. package/dist/providers/index.d.ts +4 -2
  33. package/dist/providers/index.js +380 -9
  34. package/dist/providers/index.js.map +1 -1
  35. package/dist/providers/mastra.d.ts +16 -21
  36. package/dist/providers/openai.d.ts +25 -10
  37. package/dist/providers/openrouter.d.ts +6 -8
  38. package/dist/schemas/extensions/index.js +1 -1
  39. package/dist/schemas/extensions/response/llm.d.ts +17 -0
  40. package/dist/schemas/index.js +2 -2
  41. package/dist/schemas/llm/index.js +36 -7
  42. package/dist/schemas/llm/index.js.map +1 -1
  43. package/dist/schemas/llm/simple-system-prompt.d.ts +3 -3
  44. package/dist/schemas/llm/system-prompt.d.ts +7 -7
  45. package/dist/schemas/response.d.ts +7 -7
  46. package/dist/schemas/schema-definer.d.ts +5 -5
  47. package/dist/schemas/score.d.ts +7 -7
  48. package/dist/schemas/test-case.d.ts +7 -7
  49. package/dist/scorers/abstract.d.ts +1 -1
  50. package/dist/scorers/index.js +377 -7
  51. package/dist/scorers/index.js.map +1 -1
  52. package/dist/scorers/llm-judge.d.ts +6 -6
  53. package/dist/types/index.d.ts +0 -5
  54. package/dist/types/runner.d.ts +13 -17
  55. package/package.json +8 -7
  56. package/dist/benchmarks/peerbench/runner.d.ts +0 -754
  57. package/dist/chunk-3JHDJEY3.js +0 -374
  58. package/dist/chunk-3JHDJEY3.js.map +0 -1
  59. package/dist/chunk-HMQYGCKI.js.map +0 -1
  60. package/dist/chunk-Q6GSOHOP.js +0 -44
  61. package/dist/chunk-Q6GSOHOP.js.map +0 -1
  62. package/dist/chunk-RTEAK4II.js +0 -37
  63. package/dist/chunk-RTEAK4II.js.map +0 -1
  64. package/dist/chunk-SMLNDQFX.js +0 -244
  65. package/dist/chunk-SMLNDQFX.js.map +0 -1
  66. package/dist/chunk-TRNCF2BG.js.map +0 -1
  67. package/dist/providers/abstract/llm.d.ts +0 -20
  68. /package/dist/{chunk-YY33MNMV.js.map → chunk-7KMGLEYP.js.map} +0 -0
  69. /package/dist/providers/{abstract/provider.d.ts → abstract.d.ts} +0 -0
package/dist/benchmarks/examples/exact-match-scorer/schema-sets/exact-match.v1.d.ts CHANGED
@@ -7,7 +7,7 @@ export declare const ExactMatchTestCaseSchemaV1: z.ZodObject<Omit<{
7
7
  schemaVersion: z.ZodNumber;
8
8
  kind: z.ZodString;
9
9
  metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
10
- }, "kind" | "namespace" | "schemaVersion"> & {
10
+ }, "kind" | "schemaVersion" | "namespace"> & {
11
11
  instruction: z.ZodString;
12
12
  input: z.ZodString;
13
13
  expectedOutput: z.ZodString;
@@ -27,7 +27,7 @@ export declare const ExactMatchTestCaseSchemaV1: z.ZodObject<Omit<{
27
27
  schemaVersion: 1;
28
28
  metadata?: Record<string, unknown> | undefined;
29
29
  normalize?: boolean | undefined;
30
- }, "kind" | "namespace" | "schemaVersion">) => {
30
+ }, "kind" | "schemaVersion" | "namespace">) => {
31
31
  id: string;
32
32
  instruction: string;
33
33
  input: string;
@@ -48,7 +48,7 @@ export declare const ExactMatchTestCaseSchemaV1: z.ZodObject<Omit<{
48
48
  schemaVersion: 1;
49
49
  metadata?: Record<string, unknown> | undefined;
50
50
  normalize?: boolean | undefined;
51
- }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../../index.js").IdGenerator): Promise<{
51
+ }, "id" | "kind" | "schemaVersion" | "namespace">, generator: import("../../../../index.js").IdGenerator): Promise<{
52
52
  id: string;
53
53
  instruction: string;
54
54
  input: string;
@@ -70,7 +70,7 @@ export declare const ExactMatchResponseSchemaV1: z.ZodObject<Omit<{
70
70
  completedAt: z.ZodNumber;
71
71
  testCaseId: z.ZodString;
72
72
  metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
73
- }, "kind" | "namespace" | "schemaVersion"> & {
73
+ }, "kind" | "schemaVersion" | "namespace"> & {
74
74
  data: z.ZodString;
75
75
  modelSlug: z.ZodString;
76
76
  provider: z.ZodString;
@@ -85,10 +85,10 @@ export declare const ExactMatchResponseSchemaV1: z.ZodObject<Omit<{
85
85
  schemaVersion: z.ZodLiteral<1>;
86
86
  }, z.core.$strip> & {
87
87
  new: (input: Omit<{
88
- startedAt: number;
89
- completedAt: number;
90
88
  id: string;
91
89
  testCaseId: string;
90
+ startedAt: number;
91
+ completedAt: number;
92
92
  data: string;
93
93
  modelSlug: string;
94
94
  provider: string;
@@ -101,11 +101,11 @@ export declare const ExactMatchResponseSchemaV1: z.ZodObject<Omit<{
101
101
  outputTokensUsed?: number | undefined;
102
102
  inputCost?: string | undefined;
103
103
  outputCost?: string | undefined;
104
- }, "kind" | "namespace" | "schemaVersion">) => {
105
- startedAt: number;
106
- completedAt: number;
104
+ }, "kind" | "schemaVersion" | "namespace">) => {
107
105
  id: string;
108
106
  testCaseId: string;
107
+ startedAt: number;
108
+ completedAt: number;
109
109
  data: string;
110
110
  modelSlug: string;
111
111
  provider: string;
@@ -120,10 +120,10 @@ export declare const ExactMatchResponseSchemaV1: z.ZodObject<Omit<{
120
120
  outputCost?: string | undefined;
121
121
  };
122
122
  newWithId(input: Omit<{
123
- startedAt: number;
124
- completedAt: number;
125
123
  id: string;
126
124
  testCaseId: string;
125
+ startedAt: number;
126
+ completedAt: number;
127
127
  data: string;
128
128
  modelSlug: string;
129
129
  provider: string;
@@ -136,11 +136,11 @@ export declare const ExactMatchResponseSchemaV1: z.ZodObject<Omit<{
136
136
  outputTokensUsed?: number | undefined;
137
137
  inputCost?: string | undefined;
138
138
  outputCost?: string | undefined;
139
- }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../../index.js").IdGenerator): Promise<{
140
- startedAt: number;
141
- completedAt: number;
139
+ }, "id" | "kind" | "schemaVersion" | "namespace">, generator: import("../../../../index.js").IdGenerator): Promise<{
142
140
  id: string;
143
141
  testCaseId: string;
142
+ startedAt: number;
143
+ completedAt: number;
144
144
  data: string;
145
145
  modelSlug: string;
146
146
  provider: string;
@@ -170,7 +170,7 @@ export declare const ExactMatchScoreSchemaV1: z.ZodObject<Omit<{
170
170
  readonly human: "human";
171
171
  readonly algo: "algo";
172
172
  }>;
173
- }, "kind" | "namespace" | "schemaVersion"> & {
173
+ }, "kind" | "schemaVersion" | "namespace"> & {
174
174
  match: z.ZodBoolean;
175
175
  normalized: z.ZodOptional<z.ZodObject<{
176
176
  expected: z.ZodString;
@@ -191,15 +191,15 @@ export declare const ExactMatchScoreSchemaV1: z.ZodObject<Omit<{
191
191
  }, z.core.$strip> & {
192
192
  new: (input: Omit<{
193
193
  id: string;
194
- value: number;
195
194
  responseId: string;
195
+ value: number;
196
196
  scoringMethod: "ai" | "human" | "algo";
197
197
  match: boolean;
198
198
  namespace: "example.peerbench.ai";
199
199
  kind: "llm/exact-match.sc";
200
200
  schemaVersion: 1;
201
- metadata?: Record<string, unknown> | undefined;
202
201
  explanation?: string | undefined;
202
+ metadata?: Record<string, unknown> | undefined;
203
203
  normalized?: {
204
204
  expected: string;
205
205
  actual: string;
@@ -212,17 +212,17 @@ export declare const ExactMatchScoreSchemaV1: z.ZodObject<Omit<{
212
212
  scorerAIOutputTokensUsed?: number | undefined;
213
213
  scorerAIInputCost?: string | undefined;
214
214
  scorerAIOutputCost?: string | undefined;
215
- }, "kind" | "namespace" | "schemaVersion">) => {
215
+ }, "kind" | "schemaVersion" | "namespace">) => {
216
216
  id: string;
217
- value: number;
218
217
  responseId: string;
218
+ value: number;
219
219
  scoringMethod: "ai" | "human" | "algo";
220
220
  match: boolean;
221
221
  namespace: "example.peerbench.ai";
222
222
  kind: "llm/exact-match.sc";
223
223
  schemaVersion: 1;
224
- metadata?: Record<string, unknown> | undefined;
225
224
  explanation?: string | undefined;
225
+ metadata?: Record<string, unknown> | undefined;
226
226
  normalized?: {
227
227
  expected: string;
228
228
  actual: string;
@@ -238,15 +238,15 @@ export declare const ExactMatchScoreSchemaV1: z.ZodObject<Omit<{
238
238
  };
239
239
  newWithId(input: Omit<{
240
240
  id: string;
241
- value: number;
242
241
  responseId: string;
242
+ value: number;
243
243
  scoringMethod: "ai" | "human" | "algo";
244
244
  match: boolean;
245
245
  namespace: "example.peerbench.ai";
246
246
  kind: "llm/exact-match.sc";
247
247
  schemaVersion: 1;
248
- metadata?: Record<string, unknown> | undefined;
249
248
  explanation?: string | undefined;
249
+ metadata?: Record<string, unknown> | undefined;
250
250
  normalized?: {
251
251
  expected: string;
252
252
  actual: string;
@@ -259,17 +259,17 @@ export declare const ExactMatchScoreSchemaV1: z.ZodObject<Omit<{
259
259
  scorerAIOutputTokensUsed?: number | undefined;
260
260
  scorerAIInputCost?: string | undefined;
261
261
  scorerAIOutputCost?: string | undefined;
262
- }, "kind" | "id" | "namespace" | "schemaVersion">, generator: import("../../../../index.js").IdGenerator): Promise<{
262
+ }, "id" | "kind" | "schemaVersion" | "namespace">, generator: import("../../../../index.js").IdGenerator): Promise<{
263
263
  id: string;
264
- value: number;
265
264
  responseId: string;
265
+ value: number;
266
266
  scoringMethod: "ai" | "human" | "algo";
267
267
  match: boolean;
268
268
  namespace: "example.peerbench.ai";
269
269
  kind: "llm/exact-match.sc";
270
270
  schemaVersion: 1;
271
- metadata?: Record<string, unknown> | undefined;
272
271
  explanation?: string | undefined;
272
+ metadata?: Record<string, unknown> | undefined;
273
273
  normalized?: {
274
274
  expected: string;
275
275
  actual: string;