peerbench 0.0.1 → 0.0.2-alpha.0

package/dist/chunk-HJH3SW3L.js

@@ -0,0 +1,103 @@
+import {
+  bufferToString
+} from "./chunk-DUBKY73H.js";
+
+// src/loaders/abstract/loader.ts
+var AbstractLoader = class {
+};
+
+// src/loaders/abstract/data.ts
+var AbstractDataLoader = class extends AbstractLoader {
+};
+
+// src/utils/json.ts
+import { stringify } from "safe-stable-stringify";
+function parseJSONL(str, options) {
+  return str.split("\n").filter((line) => line.trim() !== "").map((line) => {
+    const obj = tryParseJson(line);
+    if (options?.errorOnInvalid && !obj) {
+      throw new Error(`Invalid JSON line: ${line}`);
+    }
+    return obj;
+  }).filter((obj) => obj !== void 0);
+}
+function tryParseJson(content) {
+  try {
+    return JSON.parse(content);
+  } catch {
+  }
+}
+function stableStringify(value) {
+  return stringify(value);
+}
+
+// src/loaders/generic-array.ts
+var GenericJSONArrayDataLoader = class extends AbstractDataLoader {
+  async responseBuilder(data, context) {
+    return void 0;
+  }
+  async scoreBuilder(data, context) {
+    return void 0;
+  }
+  async loadData(params) {
+    const contentStr = bufferToString(params.content);
+    let data = tryParseJson(contentStr);
+    if (!data) {
+      data = parseJSONL(contentStr);
+    }
+    if (!data || !Array.isArray(data) || data.length === 0) {
+      throw new Error(
+        "Invalid data: content must be a non-empty JSON or JSONL array"
+      );
+    }
+    return this.transformArrayToResult(data);
+  }
+  async transformArrayToResult(data) {
+    const includedTestCaseIds = /* @__PURE__ */ new Set();
+    const includedResponseIds = /* @__PURE__ */ new Set();
+    const includedScoreIds = /* @__PURE__ */ new Set();
+    const result = {
+      testCases: [],
+      responses: [],
+      scores: []
+    };
+    for (const item of data) {
+      const testCase = await this.testCaseBuilder(item, { result });
+      const response = await this.responseBuilder(item, { result });
+      const score = await this.scoreBuilder(item, { result });
+      if (testCase) {
+        const testCaseIdStr = String(testCase.id);
+        if (!includedTestCaseIds.has(testCaseIdStr)) {
+          includedTestCaseIds.add(testCaseIdStr);
+          result.testCases.push(testCase);
+        }
+      }
+      if (response) {
+        const responseIdStr = String(response.id);
+        if (!includedResponseIds.has(responseIdStr)) {
+          includedResponseIds.add(responseIdStr);
+          result.responses.push(response);
+        }
+      }
+      if (score) {
+        const scoreIdStr = String(score.id);
+        if (!includedScoreIds.has(scoreIdStr)) {
+          includedScoreIds.add(scoreIdStr);
+          result.scores.push(score);
+        }
+      }
+      if (!testCase && !response && !score) {
+        throw new Error("Incompatible object format");
+      }
+    }
+    return result;
+  }
+};
+
+export {
+  AbstractLoader,
+  AbstractDataLoader,
+  stableStringify,
+  GenericJSONArrayDataLoader
+};
+//# sourceMappingURL=chunk-HJH3SW3L.js.map

package/dist/chunk-HJH3SW3L.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/loaders/abstract/loader.ts","../src/loaders/abstract/data.ts","../src/utils/json.ts","../src/loaders/generic-array.ts"],"sourcesContent":["import { BaseResponseV1, BaseScoreV1, BaseTestCaseV1 } from \"@/schemas\";\nimport { BaseBenchmarkSpecV1 } from \"@/schemas/benchmark-spec\";\nimport { MaybePromise } from \"@/types\";\n\nexport type LoaderResult<\n  TTestCase extends BaseTestCaseV1 = BaseTestCaseV1,\n  TResponse extends BaseResponseV1 = BaseResponseV1,\n  TScore extends BaseScoreV1 = BaseScoreV1,\n  TBenchmarkSpec extends BaseBenchmarkSpecV1 = BaseBenchmarkSpecV1,\n> = {\n  testCases: TTestCase[];\n  responses: TResponse[];\n  scores: TScore[];\n\n  benchmarkSpec?: TBenchmarkSpec;\n};\n\nexport abstract class AbstractLoader {\n  abstract readonly kind: string;\n\n  abstract loadData(params: unknown): MaybePromise<LoaderResult>;\n  abstract loadBenchmarkSpec(\n    params: unknown\n  ): MaybePromise<BaseBenchmarkSpecV1>;\n}\n","import { MaybePromise } from \"@/types\";\nimport { AbstractLoader, LoaderResult } from \"./loader\";\nimport { BaseBenchmarkSpecV1 } from \"@/schemas/benchmark-spec\";\n\nexport abstract class AbstractDataLoader extends AbstractLoader {\n  abstract loadData(params: {\n    content: Uint8Array;\n  }): MaybePromise<LoaderResult>;\n\n  abstract loadBenchmarkSpec(params: {\n    content: Uint8Array;\n  }): MaybePromise<BaseBenchmarkSpecV1>;\n}\n","import { stringify } from \"safe-stable-stringify\";\n\n/**\n * Parses JSONL formatted string into an array\n * @returns An array of parsed JSON lines\n */\nexport function parseJSONL<T>(\n  str: string,\n  options?: { errorOnInvalid?: boolean }\n): T[] {\n  return str\n    .split(\"\\n\") // Split per line\n    .filter((line) => line.trim() !== \"\") // Filter empty lines\n    .map((line) => {\n      const obj = tryParseJson(line);\n      if (options?.errorOnInvalid && !obj) {\n        throw new Error(`Invalid JSON line: ${line}`);\n      }\n      return obj;\n    }) // Parse line (parse as undefined if it is invalid)\n    .filter((obj) => obj !== undefined); // Filter invalid lines\n}\n\n/**\n * Tries to parse the given string as JSON.\n * Returns `undefined` if it is not a valid JSON entity.\n */\nexport function tryParseJson<T = any>(content: string): T | undefined {\n  try {\n    return JSON.parse(content);\n  } catch {\n    // Invalid JSON\n  }\n}\n\n/**\n * Stringifies the given JSON value using `safe-stable-stringify` in a stable manner.\n * This stable method generates the same string output for the same input value (including objects).\n */\nexport function stableStringify(value: any) {\n  return stringify(value);\n}\n","import { BaseTestCaseV1 } from \"@/schemas/test-case\";\nimport { BaseResponseV1 } from \"@/schemas/response\";\nimport { BaseScoreV1 } from \"@/schemas/score\";\nimport { MaybePromise } from \"@/types\";\nimport { AbstractDataLoader } from \"@/loaders/abstract/data\";\nimport { LoaderResult } from \"@/loaders/abstract/loader\";\nimport { tryParseJson, parseJSONL } from \"@/utils/json\";\nimport { bufferToString } from \"@/utils/string\";\n\nexport type GenericJSONArrayLoaderResult<\n  TTestCase extends BaseTestCaseV1 = BaseTestCaseV1,\n  TResponse extends BaseResponseV1 = BaseResponseV1,\n  TScore extends BaseScoreV1 = BaseScoreV1,\n> = LoaderResult<TTestCase, TResponse, TScore>;\n\nexport abstract class GenericJSONArrayDataLoader<\n  TTestCase extends BaseTestCaseV1 = BaseTestCaseV1,\n  TResponse extends BaseResponseV1 = BaseResponseV1,\n  TScore extends BaseScoreV1 = BaseScoreV1,\n> extends AbstractDataLoader {\n  protected abstract testCaseBuilder(\n    data: any,\n    context: {\n      result: GenericJSONArrayLoaderResult<TTestCase, TResponse, TScore>;\n    }\n  ): MaybePromise<TTestCase | undefined>;\n\n  protected async responseBuilder(\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    data: any,\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    context: {\n      result: GenericJSONArrayLoaderResult<TTestCase, TResponse, TScore>;\n    }\n  ): Promise<TResponse | undefined> {\n    return undefined;\n  }\n\n  protected async scoreBuilder(\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    data: any,\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    context: {\n      result: GenericJSONArrayLoaderResult<TTestCase, TResponse, TScore>;\n    }\n  ): Promise<TScore | undefined> {\n    return undefined;\n  }\n\n  async loadData(params: {\n    content: Uint8Array;\n  }): Promise<GenericJSONArrayLoaderResult<TTestCase, TResponse, TScore>> {\n    const contentStr = bufferToString(params.content);\n    let data: unknown[] | undefined = tryParseJson<unknown[]>(contentStr);\n\n    if (!data) {\n      data = parseJSONL<unknown>(contentStr);\n    }\n\n    if (!data || !Array.isArray(data) || data.length === 0) {\n      throw new Error(\n        \"Invalid data: content must be a non-empty JSON or JSONL array\"\n      );\n    }\n\n    return this.transformArrayToResult(data);\n  }\n\n  private async transformArrayToResult(\n    data: unknown[]\n  ): Promise<GenericJSONArrayLoaderResult<TTestCase, TResponse, TScore>> {\n    const includedTestCaseIds: Set<string> = new Set();\n    const includedResponseIds: Set<string> = new Set();\n    const includedScoreIds: Set<string> = new Set();\n    const result: GenericJSONArrayLoaderResult<TTestCase, TResponse, TScore> = {\n      testCases: [],\n      responses: [],\n      scores: [],\n    };\n\n    for (const item of data) {\n      const testCase = await this.testCaseBuilder(item, { result });\n      const response = await this.responseBuilder(item, { result });\n      const score = await this.scoreBuilder(item, { result });\n\n      if (testCase) {\n        const testCaseIdStr = String(testCase.id);\n        if (!includedTestCaseIds.has(testCaseIdStr)) {\n          includedTestCaseIds.add(testCaseIdStr);\n          result.testCases.push(testCase);\n        }\n      }\n\n      if (response) {\n        const responseIdStr = String(response.id);\n        if (!includedResponseIds.has(responseIdStr)) {\n          includedResponseIds.add(responseIdStr);\n          result.responses.push(response);\n        }\n      }\n\n      if (score) {\n        const scoreIdStr = String(score.id);\n        if (!includedScoreIds.has(scoreIdStr)) {\n          includedScoreIds.add(scoreIdStr);\n          result.scores.push(score);\n        }\n      }\n\n      if (!testCase && !response && !score) {\n        throw new Error(\"Incompatible object format\");\n      }\n    }\n\n    return result;\n  }\n}\n"],"mappings":";;;;;AAiBO,IAAe,iBAAf,MAA8B;AAOrC;;;ACpBO,IAAe,qBAAf,cAA0C,eAAe;AAQhE;;;ACZA,SAAS,iBAAiB;AAMnB,SAAS,WACd,KACA,SACK;AACL,SAAO,IACJ,MAAM,IAAI,EACV,OAAO,CAAC,SAAS,KAAK,KAAK,MAAM,EAAE,EACnC,IAAI,CAAC,SAAS;AACb,UAAM,MAAM,aAAa,IAAI;AAC7B,QAAI,SAAS,kBAAkB,CAAC,KAAK;AACnC,YAAM,IAAI,MAAM,sBAAsB,IAAI,EAAE;AAAA,IAC9C;AACA,WAAO;AAAA,EACT,CAAC,EACA,OAAO,CAAC,QAAQ,QAAQ,MAAS;AACtC;AAMO,SAAS,aAAsB,SAAgC;AACpE,MAAI;AACF,WAAO,KAAK,MAAM,OAAO;AAAA,EAC3B,QAAQ;AAAA,EAER;AACF;AAMO,SAAS,gBAAgB,OAAY;AAC1C,SAAO,UAAU,KAAK;AACxB;;;AC1BO,IAAe,6BAAf,cAIG,mBAAmB;AAAA,EAQ3B,MAAgB,gBAEd,MAEA,SAGgC;AAChC,WAAO;AAAA,EACT;AAAA,EAEA,MAAgB,aAEd,MAEA,SAG6B;AAC7B,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,SAAS,QAEyD;AACtE,UAAM,aAAa,eAAe,OAAO,OAAO;AAChD,QAAI,OAA8B,aAAwB,UAAU;AAEpE,QAAI,CAAC,MAAM;AACT,aAAO,WAAoB,UAAU;AAAA,IACvC;AAEA,QAAI,CAAC,QAAQ,CAAC,MAAM,QAAQ,IAAI,KAAK,KAAK,WAAW,GAAG;AACtD,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,KAAK,uBAAuB,IAAI;AAAA,EACzC;AAAA,EAEA,MAAc,uBACZ,MACqE;AACrE,UAAM,sBAAmC,oBAAI,IAAI;AACjD,UAAM,sBAAmC,oBAAI,IAAI;AACjD,UAAM,mBAAgC,oBAAI,IAAI;AAC9C,UAAM,SAAqE;AAAA,MACzE,WAAW,CAAC;AAAA,MACZ,WAAW,CAAC;AAAA,MACZ,QAAQ,CAAC;AAAA,IACX;AAEA,eAAW,QAAQ,MAAM;AACvB,YAAM,WAAW,MAAM,KAAK,gBAAgB,MAAM,EAAE,OAAO,CAAC;AAC5D,YAAM,WAAW,MAAM,KAAK,gBAAgB,MAAM,EAAE,OAAO,CAAC;AAC5D,YAAM,QAAQ,MAAM,KAAK,aAAa,MAAM,EAAE,OAAO,CAAC;AAEtD,UAAI,UAAU;AACZ,cAAM,gBAAgB,OAAO,SAAS,EAAE;AACxC,YAAI,CAAC,oBAAoB,IAAI,aAAa,GAAG;AAC3C,8BAAoB,IAAI,aAAa;AACrC,iBAAO,UAAU,KAAK,QAAQ;AAAA,QAChC;AAAA,MACF;AAEA,UAAI,UAAU;AACZ,cAAM,gBAAgB,OAAO,SAAS,EAAE;AACxC,YAAI,CAAC,oBAAoB,IAAI,aAAa,GAAG;AAC3C,8BAAoB,IAAI,aAAa;AACrC,iBAAO,UAAU,KAAK,QAAQ;AAAA,QAChC;AAAA,MACF;AAEA,UAAI,OAAO;AACT,cAAM,aAAa,OAAO,MAAM,EAAE;AAClC,YAAI,CAAC,iBAAiB,IAAI,UAAU,GAAG;AACrC,2BAAiB,IAAI,UAAU;AAC/B,iBAAO,OAAO,KAAK,KAAK;AAAA,QAC1B;AAAA,MACF;AAEA,UAAI,CAAC,YAAY,CAAC,YAAY,CAAC,OAAO;AACpC,cAAM,IAAI,MAAM,4BAA4B;AAAA,MAC9C;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AACF;","names":[]}

package/dist/chunk-IUN2IUCS.js

@@ -0,0 +1,58 @@
+// src/schemas/id.ts
+import z from "zod";
+var IdSchema = z.string();
+
+// src/schemas/schema-definer.ts
+import z2 from "zod";
+function buildSchemaDefiner() {
+  return function(config) {
+    const schema = config.baseSchema.omit({ kind: true, schemaVersion: true }).extend({
+      ...config.fields ?? {},
+      kind: config.kind !== void 0 ? z2.literal(config.kind) : config.baseSchema.shape.kind,
+      schemaVersion: config.schemaVersion !== void 0 ? z2.literal(config.schemaVersion) : config.baseSchema.shape.schemaVersion
+    });
+    return Object.assign(schema, {
+      new(input) {
+        return schema.parse({
+          ...input,
+          kind: config.kind,
+          schemaVersion: config.schemaVersion
+        });
+      },
+      async newWithId(input, generator) {
+        const obj = schema.parse({
+          ...input,
+          id: "",
+          kind: config.kind,
+          schemaVersion: config.schemaVersion
+        });
+        const id = await generator(obj);
+        return {
+          ...obj,
+          id
+        };
+      }
+    });
+  };
+}
+
+// src/schemas/response.ts
+import z3 from "zod";
+var BaseResponseSchemaV1 = z3.object({
+  id: IdSchema,
+  kind: z3.string(),
+  schemaVersion: z3.number(),
+  startedAt: z3.number(),
+  completedAt: z3.number(),
+  testCaseId: IdSchema,
+  metadata: z3.record(z3.string(), z3.unknown()).optional()
+});
+var defineResponseSchema = buildSchemaDefiner();
+
+export {
+  buildSchemaDefiner,
+  IdSchema,
+  BaseResponseSchemaV1,
+  defineResponseSchema
+};
+//# sourceMappingURL=chunk-IUN2IUCS.js.map

package/dist/chunk-IUN2IUCS.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/schemas/id.ts","../src/schemas/schema-definer.ts","../src/schemas/response.ts"],"sourcesContent":["import z from \"zod\";\n\nexport const IdSchema = z.string();\n","import { IdGenerator } from \"@/types\";\nimport z from \"zod\";\n\nexport function buildSchemaDefiner<\n  TBaseShape extends {\n    kind: z.ZodType;\n    schemaVersion: z.ZodType;\n    [key: string]: z.ZodType;\n  },\n>() {\n  return function <\n    TBaseSchema extends z.ZodObject<\n      Omit<TBaseShape, \"kind\" | \"schemaVersion\"> & {\n        kind: z.ZodString | z.ZodLiteral<string>;\n        schemaVersion: z.ZodNumber | z.ZodLiteral<number>;\n      }\n    >,\n    TKind extends string | undefined = undefined,\n    TSchemaVersion extends number | undefined = undefined,\n    TFields extends z.ZodRawShape = {},\n  >(config: {\n    baseSchema: TBaseSchema;\n    kind?: TKind;\n    schemaVersion?: TSchemaVersion;\n    fields?: TFields;\n  }) {\n    const schema = config.baseSchema\n      .omit({ kind: true, schemaVersion: true })\n      .extend({\n        ...((config.fields ?? {}) as TFields),\n        kind: (config.kind !== undefined\n          ? z.literal(config.kind)\n          : config.baseSchema.shape.kind) as TKind extends undefined\n          ? TBaseSchema[\"shape\"][\"kind\"]\n          : z.ZodLiteral<TKind>,\n        schemaVersion: (config.schemaVersion !== undefined\n          ? z.literal(config.schemaVersion)\n          : config.baseSchema.shape\n              .schemaVersion) as TSchemaVersion extends undefined\n          ? TBaseSchema[\"shape\"][\"schemaVersion\"]\n          : z.ZodLiteral<TSchemaVersion>,\n      });\n\n    type SchemaType =\n      TBaseSchema extends z.ZodObject<infer U>\n        ? z.ZodObject<\n            Omit<U, \"kind\" | \"schemaVersion\"> &\n              TFields & {\n                kind: TKind extends undefined ? U[\"kind\"] : z.ZodLiteral<TKind>;\n                schemaVersion: TSchemaVersion extends undefined\n                  ? U[\"schemaVersion\"]\n                  : z.ZodLiteral<TSchemaVersion>;\n              }\n          >\n        : never;\n\n    return Object.assign(schema, {\n      new(input: Omit<z.infer<typeof schema>, \"kind\" | \"schemaVersion\">) {\n        return schema.parse({\n          ...input,\n          kind: config.kind,\n          schemaVersion: config.schemaVersion,\n        });\n      },\n      async newWithId(\n        input: Omit<z.infer<typeof schema>, \"kind\" | \"schemaVersion\" | \"id\">,\n        generator: IdGenerator\n      ) {\n        const obj = schema.parse({\n          ...input,\n          id: \"\",\n          kind: config.kind,\n          schemaVersion: config.schemaVersion,\n        });\n        const id = await generator(obj);\n\n        return {\n          ...obj,\n          id,\n        };\n      },\n    }) as unknown as SchemaType & {\n      /**\n       * Creates a new object with the given input. Uses the `kind` and\n       * `schemaVersion` from the schema config. Although the `input` is already typed,\n       * it still uses `.parse()` to validate it.\n       * @param input\n       * @returns The object that follows the schema\n       */\n      new: (\n        input: Omit<z.infer<SchemaType>, \"kind\" | \"schemaVersion\">\n      ) => z.infer<SchemaType>;\n\n      newWithId(\n        input: Omit<z.infer<SchemaType>, \"id\" | \"kind\" | \"schemaVersion\">,\n        generator: IdGenerator\n      ): Promise<z.infer<SchemaType>>;\n    };\n  };\n}\n","import { IdSchema } from \"./id\";\nimport { buildSchemaDefiner } from \"./schema-definer\";\nimport z from \"zod\";\n\nexport const BaseResponseSchemaV1 = z.object({\n  id: IdSchema,\n  kind: z.string(),\n  schemaVersion: z.number(),\n\n  startedAt: z.number(),\n  completedAt: z.number(),\n  testCaseId: IdSchema,\n  metadata: z.record(z.string(), z.unknown()).optional(),\n});\nexport type BaseResponseV1 = z.infer<typeof BaseResponseSchemaV1>;\n\nexport const defineResponseSchema =\n  buildSchemaDefiner<typeof BaseResponseSchemaV1.shape>();\n"],"mappings":";AAAA,OAAO,OAAO;AAEP,IAAM,WAAW,EAAE,OAAO;;;ACDjC,OAAOA,QAAO;AAEP,SAAS,qBAMZ;AACF,SAAO,SAUL,QAKC;AACD,UAAM,SAAS,OAAO,WACnB,KAAK,EAAE,MAAM,MAAM,eAAe,KAAK,CAAC,EACxC,OAAO;AAAA,MACN,GAAK,OAAO,UAAU,CAAC;AAAA,MACvB,MAAO,OAAO,SAAS,SACnBA,GAAE,QAAQ,OAAO,IAAI,IACrB,OAAO,WAAW,MAAM;AAAA,MAG5B,eAAgB,OAAO,kBAAkB,SACrCA,GAAE,QAAQ,OAAO,aAAa,IAC9B,OAAO,WAAW,MACf;AAAA,IAGT,CAAC;AAeH,WAAO,OAAO,OAAO,QAAQ;AAAA,MAC3B,IAAI,OAA+D;AACjE,eAAO,OAAO,MAAM;AAAA,UAClB,GAAG;AAAA,UACH,MAAM,OAAO;AAAA,UACb,eAAe,OAAO;AAAA,QACxB,CAAC;AAAA,MACH;AAAA,MACA,MAAM,UACJ,OACA,WACA;AACA,cAAM,MAAM,OAAO,MAAM;AAAA,UACvB,GAAG;AAAA,UACH,IAAI;AAAA,UACJ,MAAM,OAAO;AAAA,UACb,eAAe,OAAO;AAAA,QACxB,CAAC;AACD,cAAM,KAAK,MAAM,UAAU,GAAG;AAE9B,eAAO;AAAA,UACL,GAAG;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EAiBH;AACF;;;ACjGA,OAAOC,QAAO;AAEP,IAAM,uBAAuBA,GAAE,OAAO;AAAA,EAC3C,IAAI;AAAA,EACJ,MAAMA,GAAE,OAAO;AAAA,EACf,eAAeA,GAAE,OAAO;AAAA,EAExB,WAAWA,GAAE,OAAO;AAAA,EACpB,aAAaA,GAAE,OAAO;AAAA,EACtB,YAAY;AAAA,EACZ,UAAUA,GAAE,OAAOA,GAAE,OAAO,GAAGA,GAAE,QAAQ,CAAC,EAAE,SAAS;AACvD,CAAC;AAGM,IAAM,uBACX,mBAAsD;","names":["z","z"]}

package/dist/chunk-PZ5AY32C.js

@@ -0,0 +1,10 @@
+var __defProp = Object.defineProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+
+export {
+  __export
+};
+//# sourceMappingURL=chunk-PZ5AY32C.js.map

package/dist/chunk-PZ5AY32C.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/chunk-VBOM2YEG.js

@@ -0,0 +1,47 @@
+// src/errors/polyfill.ts
+function captureStackTrace(error, constructor) {
+  if (typeof window === "undefined") {
+    Error.captureStackTrace(error, constructor);
+  } else {
+    error.stack = new Error(error.message).stack;
+  }
+}
+
+// src/errors/peerbench.ts
+var PeerbenchError = class extends Error {
+  code;
+  constructor(message, options) {
+    super(message, options);
+    this.code = options?.code ?? 0;
+    captureStackTrace(this, this.constructor);
+  }
+};
+
+// src/errors/catalog.ts
+var CatalogItemNotFoundError = class extends PeerbenchError {
+  constructor(itemName) {
+    super(
+      typeof itemName === "string" ? `Item with name "${itemName}" not found in the catalog` : itemName.message,
+      {
+        code: 100
+      }
+    );
+    this.itemName = itemName;
+  }
+};
+var CatalogItemHasNoInstantiateMethodError = class extends PeerbenchError {
+  constructor(itemName) {
+    super(`Item with name "${itemName}" has no instantiate method`, {
+      code: 101
+    });
+    this.itemName = itemName;
+  }
+};
+
+export {
+  captureStackTrace,
+  PeerbenchError,
+  CatalogItemNotFoundError,
+  CatalogItemHasNoInstantiateMethodError
+};
+//# sourceMappingURL=chunk-VBOM2YEG.js.map

package/dist/chunk-VBOM2YEG.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/errors/polyfill.ts","../src/errors/peerbench.ts","../src/errors/catalog.ts"],"sourcesContent":["// This is an internal polyfill that is used by custom Error classes\n// to make them work in both Node.js and browser environments.\n\nexport function captureStackTrace(\n  error: Error,\n  // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type\n  constructor: Function\n) {\n  if (typeof window === \"undefined\") {\n    Error.captureStackTrace(error, constructor);\n  } else {\n    error.stack = new Error(error.message).stack;\n  }\n}\n","import { captureStackTrace } from \"./polyfill\";\n\nexport class PeerbenchError extends Error {\n  code: number;\n\n  constructor(message?: string, options?: ErrorOptions & { code: number }) {\n    super(message, options);\n    this.code = options?.code ?? 0;\n    captureStackTrace(this, this.constructor);\n  }\n}\n","import { PeerbenchError } from \"./peerbench\";\n\nexport class CatalogItemNotFoundError extends PeerbenchError {\n  constructor(public readonly itemName: string | { message: string }) {\n    super(\n      typeof itemName === \"string\"\n        ? `Item with name \"${itemName}\" not found in the catalog`\n        : itemName.message,\n      {\n        code: 100,\n      }\n    );\n  }\n}\n\nexport class CatalogItemHasNoInstantiateMethodError extends PeerbenchError {\n  constructor(public readonly itemName: string) {\n    super(`Item with name \"${itemName}\" has no instantiate method`, {\n      code: 101,\n    });\n  }\n}\n"],"mappings":";AAGO,SAAS,kBACd,OAEA,aACA;AACA,MAAI,OAAO,WAAW,aAAa;AACjC,UAAM,kBAAkB,OAAO,WAAW;AAAA,EAC5C,OAAO;AACL,UAAM,QAAQ,IAAI,MAAM,MAAM,OAAO,EAAE;AAAA,EACzC;AACF;;;ACXO,IAAM,iBAAN,cAA6B,MAAM;AAAA,EACxC;AAAA,EAEA,YAAY,SAAkB,SAA2C;AACvE,UAAM,SAAS,OAAO;AACtB,SAAK,OAAO,SAAS,QAAQ;AAC7B,sBAAkB,MAAM,KAAK,WAAW;AAAA,EAC1C;AACF;;;ACRO,IAAM,2BAAN,cAAuC,eAAe;AAAA,EAC3D,YAA4B,UAAwC;AAClE;AAAA,MACE,OAAO,aAAa,WAChB,mBAAmB,QAAQ,+BAC3B,SAAS;AAAA,MACb;AAAA,QACE,MAAM;AAAA,MACR;AAAA,IACF;AAR0B;AAAA,EAS5B;AACF;AAEO,IAAM,yCAAN,cAAqD,eAAe;AAAA,EACzE,YAA4B,UAAkB;AAC5C,UAAM,mBAAmB,QAAQ,+BAA+B;AAAA,MAC9D,MAAM;AAAA,IACR,CAAC;AAHyB;AAAA,EAI5B;AACF;","names":[]}

package/dist/chunk-ZJWSK4VO.js

@@ -0,0 +1,11 @@
+// src/types/index.ts
+var ScoringMethod = {
+  ai: "ai",
+  human: "human",
+  algo: "algo"
+};
+
+export {
+  ScoringMethod
+};
+//# sourceMappingURL=chunk-ZJWSK4VO.js.map

package/dist/chunk-ZJWSK4VO.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/types/index.ts"],"sourcesContent":["export * from \"./runner\";\n\nimport { IdSchema } from \"@/schemas/id\";\nimport z from \"zod\";\n\nexport type Id = z.infer<typeof IdSchema>;\n\nexport type IdGenerator<TInput = unknown> = (input: TInput) => MaybePromise<Id>;\n\nexport type MaybePromise<T> = T | Promise<T>;\n\nexport const ScoringMethod = {\n  ai: \"ai\",\n  human: \"human\",\n  algo: \"algo\",\n} as const;\nexport type ScoringMethod = (typeof ScoringMethod)[keyof typeof ScoringMethod];\n"],"mappings":";AAWO,IAAM,gBAAgB;AAAA,EAC3B,IAAI;AAAA,EACJ,OAAO;AAAA,EACP,MAAM;AACR;","names":[]}

package/dist/data-BmN5WjZ4.d.ts

@@ -0,0 +1,57 @@
+import { BaseTestCaseV1, BaseBenchmarkSpecV1, BaseResponseV1, BaseScoreV1 } from './schemas/index.js';
+import { A as AbstractProvider } from './provider-BDjGp2y-.js';
+import { I as IdGenerator, M as MaybePromise } from './index-WiPjF2AL.js';
+import { A as AbstractScorer } from './abstract-Dec9Sc5O.js';
+import { a as BaseSystemPromptV1 } from './system-prompt--0FdPWqK.js';
+
+/**
+ * Minimum set of parameters that a runner needs to accept
+ */
+type RunnerParams = {
+    testCase: BaseTestCaseV1;
+    provider: AbstractProvider;
+    spec?: BaseBenchmarkSpecV1;
+    scorer?: AbstractScorer;
+    runConfig: {
+        model: string;
+        llmJudgeModel?: string;
+    };
+    idGenerators?: {
+        response?: IdGenerator;
+        score?: IdGenerator;
+    };
+    [key: string]: unknown;
+};
+type RunnerResult<TResponse extends BaseResponseV1 = BaseResponseV1, TScore extends BaseScoreV1 = BaseScoreV1> = {
+    response: TResponse;
+    score?: TScore;
+    [key: string]: unknown;
+};
+type Runner<TResponse extends BaseResponseV1 = BaseResponseV1, TScore extends BaseScoreV1 = BaseScoreV1, TParams extends RunnerParams = RunnerParams> = (params: TParams) => Promise<RunnerResult<TResponse, TScore>>;
+type LLMChatRunnerParams = RunnerParams & {
+    systemPrompt?: BaseSystemPromptV1;
+};
+type LLMChatRunner<TResponse extends BaseResponseV1 = BaseResponseV1, TScore extends BaseScoreV1 = BaseScoreV1, TParams extends LLMChatRunnerParams = LLMChatRunnerParams> = (params: TParams) => Promise<RunnerResult<TResponse, TScore>>;
+
+type LoaderResult<TTestCase extends BaseTestCaseV1 = BaseTestCaseV1, TResponse extends BaseResponseV1 = BaseResponseV1, TScore extends BaseScoreV1 = BaseScoreV1, TBenchmarkSpec extends BaseBenchmarkSpecV1 = BaseBenchmarkSpecV1> = {
+    testCases: TTestCase[];
+    responses: TResponse[];
+    scores: TScore[];
+    benchmarkSpec?: TBenchmarkSpec;
+};
+declare abstract class AbstractLoader {
+    abstract readonly kind: string;
+    abstract loadData(params: unknown): MaybePromise<LoaderResult>;
+    abstract loadBenchmarkSpec(params: unknown): MaybePromise<BaseBenchmarkSpecV1>;
+}
+
+declare abstract class AbstractDataLoader extends AbstractLoader {
+    abstract loadData(params: {
+        content: Uint8Array;
+    }): MaybePromise<LoaderResult>;
+    abstract loadBenchmarkSpec(params: {
+        content: Uint8Array;
+    }): MaybePromise<BaseBenchmarkSpecV1>;
+}
+
+export { AbstractDataLoader as A, type LoaderResult as L, type Runner as R, AbstractLoader as a, type RunnerParams as b, type RunnerResult as c, type LLMChatRunnerParams as d, type LLMChatRunner as e };

package/dist/generic-array-DLHWSvf1.d.ts

@@ -0,0 +1,22 @@
+import { BaseTestCaseV1, BaseResponseV1, BaseScoreV1 } from './schemas/index.js';
+import { M as MaybePromise } from './index-WiPjF2AL.js';
+import { L as LoaderResult, A as AbstractDataLoader } from './data-BmN5WjZ4.js';
+
+type GenericJSONArrayLoaderResult<TTestCase extends BaseTestCaseV1 = BaseTestCaseV1, TResponse extends BaseResponseV1 = BaseResponseV1, TScore extends BaseScoreV1 = BaseScoreV1> = LoaderResult<TTestCase, TResponse, TScore>;
+declare abstract class GenericJSONArrayDataLoader<TTestCase extends BaseTestCaseV1 = BaseTestCaseV1, TResponse extends BaseResponseV1 = BaseResponseV1, TScore extends BaseScoreV1 = BaseScoreV1> extends AbstractDataLoader {
+    protected abstract testCaseBuilder(data: any, context: {
+        result: GenericJSONArrayLoaderResult<TTestCase, TResponse, TScore>;
+    }): MaybePromise<TTestCase | undefined>;
+    protected responseBuilder(data: any, context: {
+        result: GenericJSONArrayLoaderResult<TTestCase, TResponse, TScore>;
+    }): Promise<TResponse | undefined>;
+    protected scoreBuilder(data: any, context: {
+        result: GenericJSONArrayLoaderResult<TTestCase, TResponse, TScore>;
+    }): Promise<TScore | undefined>;
+    loadData(params: {
+        content: Uint8Array;
+    }): Promise<GenericJSONArrayLoaderResult<TTestCase, TResponse, TScore>>;
+    private transformArrayToResult;
+}
+
+export { type GenericJSONArrayLoaderResult as G, GenericJSONArrayDataLoader as a };

package/dist/index-WiPjF2AL.d.ts

@@ -0,0 +1,15 @@
+import z__default from 'zod';
+
+declare const IdSchema: z__default.ZodString;
+
+type Id = z__default.infer<typeof IdSchema>;
+type IdGenerator<TInput = unknown> = (input: TInput) => MaybePromise<Id>;
+type MaybePromise<T> = T | Promise<T>;
+declare const ScoringMethod: {
+    readonly ai: "ai";
+    readonly human: "human";
+    readonly algo: "algo";
+};
+type ScoringMethod = (typeof ScoringMethod)[keyof typeof ScoringMethod];
+
+export { type IdGenerator as I, type MaybePromise as M, ScoringMethod as S, type Id as a, IdSchema as b };