peerbench 0.0.1 → 0.0.2-alpha-dev.0

package/dist/chunk-ERALDEZY.js

@@ -0,0 +1,112 @@
+import {
+  bufferToString
+} from "./chunk-4UBK6452.js";
+
+// src/storages/abstract.ts
+var AbstractStorage = class {
+};
+
+// src/storages/file.ts
+import { open } from "fs/promises";
+var FileStorage = class extends AbstractStorage {
+  path;
+  codec;
+  fileHandle;
+  constructor(config) {
+    super();
+    this.path = config.path;
+    this.codec = config.codec;
+  }
+  async init() {
+    this.fileHandle = await open(this.path, "r+");
+  }
+  async read(_key, _params) {
+    throw new Error("Method not implemented.");
+  }
+  async readAll(_params) {
+    this.assertInitialized();
+    return await this.codec.readAll({
+      fileHandle: {
+        readAt: this.readAt.bind(this),
+        readChunks: this.readChunks.bind(this),
+        size: this.size.bind(this)
+      }
+    });
+  }
+  async write(_key, _value, _params) {
+    throw new Error("Method not implemented.");
+  }
+  async *readChunks(params) {
+    this.assertInitialized();
+    const fileSize = await this.size();
+    let offset = params.startOffset ?? 0;
+    while (offset < fileSize) {
+      const length = Math.min(params.chunkSize, fileSize - offset);
+      if (length === 0) break;
+      const bytes = await this.readAt(offset, length);
+      if (bytes.length === 0) break;
+      yield { offset, bytes };
+      offset += bytes.length;
+    }
+  }
+  async size() {
+    this.assertInitialized();
+    return await this.fileHandle.stat().then((stat) => stat.size);
+  }
+  async readAt(offset, length) {
+    this.assertInitialized();
+    const buffer = new Uint8Array(length);
+    const result = await this.fileHandle.read(buffer, 0, length, offset);
+    return new Uint8Array(buffer.subarray(0, result.bytesRead));
+  }
+  assertInitialized() {
+    if (!this.fileHandle) {
+      throw new Error("File Storage is not initialized");
+    }
+  }
+};
+var AbstractFileStorageCodec = class {
+  // TODO: Add other methods like write, append, read one etc.
+};
+
+// src/storages/json-file.ts
+var JSONFileStorage = class extends FileStorage {
+  constructor(config) {
+    super({
+      path: config.path,
+      codec: new JSONFileStorageCodec({
+        chunkSize: config.chunkSize,
+        schema: config.schema
+      })
+    });
+  }
+};
+var JSONFileStorageCodec = class extends AbstractFileStorageCodec {
+  chunkSize;
+  schema;
+  constructor(config) {
+    super();
+    this.chunkSize = config?.chunkSize ?? 64 * 1024;
+    this.schema = config.schema;
+  }
+  async readAll(params) {
+    const wholeFile = new Uint8Array(
+      await params.fileHandle.size()
+    );
+    for await (const chunk of params.fileHandle.readChunks({
+      chunkSize: this.chunkSize
+    })) {
+      wholeFile.set(chunk.bytes, chunk.offset);
+    }
+    return this.schema.array().parse(JSON.parse(bufferToString(wholeFile)));
+  }
+};
+
+export {
+  AbstractStorage,
+  FileStorage,
+  AbstractFileStorageCodec,
+  JSONFileStorage,
+  JSONFileStorageCodec
+};
+//# sourceMappingURL=chunk-ERALDEZY.js.map

package/dist/chunk-ERALDEZY.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/storages/abstract.ts","../src/storages/file.ts","../src/storages/json-file.ts"],"sourcesContent":["export abstract class AbstractStorage<TObject> {\n  abstract init(params?: unknown): Promise<void>;\n  abstract read(key: string, params?: unknown): Promise<TObject | null>;\n  abstract readAll(params?: unknown): Promise<TObject[]>;\n  abstract write(\n    key: string,\n    value: TObject,\n    params?: unknown\n  ): Promise<unknown>;\n}\n","import { FileHandle, open } from \"node:fs/promises\";\nimport { AbstractStorage } from \"./abstract\";\n\nexport class FileStorage<TObject> extends AbstractStorage<TObject> {\n  protected path: string;\n  protected codec: AbstractFileStorageCodec<TObject>;\n  protected fileHandle: FileHandle | undefined;\n\n  constructor(config: {\n    path: string;\n    codec: AbstractFileStorageCodec<TObject>;\n  }) {\n    super();\n    this.path = config.path;\n    this.codec = config.codec;\n  }\n\n  async init(): Promise<void> {\n    this.fileHandle = await open(this.path, \"r+\");\n  }\n\n  override async read(\n    _key: string,\n    _params?: unknown\n  ): Promise<TObject | null> {\n    throw new Error(\"Method not implemented.\");\n  }\n\n  override async readAll(_params?: unknown): Promise<TObject[]> {\n    this.assertInitialized();\n\n    return await this.codec.readAll({\n      fileHandle: {\n        readAt: this.readAt.bind(this),\n        readChunks: this.readChunks.bind(this),\n        size: this.size.bind(this),\n      },\n    });\n  }\n\n  override async write(\n    _key: string,\n    _value: TObject,\n    _params?: unknown\n  ): Promise<unknown> {\n    throw new Error(\"Method not implemented.\");\n  }\n\n  protected async *readChunks(params: {\n    startOffset?: number;\n    chunkSize: number;\n  }): AsyncIterable<{ offset: number; bytes: Uint8Array }> {\n    this.assertInitialized();\n\n    const fileSize = await this.size();\n    let offset = params.startOffset ?? 0;\n\n    // NOTE: What if the file size changed during the iteration?\n    while (offset < fileSize) {\n      const length = Math.min(params.chunkSize, fileSize - offset);\n      if (length === 0) break;\n\n      const bytes = await this.readAt(offset, length);\n      if (bytes.length === 0) break;\n      yield { offset, bytes };\n      offset += bytes.length;\n    }\n  }\n\n  protected async size(): Promise<number> {\n    this.assertInitialized();\n\n    return await this.fileHandle.stat().then((stat) => stat.size);\n  }\n\n  protected async readAt(offset: number, length: number): Promise<Uint8Array> {\n    this.assertInitialized();\n\n    const buffer = new Uint8Array(length);\n    const result = await this.fileHandle.read(buffer, 0, length, offset);\n\n    return new Uint8Array(buffer.subarray(0, result.bytesRead));\n  }\n\n  protected assertInitialized(): asserts this is this & {\n    fileHandle: FileHandle;\n  } {\n    if (!this.fileHandle) {\n      throw new Error(\"File Storage is not initialized\");\n    }\n  }\n}\n\nexport type FileStorageFileHandle = {\n  size(): Promise<number>;\n  readAt(offset: number, length: number): Promise<Uint8Array>;\n  readChunks(params: {\n    startOffset?: number;\n    chunkSize: number;\n  }): AsyncIterable<{ offset: number; bytes: Uint8Array }>;\n};\n\nexport abstract class AbstractFileStorageCodec<TObject> {\n  abstract readAll(params: {\n    fileHandle: FileStorageFileHandle;\n  }): Promise<TObject[]>;\n  // TODO: Add other methods like write, append, read one etc.\n}\n","import { bufferToString } from \"@/utils\";\nimport {\n  AbstractFileStorageCodec,\n  FileStorage,\n  FileStorageFileHandle,\n} from \"./file\";\nimport z from \"zod\";\n\nexport class JSONFileStorage<TObject> extends FileStorage<TObject> {\n  declare codec: JSONFileStorageCodec<TObject>;\n\n  constructor(config: {\n    path: string;\n    chunkSize?: number;\n    schema: z.ZodType<TObject>;\n  }) {\n    super({\n      path: config.path,\n      codec: new JSONFileStorageCodec({\n        chunkSize: config.chunkSize,\n        schema: config.schema,\n      }),\n    });\n  }\n}\n\nexport class JSONFileStorageCodec<\n  TObject,\n> extends AbstractFileStorageCodec<TObject> {\n  private chunkSize: number;\n  private schema: z.ZodType<TObject>;\n\n  constructor(config: { chunkSize?: number; schema: z.ZodType<TObject> }) {\n    super();\n    this.chunkSize = config?.chunkSize ?? 64 * 1024;\n    this.schema = config.schema;\n  }\n\n  async readAll(params: {\n    fileHandle: FileStorageFileHandle;\n  }): Promise<TObject[]> {\n    const wholeFile: Uint8Array = new Uint8Array(\n      await params.fileHandle.size()\n    );\n    for await (const chunk of params.fileHandle.readChunks({\n      chunkSize: this.chunkSize,\n    })) {\n      wholeFile.set(chunk.bytes, chunk.offset);\n    }\n\n    return this.schema.array().parse(JSON.parse(bufferToString(wholeFile)));\n  }\n}\n"],"mappings":";;;;;AAAO,IAAe,kBAAf,MAAwC;AAS/C;;;ACTA,SAAqB,YAAY;AAG1B,IAAM,cAAN,cAAmC,gBAAyB;AAAA,EACvD;AAAA,EACA;AAAA,EACA;AAAA,EAEV,YAAY,QAGT;AACD,UAAM;AACN,SAAK,OAAO,OAAO;AACnB,SAAK,QAAQ,OAAO;AAAA,EACtB;AAAA,EAEA,MAAM,OAAsB;AAC1B,SAAK,aAAa,MAAM,KAAK,KAAK,MAAM,IAAI;AAAA,EAC9C;AAAA,EAEA,MAAe,KACb,MACA,SACyB;AACzB,UAAM,IAAI,MAAM,yBAAyB;AAAA,EAC3C;AAAA,EAEA,MAAe,QAAQ,SAAuC;AAC5D,SAAK,kBAAkB;AAEvB,WAAO,MAAM,KAAK,MAAM,QAAQ;AAAA,MAC9B,YAAY;AAAA,QACV,QAAQ,KAAK,OAAO,KAAK,IAAI;AAAA,QAC7B,YAAY,KAAK,WAAW,KAAK,IAAI;AAAA,QACrC,MAAM,KAAK,KAAK,KAAK,IAAI;AAAA,MAC3B;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAe,MACb,MACA,QACA,SACkB;AAClB,UAAM,IAAI,MAAM,yBAAyB;AAAA,EAC3C;AAAA,EAEA,OAAiB,WAAW,QAG6B;AACvD,SAAK,kBAAkB;AAEvB,UAAM,WAAW,MAAM,KAAK,KAAK;AACjC,QAAI,SAAS,OAAO,eAAe;AAGnC,WAAO,SAAS,UAAU;AACxB,YAAM,SAAS,KAAK,IAAI,OAAO,WAAW,WAAW,MAAM;AAC3D,UAAI,WAAW,EAAG;AAElB,YAAM,QAAQ,MAAM,KAAK,OAAO,QAAQ,MAAM;AAC9C,UAAI,MAAM,WAAW,EAAG;AACxB,YAAM,EAAE,QAAQ,MAAM;AACtB,gBAAU,MAAM;AAAA,IAClB;AAAA,EACF;AAAA,EAEA,MAAgB,OAAwB;AACtC,SAAK,kBAAkB;AAEvB,WAAO,MAAM,KAAK,WAAW,KAAK,EAAE,KAAK,CAAC,SAAS,KAAK,IAAI;AAAA,EAC9D;AAAA,EAEA,MAAgB,OAAO,QAAgB,QAAqC;AAC1E,SAAK,kBAAkB;AAEvB,UAAM,SAAS,IAAI,WAAW,MAAM;AACpC,UAAM,SAAS,MAAM,KAAK,WAAW,KAAK,QAAQ,GAAG,QAAQ,MAAM;AAEnE,WAAO,IAAI,WAAW,OAAO,SAAS,GAAG,OAAO,SAAS,CAAC;AAAA,EAC5D;AAAA,EAEU,oBAER;AACA,QAAI,CAAC,KAAK,YAAY;AACpB,YAAM,IAAI,MAAM,iCAAiC;AAAA,IACnD;AAAA,EACF;AACF;AAWO,IAAe,2BAAf,MAAiD;AAAA;AAKxD;;;ACnGO,IAAM,kBAAN,cAAuC,YAAqB;AAAA,EAGjE,YAAY,QAIT;AACD,UAAM;AAAA,MACJ,MAAM,OAAO;AAAA,MACb,OAAO,IAAI,qBAAqB;AAAA,QAC9B,WAAW,OAAO;AAAA,QAClB,QAAQ,OAAO;AAAA,MACjB,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AACF;AAEO,IAAM,uBAAN,cAEG,yBAAkC;AAAA,EAClC;AAAA,EACA;AAAA,EAER,YAAY,QAA4D;AACtE,UAAM;AACN,SAAK,YAAY,QAAQ,aAAa,KAAK;AAC3C,SAAK,SAAS,OAAO;AAAA,EACvB;AAAA,EAEA,MAAM,QAAQ,QAES;AACrB,UAAM,YAAwB,IAAI;AAAA,MAChC,MAAM,OAAO,WAAW,KAAK;AAAA,IAC/B;AACA,qBAAiB,SAAS,OAAO,WAAW,WAAW;AAAA,MACrD,WAAW,KAAK;AAAA,IAClB,CAAC,GAAG;AACF,gBAAU,IAAI,MAAM,OAAO,MAAM,MAAM;AAAA,IACzC;AAEA,WAAO,KAAK,OAAO,MAAM,EAAE,MAAM,KAAK,MAAM,eAAe,SAAS,CAAC,CAAC;AAAA,EACxE;AACF;","names":[]}

package/dist/chunk-HMQYGCKI.js

@@ -0,0 +1,11 @@
+// src/types/index.ts
+var ScoringMethod = {
+  ai: "ai",
+  human: "human",
+  algo: "algo"
+};
+
+export {
+  ScoringMethod
+};
+//# sourceMappingURL=chunk-HMQYGCKI.js.map

package/dist/chunk-HMQYGCKI.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/types/index.ts"],"sourcesContent":["export * from \"./runner\";\n\nimport { AbstractProvider } from \"@/providers\";\nimport { IdSchema } from \"@/schemas/id\";\nimport { AbstractScorer } from \"@/scorers/abstract\";\nimport { AbstractClassConstructor, ClassConstructor } from \"@/utilities\";\nimport z from \"zod\";\n\nexport type Id = z.infer<typeof IdSchema>;\n\nexport type IdGenerator<TInput = unknown> = (input: TInput) => MaybePromise<Id>;\n\nexport type MaybePromise<T> = T | Promise<T>;\n\nexport const ScoringMethod = {\n  ai: \"ai\",\n  human: \"human\",\n  algo: \"algo\",\n} as const;\nexport type ScoringMethod = (typeof ScoringMethod)[keyof typeof ScoringMethod];\n\nexport type ScorerCtor =\n  | ClassConstructor<AbstractScorer>\n  | AbstractClassConstructor<AbstractScorer>;\nexport type ProviderCtor =\n  | ClassConstructor<AbstractProvider>\n  | AbstractClassConstructor<AbstractProvider>;\n"],"mappings":";AAcO,IAAM,gBAAgB;AAAA,EAC3B,IAAI;AAAA,EACJ,OAAO;AAAA,EACP,MAAM;AACR;","names":[]}

package/dist/chunk-NUEOE3K5.js

@@ -0,0 +1,8 @@
+// src/schemas/id.ts
+import z from "zod";
+var IdSchema = z.string();
+
+export {
+  IdSchema
+};
+//# sourceMappingURL=chunk-NUEOE3K5.js.map

package/dist/chunk-NUEOE3K5.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/schemas/id.ts"],"sourcesContent":["import z from \"zod\";\n\nexport const IdSchema = z.string();\n"],"mappings":";AAAA,OAAO,OAAO;AAEP,IAAM,WAAW,EAAE,OAAO;","names":[]}

package/dist/chunk-OQE6TQXZ.js

@@ -0,0 +1,42 @@
+// src/schemas/schema-definer.ts
+import z from "zod";
+function buildSchemaDefiner(_schema, kindSuffix) {
+  return function(config) {
+    const kind = `${config.kind}.${kindSuffix}`;
+    const schema = config.baseSchema.extend({
+      ...config.fields ?? {},
+      namespace: config.namespace !== void 0 ? z.literal(config.namespace) : config.baseSchema.shape.namespace,
+      kind: config.kind !== void 0 ? z.literal(kind) : config.baseSchema.shape.kind,
+      schemaVersion: config.schemaVersion !== void 0 ? z.literal(config.schemaVersion) : config.baseSchema.shape.schemaVersion
+    });
+    return Object.assign(schema, {
+      new(input) {
+        return schema.parse({
+          ...input,
+          kind,
+          namespace: config.namespace,
+          schemaVersion: config.schemaVersion
+        });
+      },
+      async newWithId(input, generator) {
+        const obj = schema.parse({
+          ...input,
+          id: "",
+          kind,
+          namespace: config.namespace,
+          schemaVersion: config.schemaVersion
+        });
+        const id = await generator(obj);
+        return {
+          ...obj,
+          id
+        };
+      }
+    });
+  };
+}
+
+export {
+  buildSchemaDefiner
+};
+//# sourceMappingURL=chunk-OQE6TQXZ.js.map

package/dist/chunk-OQE6TQXZ.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/schemas/schema-definer.ts"],"sourcesContent":["import { IdGenerator } from \"@/types\";\nimport { WidenZodObject } from \"@/utilities\";\nimport z from \"zod\";\n\nexport function buildSchemaDefiner<\n  TBaseShape extends {\n    kind: z.ZodString | z.ZodLiteral<string>;\n    namespace: z.ZodString | z.ZodLiteral<string>;\n    schemaVersion: z.ZodNumber | z.ZodLiteral<number>;\n  },\n  TKindSuffix extends string = \"\",\n>(_schema: z.ZodObject<TBaseShape>, kindSuffix: TKindSuffix) {\n  return function <\n    TBaseSchema extends WidenZodObject<z.ZodObject<TBaseShape>>,\n    TNamespace extends string | undefined = undefined,\n    TKind extends string | undefined = undefined,\n    TSchemaVersion extends number | undefined = undefined,\n    TFields extends z.ZodRawShape = {},\n  >(config: {\n    baseSchema: TBaseSchema;\n    namespace?: TNamespace;\n    kind?: TKind;\n    schemaVersion?: TSchemaVersion;\n    fields?: TFields;\n  }) {\n    const kind = `${config.kind}.${kindSuffix}` as const;\n    const schema = config.baseSchema.extend({\n      ...((config.fields ?? {}) as TFields),\n      namespace: (config.namespace !== undefined\n        ? z.literal(config.namespace)\n        : config.baseSchema.shape.namespace) as TNamespace extends undefined\n        ? TBaseSchema[\"shape\"][\"namespace\"]\n        : z.ZodLiteral<TNamespace>,\n      kind: (config.kind !== undefined\n        ? z.literal(kind)\n        : config.baseSchema.shape.kind) as TKind extends undefined\n        ? TBaseSchema[\"shape\"][\"kind\"]\n        : z.ZodLiteral<`${TKind}.${TKindSuffix}`>,\n      schemaVersion: (config.schemaVersion !== undefined\n        ? z.literal(config.schemaVersion)\n        : config.baseSchema.shape\n            .schemaVersion) as TSchemaVersion extends undefined\n        ? TBaseSchema[\"shape\"][\"schemaVersion\"]\n        : z.ZodLiteral<TSchemaVersion>,\n    });\n\n    type SchemaType =\n      TBaseSchema extends z.ZodObject<infer U>\n        ? z.ZodObject<\n            Omit<U, \"kind\" | \"schemaVersion\" | \"namespace\"> &\n              TFields & {\n                namespace: TNamespace extends undefined\n                  ? U[\"namespace\"]\n                  : z.ZodLiteral<TNamespace>;\n                kind: TKind extends undefined\n                  ? U[\"kind\"]\n                  : z.ZodLiteral<`${TKind}.${TKindSuffix}`>;\n                schemaVersion: TSchemaVersion extends undefined\n                  ? U[\"schemaVersion\"]\n                  : z.ZodLiteral<TSchemaVersion>;\n              }\n          >\n        : never;\n\n    return Object.assign(schema, {\n      new(\n        input: Omit<\n          z.infer<typeof schema>,\n          \"kind\" | \"schemaVersion\" | \"namespace\"\n        >\n      ) {\n        return schema.parse({\n          ...input,\n          kind,\n          namespace: config.namespace,\n          schemaVersion: config.schemaVersion,\n        });\n      },\n      async newWithId(\n        input: Omit<\n          z.infer<typeof schema>,\n          \"kind\" | \"schemaVersion\" | \"id\" | \"namespace\"\n        >,\n        generator: IdGenerator\n      ) {\n        const obj = schema.parse({\n          ...input,\n          id: \"\",\n          kind,\n          namespace: config.namespace,\n          schemaVersion: config.schemaVersion,\n        });\n        const id = await generator(obj);\n\n        return {\n          ...obj,\n          id,\n        };\n      },\n    }) as unknown as SchemaType & {\n      new: (\n        input: Omit<z.infer<SchemaType>, \"kind\" | \"schemaVersion\" | \"namespace\">\n      ) => z.infer<SchemaType>;\n\n      newWithId(\n        input: Omit<\n          z.infer<SchemaType>,\n          \"id\" | \"kind\" | \"schemaVersion\" | \"namespace\"\n        >,\n        generator: IdGenerator\n      ): Promise<z.infer<SchemaType>>;\n    };\n  };\n}\n"],"mappings":";AAEA,OAAO,OAAO;AAEP,SAAS,mBAOd,SAAkC,YAAyB;AAC3D,SAAO,SAML,QAMC;AACD,UAAM,OAAO,GAAG,OAAO,IAAI,IAAI,UAAU;AACzC,UAAM,SAAS,OAAO,WAAW,OAAO;AAAA,MACtC,GAAK,OAAO,UAAU,CAAC;AAAA,MACvB,WAAY,OAAO,cAAc,SAC7B,EAAE,QAAQ,OAAO,SAAS,IAC1B,OAAO,WAAW,MAAM;AAAA,MAG5B,MAAO,OAAO,SAAS,SACnB,EAAE,QAAQ,IAAI,IACd,OAAO,WAAW,MAAM;AAAA,MAG5B,eAAgB,OAAO,kBAAkB,SACrC,EAAE,QAAQ,OAAO,aAAa,IAC9B,OAAO,WAAW,MACf;AAAA,IAGT,CAAC;AAoBD,WAAO,OAAO,OAAO,QAAQ;AAAA,MAC3B,IACE,OAIA;AACA,eAAO,OAAO,MAAM;AAAA,UAClB,GAAG;AAAA,UACH;AAAA,UACA,WAAW,OAAO;AAAA,UAClB,eAAe,OAAO;AAAA,QACxB,CAAC;AAAA,MACH;AAAA,MACA,MAAM,UACJ,OAIA,WACA;AACA,cAAM,MAAM,OAAO,MAAM;AAAA,UACvB,GAAG;AAAA,UACH,IAAI;AAAA,UACJ;AAAA,UACA,WAAW,OAAO;AAAA,UAClB,eAAe,OAAO;AAAA,QACxB,CAAC;AACD,cAAM,KAAK,MAAM,UAAU,GAAG;AAE9B,eAAO;AAAA,UACL,GAAG;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EAaH;AACF;","names":[]}

package/dist/chunk-PZ5AY32C.js

@@ -0,0 +1,10 @@
+var __defProp = Object.defineProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+
+export {
+  __export
+};
+//# sourceMappingURL=chunk-PZ5AY32C.js.map

package/dist/chunk-PZ5AY32C.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/chunk-Q6GSOHOP.js

@@ -0,0 +1,44 @@
+import {
+  buildSchemaDefiner
+} from "./chunk-OQE6TQXZ.js";
+import {
+  CATEGORIES,
+  PEERBENCH_NAMESPACE
+} from "./chunk-UHHHSYVE.js";
+import {
+  IdSchema
+} from "./chunk-NUEOE3K5.js";
+
+// src/schemas/llm/system-prompt.ts
+import { z } from "zod";
+var BaseSystemPromptSchemaV1 = z.object({
+  id: IdSchema,
+  namespace: z.string(),
+  kind: z.string(),
+  schemaVersion: z.number(),
+  version: z.number(),
+  metadata: z.record(z.string(), z.unknown()).optional()
+});
+var defineSystemPromptSchema = buildSchemaDefiner(
+  BaseSystemPromptSchemaV1,
+  "sys-prompt"
+);
+
+// src/schemas/llm/simple-system-prompt.ts
+import { z as z2 } from "zod";
+var SimpleSystemPromptSchemaV1 = defineSystemPromptSchema({
+  baseSchema: BaseSystemPromptSchemaV1,
+  namespace: PEERBENCH_NAMESPACE,
+  kind: `${CATEGORIES.LLM}/simple`,
+  schemaVersion: 1,
+  fields: {
+    content: z2.string()
+  }
+});
+
+export {
+  BaseSystemPromptSchemaV1,
+  defineSystemPromptSchema,
+  SimpleSystemPromptSchemaV1
+};
+//# sourceMappingURL=chunk-Q6GSOHOP.js.map

package/dist/chunk-Q6GSOHOP.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/schemas/llm/system-prompt.ts","../src/schemas/llm/simple-system-prompt.ts"],"sourcesContent":["import { IdSchema } from \"../id\";\nimport { z } from \"zod\";\nimport { buildSchemaDefiner } from \"../schema-definer\";\n\nexport const BaseSystemPromptSchemaV1 = z.object({\n  id: IdSchema,\n  namespace: z.string(),\n  kind: z.string(),\n  schemaVersion: z.number(),\n  version: z.number(),\n  metadata: z.record(z.string(), z.unknown()).optional(),\n});\nexport type BaseSystemPromptV1 = z.infer<typeof BaseSystemPromptSchemaV1>;\n\nexport const defineSystemPromptSchema = buildSchemaDefiner(\n  BaseSystemPromptSchemaV1,\n  \"sys-prompt\"\n);\n","import { z } from \"zod\";\nimport { defineSystemPromptSchema } from \"./system-prompt\";\nimport { BaseSystemPromptSchemaV1 } from \"./system-prompt\";\nimport { CATEGORIES, PEERBENCH_NAMESPACE } from \"@/constants\";\n\nexport const SimpleSystemPromptSchemaV1 = defineSystemPromptSchema({\n  baseSchema: BaseSystemPromptSchemaV1,\n  namespace: PEERBENCH_NAMESPACE,\n  kind: `${CATEGORIES.LLM}/simple`,\n  schemaVersion: 1,\n  fields: {\n    content: z.string(),\n  },\n});\nexport type SimpleSystemPromptV1 = z.infer<typeof SimpleSystemPromptSchemaV1>;\n"],"mappings":";;;;;;;;;;;;AACA,SAAS,SAAS;AAGX,IAAM,2BAA2B,EAAE,OAAO;AAAA,EAC/C,IAAI;AAAA,EACJ,WAAW,EAAE,OAAO;AAAA,EACpB,MAAM,EAAE,OAAO;AAAA,EACf,eAAe,EAAE,OAAO;AAAA,EACxB,SAAS,EAAE,OAAO;AAAA,EAClB,UAAU,EAAE,OAAO,EAAE,OAAO,GAAG,EAAE,QAAQ,CAAC,EAAE,SAAS;AACvD,CAAC;AAGM,IAAM,2BAA2B;AAAA,EACtC;AAAA,EACA;AACF;;;ACjBA,SAAS,KAAAA,UAAS;AAKX,IAAM,6BAA6B,yBAAyB;AAAA,EACjE,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,MAAM,GAAG,WAAW,GAAG;AAAA,EACvB,eAAe;AAAA,EACf,QAAQ;AAAA,IACN,SAASC,GAAE,OAAO;AAAA,EACpB;AACF,CAAC;","names":["z","z"]}

package/dist/chunk-QY5MPNNB.js

@@ -0,0 +1,28 @@
+import {
+  idGeneratorUUIDv7
+} from "./chunk-4UBK6452.js";
+
+// src/helpers/define-runner.ts
+import z from "zod";
+function defineRunner(config, fn) {
+  return async (params) => {
+    if (config.runConfigSchema && config.parseRunConfig !== false) {
+      z.object(config.runConfigSchema).parse(params.runConfig);
+    }
+    if (params.idGenerators && !params.idGenerators.response) {
+      params.idGenerators.response = config.defaults?.responseIdGenerator ?? idGeneratorUUIDv7;
+    }
+    if (params.idGenerators && !params.idGenerators.score) {
+      params.idGenerators.score = config.defaults?.scoreIdGenerator ?? idGeneratorUUIDv7;
+    }
+    if (params.scorer === void 0) {
+      params.scorer = config.defaults?.scorer ?? void 0;
+    }
+    return await fn(params);
+  };
+}
+
+export {
+  defineRunner
+};
+//# sourceMappingURL=chunk-QY5MPNNB.js.map

package/dist/chunk-QY5MPNNB.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/helpers/define-runner.ts"],"sourcesContent":["import {\n  IdGenerator,\n  InferRunConfig,\n  ProviderCtor,\n  Runner,\n  ScorerCtor,\n} from \"@/types\";\nimport { idGeneratorUUIDv7 } from \"@/utils\";\nimport z from \"zod\";\n\nexport function defineRunner<\n  const TProviders extends ProviderCtor[],\n  const TScorers extends ScorerCtor[],\n  const TSchemaSets extends SchemaSetDefinition[],\n  const TRunConfigSchema extends z.ZodRawShape = {},\n>(\n  config: {\n    schemaSets: TSchemaSets;\n    providers: TProviders;\n    scorers: TScorers;\n    runConfigSchema?: TRunConfigSchema;\n\n    /**\n     * @default true\n     */\n    parseRunConfig?: boolean;\n    defaults?: {\n      scorer?: InstanceType<TScorers[number]>;\n      responseIdGenerator?: IdGenerator;\n      scoreIdGenerator?: IdGenerator;\n    };\n  },\n  fn: Runner<\n    TSchemaSets[number][\"testCase\"],\n    TSchemaSets[number][\"response\"],\n    TSchemaSets[number][\"score\"],\n    InstanceType<TProviders[number]>,\n    InstanceType<TScorers[number]>,\n    InferRunConfig<TRunConfigSchema>\n  >\n) {\n  return async (params: Parameters<typeof fn>[0]) => {\n    if (config.runConfigSchema && config.parseRunConfig !== false) {\n      z.object(config.runConfigSchema).parse(params.runConfig);\n    }\n\n    if (params.idGenerators && !params.idGenerators.response) {\n      params.idGenerators.response =\n        config.defaults?.responseIdGenerator ?? idGeneratorUUIDv7;\n    }\n\n    if (params.idGenerators && !params.idGenerators.score) {\n      params.idGenerators.score =\n        config.defaults?.scoreIdGenerator ?? idGeneratorUUIDv7;\n    }\n\n    if (params.scorer === undefined) {\n      params.scorer = config.defaults?.scorer ?? undefined;\n    }\n\n    return await fn(params);\n  };\n}\n\ntype SchemaSetDefinition<\n  TTestCase extends z.ZodObject = z.ZodObject,\n  TResponse extends z.ZodObject = z.ZodObject,\n  TScore extends z.ZodObject = z.ZodObject,\n> = {\n  testCase: TTestCase;\n  response: TResponse;\n  score: TScore;\n};\n"],"mappings":";;;;;AAQA,OAAO,OAAO;AAEP,SAAS,aAMd,QAgBA,IAQA;AACA,SAAO,OAAO,WAAqC;AACjD,QAAI,OAAO,mBAAmB,OAAO,mBAAmB,OAAO;AAC7D,QAAE,OAAO,OAAO,eAAe,EAAE,MAAM,OAAO,SAAS;AAAA,IACzD;AAEA,QAAI,OAAO,gBAAgB,CAAC,OAAO,aAAa,UAAU;AACxD,aAAO,aAAa,WAClB,OAAO,UAAU,uBAAuB;AAAA,IAC5C;AAEA,QAAI,OAAO,gBAAgB,CAAC,OAAO,aAAa,OAAO;AACrD,aAAO,aAAa,QAClB,OAAO,UAAU,oBAAoB;AAAA,IACzC;AAEA,QAAI,OAAO,WAAW,QAAW;AAC/B,aAAO,SAAS,OAAO,UAAU,UAAU;AAAA,IAC7C;AAEA,WAAO,MAAM,GAAG,MAAM;AAAA,EACxB;AACF;","names":[]}

package/dist/chunk-R76XA2K6.js

@@ -0,0 +1,229 @@
+import {
+  PEERBENCH_NAMESPACE
+} from "./chunk-UHHHSYVE.js";
+import {
+  RateLimiter
+} from "./chunk-4UBK6452.js";
+
+// src/providers/abstract/provider.ts
+var AbstractProvider = class {
+};
+
+// src/providers/abstract/llm.ts
+var AbstractLLMProvider = class extends AbstractProvider {
+};
+
+// src/providers/mastra.ts
+import { MastraClient } from "@mastra/client-js";
+var MastraProvider = class extends AbstractLLMProvider {
+  kind = "mastra";
+  endpoint;
+  authToken;
+  client;
+  warnedAboutSystemMessages = false;
+  warnedAboutResponseFormat = false;
+  constructor(params) {
+    super();
+    this.endpoint = params.endpoint;
+    this.authToken = params.authToken;
+    this.client = new MastraClient({
+      baseUrl: this.endpoint,
+      headers: {
+        Authorization: `Bearer ${this.authToken}`
+      }
+    });
+  }
+  async forward(args) {
+    const apiMessages = args.messages.filter((m) => m.role === "user" || m.role === "assistant").map((m) => ({
+      role: m.role,
+      content: String(m.content ?? "")
+    }));
+    const agent = this.client.getAgent(args.model);
+    const startedAt = Date.now();
+    const response = await agent.generate(
+      {
+        messages: apiMessages,
+        runtimeContext: {
+          "model-id": args.model
+        }
+      },
+      { memory: args.memory }
+    );
+    return {
+      data: response.text,
+      startedAt,
+      completedAt: Date.now()
+    };
+  }
+};
+
+// src/providers/openai.ts
+import OpenAI, { APIError } from "openai";
+var OpenAIProvider = class extends AbstractLLMProvider {
+  kind = `${PEERBENCH_NAMESPACE}/llm/openai`;
+  client;
+  rateLimiter;
+  maxRetries;
+  constructor(config) {
+    super();
+    this.maxRetries = config.maxRetries ?? 3;
+    this.rateLimiter = config.rateLimiter ?? new RateLimiter({
+      maxWeight: 20,
+      timeWindow: 3e3
+    });
+    this.client = new OpenAI({
+      baseURL: config.baseURL,
+      apiKey: config.apiKey,
+      timeout: config.timeout,
+      dangerouslyAllowBrowser: true
+    });
+  }
+  async forward(args) {
+    let retryCount = this.maxRetries;
+    while (retryCount > 0) {
+      let startedAt = /* @__PURE__ */ new Date();
+      try {
+        const response = await this.rateLimiter.execute(
+          async () => {
+            startedAt = /* @__PURE__ */ new Date();
+            return await this.client.chat.completions.create(
+              {
+                model: args.model,
+                messages: args.messages,
+                temperature: args.temperature,
+                response_format: args.responseFormat
+              },
+              // Signal for request
+              { signal: args.abortSignal }
+            );
+          },
+          // Signal for rate limiting
+          { signal: args.abortSignal }
+        );
+        if ("error" in response) {
+          const err = response.error;
+          throw new Error(
+            `${err.message} - Code ${err.code} - ${JSON.stringify(err)}`
+          );
+        }
+        if (!response?.choices?.[0]?.message?.content) {
+          throw new Error("No content returned from the model");
+        }
+        return {
+          data: response.choices[0].message.content,
+          inputTokensUsed: response?.usage?.prompt_tokens,
+          outputTokensUsed: response?.usage?.completion_tokens,
+          startedAt: startedAt.getTime(),
+          completedAt: Date.now()
+        };
+      } catch (err) {
+        if (err instanceof APIError && err.status === 401) {
+          throw new Error(`Invalid credentials provided`, { cause: err });
+        }
+        retryCount--;
+        if (err instanceof SyntaxError) {
+          console.debug(err);
+          continue;
+        }
+        if (retryCount !== 0) {
+          continue;
+        }
+        throw new Error(
+          `Failed to forward prompt to the model: ${err instanceof Error ? err.message : err}`,
+          { cause: err }
+        );
+      }
+    }
+    throw new Error(
+      `Failed to forward prompt to the model: Max retries reached`,
+      { cause: new Error("Max retries reached") }
+    );
+  }
+};
+
+// src/providers/openrouter.ts
+import Decimal from "decimal.js";
+import axios from "axios";
+var baseURL = "https://openrouter.ai/api/v1";
+var MODELS_CACHE_TTL = 1e3 * 60 * 60 * 24;
+var OpenRouterProvider = class extends AbstractLLMProvider {
+  kind = `${PEERBENCH_NAMESPACE}/llm/openrouter.ai`;
+  models = void 0;
+  modelsCachePromise = Promise.resolve(void 0);
+  modelsUpdatedAt = 0;
+  openAIProvider;
+  constructor(config) {
+    super();
+    this.openAIProvider = new OpenAIProvider({
+      baseURL,
+      apiKey: config.apiKey,
+      maxRetries: config.maxRetries,
+      timeout: config.timeout,
+      rateLimiter: config.rateLimiter
+    });
+  }
+  async forward(args) {
+    const [response] = await Promise.all([
+      this.openAIProvider.forward(args),
+      this.updateModelsCache().catch(() => {
+      })
+    ]);
+    const modelInfo = this.models?.data.find((m) => m.id === args.model);
+    let inputCost = void 0;
+    let outputCost = void 0;
+    if (modelInfo !== void 0) {
+      if (response.inputTokensUsed !== void 0) {
+        inputCost = new Decimal(modelInfo.pricing.prompt).mul(response.inputTokensUsed).toFixed(10);
+      }
+      if (response.outputTokensUsed !== void 0) {
+        outputCost = new Decimal(modelInfo.pricing.completion).mul(response.outputTokensUsed).toFixed(10);
+      }
+    }
+    return {
+      ...response,
+      inputCost,
+      outputCost
+    };
+  }
+  /**
+   * Updates the cache that holds information about OpenRouter models
+   * including pricing information. It will be valid for 24 hours as
+   * long as the instance of this Provider object is alive.
+   */
+  async updateModelsCache() {
+    this.modelsCachePromise = this.modelsCachePromise.then(async () => {
+      if (
+        // The data presented in the cache
+        this.models !== void 0 && // The cache is still valid
+        Date.now() - this.modelsUpdatedAt < MODELS_CACHE_TTL
+      ) {
+        return this.models;
+      }
+      return axios.get(`${baseURL}/models`).then((res) => res.data).then((data) => {
+        data = {
+          data: data.data.filter(
+            (m) => m.architecture.input_modalities.includes("text") && m.architecture.output_modalities.includes("text") && // These models are "fast apply model" and don't support multi turn conversations so don't include them
+            ![
+              "morph/morph-v3-large",
+              "morph/morph-v3-fast",
+              "relace/relace-apply-3"
+            ].includes(m.id)
+          )
+        };
+        this.models = data;
+        this.modelsUpdatedAt = Date.now();
+        return data;
+      });
+    }).catch(() => void 0);
+    await this.modelsCachePromise;
+  }
+};
+
+export {
+  AbstractProvider,
+  AbstractLLMProvider,
+  MastraProvider,
+  OpenAIProvider,
+  OpenRouterProvider
+};
+//# sourceMappingURL=chunk-R76XA2K6.js.map

package/dist/chunk-R76XA2K6.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/providers/abstract/provider.ts","../src/providers/abstract/llm.ts","../src/providers/mastra.ts","../src/providers/openai.ts","../src/providers/openrouter.ts"],"sourcesContent":["export abstract class AbstractProvider {\n  abstract readonly kind: string;\n}\n\nexport type ProviderResponse<TData = unknown> = {\n  startedAt: number;\n  completedAt: number;\n  data: TData;\n};\n","import { AbstractProvider, ProviderResponse } from \"./provider\";\nimport {\n  ResponseFormatJSONObject,\n  ResponseFormatJSONSchema,\n  ResponseFormatText,\n} from \"openai/resources/shared\";\nimport { ChatCompletionMessageParam } from \"openai/resources/chat/completions\";\n\nexport abstract class AbstractLLMProvider extends AbstractProvider {\n  abstract forward(args: LLMProviderForwardArgs): Promise<ChatResponse>;\n}\n\nexport type LLMProviderForwardArgs = {\n  messages: ChatCompletionMessageParam[];\n  model: string;\n  abortSignal?: AbortSignal;\n  temperature?: number;\n  responseFormat?:\n    | ResponseFormatText\n    | ResponseFormatJSONSchema\n    | ResponseFormatJSONObject;\n};\n\nexport type ChatResponse = ProviderResponse<string> & {\n  inputTokensUsed?: number;\n  outputTokensUsed?: number;\n  inputCost?: string;\n  outputCost?: string;\n};\n","import {\n  AbstractLLMProvider,\n  type ChatResponse,\n  type LLMProviderForwardArgs,\n} from \"./abstract/llm\";\nimport { MastraClient } from \"@mastra/client-js\";\n\nexport class MastraProvider extends AbstractLLMProvider {\n  override readonly kind = \"mastra\";\n\n  private readonly endpoint: string;\n  private readonly authToken?: string;\n  private client: MastraClient;\n  private warnedAboutSystemMessages = false;\n  private warnedAboutResponseFormat = false;\n\n  constructor(params: { endpoint: string; authToken?: string }) {\n    super();\n    this.endpoint = params.endpoint;\n    this.authToken = params.authToken;\n    this.client = new MastraClient({\n      baseUrl: this.endpoint,\n      headers: {\n        Authorization: `Bearer ${this.authToken}`,\n      },\n    });\n  }\n\n  override async forward(\n    args: LLMProviderForwardArgs & {\n      memory?: AgentMemoryOption;\n    }\n  ): Promise<ChatResponse> {\n    const apiMessages = args.messages\n      .filter((m) => m.role === \"user\" || m.role === \"assistant\")\n      .map((m) => ({\n        role: m.role as \"user\" | \"assistant\",\n        content: String((m as any).content ?? \"\"),\n      }));\n\n    const agent = this.client.getAgent(args.model);\n\n    const startedAt = Date.now();\n    const response = await agent.generate(\n      {\n        messages: apiMessages,\n        runtimeContext: {\n          \"model-id\": args.model,\n        },\n      },\n      { memory: args.memory }\n    );\n\n    return {\n      data: response.text,\n      startedAt,\n      completedAt: Date.now(),\n    };\n  }\n}\n\n// NOTE: Mastra client does not export this type\nexport type AgentMemoryOption = Parameters<\n  Parameters<MastraClient[\"getAgent\"]>[\"0\"] extends string\n    ? ReturnType<MastraClient[\"getAgent\"]>[\"generate\"]\n    : never\n>[0] extends { memory?: infer M }\n  ? M\n  : never;\n","import { RateLimiter } from \"@/utils\";\nimport { ChatCompletionMessageParam } from \"openai/resources/chat/completions\";\nimport {\n  ResponseFormatJSONObject,\n  ResponseFormatJSONSchema,\n  ResponseFormatText,\n} from \"openai/resources/shared\";\nimport OpenAI, { APIError } from \"openai\";\nimport { AbstractLLMProvider, ChatResponse } from \"./abstract/llm\";\nimport { PEERBENCH_NAMESPACE } from \"@/constants\";\n\nexport class OpenAIProvider extends AbstractLLMProvider {\n  override readonly kind = `${PEERBENCH_NAMESPACE}/llm/openai` as const;\n\n  private client: OpenAI;\n  private rateLimiter: RateLimiter;\n  private maxRetries: number;\n\n  constructor(config: {\n    apiKey: string;\n    baseURL: string;\n    maxRetries?: number;\n    timeout?: number;\n    rateLimiter?: RateLimiter;\n  }) {\n    super();\n    this.maxRetries = config.maxRetries ?? 3;\n    this.rateLimiter =\n      config.rateLimiter ??\n      new RateLimiter({\n        maxWeight: 20,\n        timeWindow: 3_000,\n      });\n\n    this.client = new OpenAI({\n      baseURL: config.baseURL,\n      apiKey: config.apiKey,\n      timeout: config.timeout,\n      dangerouslyAllowBrowser: true,\n    });\n  }\n\n  async forward(args: {\n    messages: ChatCompletionMessageParam[];\n    model: string;\n    abortSignal?: AbortSignal;\n    temperature?: number;\n    responseFormat?:\n      | ResponseFormatText\n      | ResponseFormatJSONSchema\n      | ResponseFormatJSONObject;\n  }): Promise<ChatResponse> {\n    let retryCount = this.maxRetries;\n    while (retryCount > 0) {\n      let startedAt: Date = new Date();\n\n      try {\n        const response = await this.rateLimiter.execute(\n          async () => {\n            // Capture the start time of the request\n            startedAt = new Date();\n            return await this.client.chat.completions.create(\n              {\n                model: args.model,\n                messages: args.messages,\n                temperature: args.temperature,\n                response_format: args.responseFormat,\n              },\n              // Signal for request\n              { signal: args.abortSignal }\n            );\n          },\n          // Signal for rate limiting\n          { signal: args.abortSignal }\n        );\n\n        if (\"error\" in response) {\n          const err = response.error as any;\n          throw new Error(\n            `${err.message} - Code ${err.code} - ${JSON.stringify(err)}`\n          );\n        }\n\n        if (!response?.choices?.[0]?.message?.content) {\n          throw new Error(\"No content returned from the model\");\n        }\n\n        return {\n          data: response.choices[0].message.content,\n\n          inputTokensUsed: response?.usage?.prompt_tokens,\n          outputTokensUsed: response?.usage?.completion_tokens,\n\n          startedAt: startedAt.getTime(),\n          completedAt: Date.now(),\n        };\n      } catch (err) {\n        if (err instanceof APIError && err.status === 401) {\n          throw new Error(`Invalid credentials provided`, { cause: err });\n        }\n\n        retryCount--;\n\n        // More likely an empty HTTP response returned by the Provider\n        // and it couldn't be parsed as JSON by the OpenAI SDK. We need to retry the request\n        // More info can be found in the following links:\n        // https://www.reddit.com/r/SillyTavernAI/comments/1ik95vr/deepseek_r1_on_openrouter_returning_blank_messages/\n        // https://github.com/cline/cline/issues/60\n        if (err instanceof SyntaxError) {\n          console.debug(err);\n          continue;\n        }\n\n        // If it was another error, just continue until we run out of retries\n        if (retryCount !== 0) {\n          continue;\n        }\n\n        throw new Error(\n          `Failed to forward prompt to the model: ${err instanceof Error ? err.message : err}`,\n          { cause: err }\n        );\n      }\n    }\n\n    throw new Error(\n      `Failed to forward prompt to the model: Max retries reached`,\n      { cause: new Error(\"Max retries reached\") }\n    );\n  }\n}\n","import {\n  AbstractLLMProvider,\n  ChatResponse,\n  LLMProviderForwardArgs,\n} from \"./abstract/llm\";\nimport { RateLimiter } from \"@/utils\";\nimport { OpenAIProvider } from \"./openai\";\nimport { PEERBENCH_NAMESPACE } from \"@/constants\";\nimport Decimal from \"decimal.js\";\nimport axios from \"axios\";\n\nconst baseURL = \"https://openrouter.ai/api/v1\";\nconst MODELS_CACHE_TTL = 1000 * 60 * 60 * 24; // 24 hours\n\nexport class OpenRouterProvider extends AbstractLLMProvider {\n  override readonly kind = `${PEERBENCH_NAMESPACE}/llm/openrouter.ai` as const;\n\n  private models: ModelsResponse | undefined = undefined;\n  private modelsCachePromise: Promise<ModelsResponse | undefined> =\n    Promise.resolve(undefined);\n  private modelsUpdatedAt = 0;\n  private openAIProvider: OpenAIProvider;\n\n  constructor(config: {\n    apiKey: string;\n    maxRetries?: number;\n    timeout?: number;\n    rateLimiter?: RateLimiter;\n  }) {\n    super();\n    this.openAIProvider = new OpenAIProvider({\n      baseURL,\n      apiKey: config.apiKey,\n      maxRetries: config.maxRetries,\n      timeout: config.timeout,\n      rateLimiter: config.rateLimiter,\n    });\n  }\n\n  override async forward(args: LLMProviderForwardArgs): Promise<ChatResponse> {\n    // Update models cache concurrently (non-blocking)\n    const [response] = await Promise.all([\n      this.openAIProvider.forward(args),\n      this.updateModelsCache().catch(() => {\n        // Silently fail if cache update fails so we won't have cost info in the result\n      }),\n    ]);\n\n    // Get the model info from the cache\n    const modelInfo = this.models?.data.find((m) => m.id === args.model);\n    let inputCost: string | undefined = undefined;\n    let outputCost: string | undefined = undefined;\n\n    if (modelInfo !== undefined) {\n      // Use Decimal.js for more accurate calculation\n      if (response.inputTokensUsed !== undefined) {\n        inputCost = new Decimal(modelInfo.pricing.prompt)\n          .mul(response.inputTokensUsed)\n          .toFixed(10);\n      }\n      if (response.outputTokensUsed !== undefined) {\n        outputCost = new Decimal(modelInfo.pricing.completion)\n          .mul(response.outputTokensUsed)\n          .toFixed(10);\n      }\n    }\n\n    return {\n      ...response,\n      inputCost,\n      outputCost,\n    };\n  }\n\n  /**\n   * Updates the cache that holds information about OpenRouter models\n   * including pricing information. It will be valid for 24 hours as\n   * long as the instance of this Provider object is alive.\n   */\n  private async updateModelsCache() {\n    // Chain each update method call to the promise.\n    // This approach prevents race conditions between multiple calls.\n    // Since each call is chained to the end of the previous one,\n    // each promise makes a request only if the models cache is not updated\n    // in the last call. Otherwise it simply resolves to the cached value.\n    this.modelsCachePromise = this.modelsCachePromise\n      .then(async () => {\n        if (\n          // The data presented in the cache\n          this.models !== undefined &&\n          // The cache is still valid\n          Date.now() - this.modelsUpdatedAt < MODELS_CACHE_TTL\n        ) {\n          return this.models;\n        }\n\n        // If the cache is not valid, update it\n        return axios\n          .get<ModelsResponse>(`${baseURL}/models`)\n          .then((res) => res.data)\n          .then((data) => {\n            // Only get the models that supports text input and output\n            data = {\n              data: data.data.filter(\n                (m) =>\n                  m.architecture.input_modalities.includes(\"text\") &&\n                  m.architecture.output_modalities.includes(\"text\") &&\n                  // These models are \"fast apply model\" and don't support multi turn conversations so don't include them\n                  ![\n                    \"morph/morph-v3-large\",\n                    \"morph/morph-v3-fast\",\n                    \"relace/relace-apply-3\",\n                  ].includes(m.id)\n              ),\n            };\n\n            this.models = data;\n            this.modelsUpdatedAt = Date.now();\n\n            return data;\n          });\n      })\n      .catch(() => undefined);\n\n    // Wait for the promise chain to resolve\n    await this.modelsCachePromise;\n  }\n}\n\ntype PutModality = \"text\" | \"image\" | \"file\" | \"audio\";\ntype Modality = \"text->text\" | \"text+image->text\" | \"text+image->text+image\";\ntype ModelsResponse = {\n  data: {\n    readonly id: string;\n    readonly canonical_slug: string;\n    readonly hugging_face_id: null | string;\n    readonly name: string;\n    readonly created: number;\n    readonly description: string;\n    readonly context_length: number;\n    readonly architecture: {\n      readonly modality: Modality;\n      readonly input_modalities: PutModality[];\n      readonly output_modalities: PutModality[];\n      readonly instruct_type: null | string;\n    };\n    readonly pricing: {\n      readonly prompt: string;\n      readonly completion: string;\n      readonly request?: string;\n      readonly image?: string;\n      readonly web_search?: string;\n      readonly internal_reasoning?: string;\n      readonly input_cache_read?: string;\n      readonly input_cache_write?: string;\n      readonly audio?: string;\n    };\n  }[];\n};\n"],"mappings":";;;;;;;;AAAO,IAAe,mBAAf,MAAgC;AAEvC;;;ACMO,IAAe,sBAAf,cAA2C,iBAAiB;AAEnE;;;ACLA,SAAS,oBAAoB;AAEtB,IAAM,iBAAN,cAA6B,oBAAoB;AAAA,EACpC,OAAO;AAAA,EAER;AAAA,EACA;AAAA,EACT;AAAA,EACA,4BAA4B;AAAA,EAC5B,4BAA4B;AAAA,EAEpC,YAAY,QAAkD;AAC5D,UAAM;AACN,SAAK,WAAW,OAAO;AACvB,SAAK,YAAY,OAAO;AACxB,SAAK,SAAS,IAAI,aAAa;AAAA,MAC7B,SAAS,KAAK;AAAA,MACd,SAAS;AAAA,QACP,eAAe,UAAU,KAAK,SAAS;AAAA,MACzC;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAe,QACb,MAGuB;AACvB,UAAM,cAAc,KAAK,SACtB,OAAO,CAAC,MAAM,EAAE,SAAS,UAAU,EAAE,SAAS,WAAW,EACzD,IAAI,CAAC,OAAO;AAAA,MACX,MAAM,EAAE;AAAA,MACR,SAAS,OAAQ,EAAU,WAAW,EAAE;AAAA,IAC1C,EAAE;AAEJ,UAAM,QAAQ,KAAK,OAAO,SAAS,KAAK,KAAK;AAE7C,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,WAAW,MAAM,MAAM;AAAA,MAC3B;AAAA,QACE,UAAU;AAAA,QACV,gBAAgB;AAAA,UACd,YAAY,KAAK;AAAA,QACnB;AAAA,MACF;AAAA,MACA,EAAE,QAAQ,KAAK,OAAO;AAAA,IACxB;AAEA,WAAO;AAAA,MACL,MAAM,SAAS;AAAA,MACf;AAAA,MACA,aAAa,KAAK,IAAI;AAAA,IACxB;AAAA,EACF;AACF;;;ACpDA,OAAO,UAAU,gBAAgB;AAI1B,IAAM,iBAAN,cAA6B,oBAAoB;AAAA,EACpC,OAAO,GAAG,mBAAmB;AAAA,EAEvC;AAAA,EACA;AAAA,EACA;AAAA,EAER,YAAY,QAMT;AACD,UAAM;AACN,SAAK,aAAa,OAAO,cAAc;AACvC,SAAK,cACH,OAAO,eACP,IAAI,YAAY;AAAA,MACd,WAAW;AAAA,MACX,YAAY;AAAA,IACd,CAAC;AAEH,SAAK,SAAS,IAAI,OAAO;AAAA,MACvB,SAAS,OAAO;AAAA,MAChB,QAAQ,OAAO;AAAA,MACf,SAAS,OAAO;AAAA,MAChB,yBAAyB;AAAA,IAC3B,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,QAAQ,MASY;AACxB,QAAI,aAAa,KAAK;AACtB,WAAO,aAAa,GAAG;AACrB,UAAI,YAAkB,oBAAI,KAAK;AAE/B,UAAI;AACF,cAAM,WAAW,MAAM,KAAK,YAAY;AAAA,UACtC,YAAY;AAEV,wBAAY,oBAAI,KAAK;AACrB,mBAAO,MAAM,KAAK,OAAO,KAAK,YAAY;AAAA,cACxC;AAAA,gBACE,OAAO,KAAK;AAAA,gBACZ,UAAU,KAAK;AAAA,gBACf,aAAa,KAAK;AAAA,gBAClB,iBAAiB,KAAK;AAAA,cACxB;AAAA;AAAA,cAEA,EAAE,QAAQ,KAAK,YAAY;AAAA,YAC7B;AAAA,UACF;AAAA;AAAA,UAEA,EAAE,QAAQ,KAAK,YAAY;AAAA,QAC7B;AAEA,YAAI,WAAW,UAAU;AACvB,gBAAM,MAAM,SAAS;AACrB,gBAAM,IAAI;AAAA,YACR,GAAG,IAAI,OAAO,WAAW,IAAI,IAAI,MAAM,KAAK,UAAU,GAAG,CAAC;AAAA,UAC5D;AAAA,QACF;AAEA,YAAI,CAAC,UAAU,UAAU,CAAC,GAAG,SAAS,SAAS;AAC7C,gBAAM,IAAI,MAAM,oCAAoC;AAAA,QACtD;AAEA,eAAO;AAAA,UACL,MAAM,SAAS,QAAQ,CAAC,EAAE,QAAQ;AAAA,UAElC,iBAAiB,UAAU,OAAO;AAAA,UAClC,kBAAkB,UAAU,OAAO;AAAA,UAEnC,WAAW,UAAU,QAAQ;AAAA,UAC7B,aAAa,KAAK,IAAI;AAAA,QACxB;AAAA,MACF,SAAS,KAAK;AACZ,YAAI,eAAe,YAAY,IAAI,WAAW,KAAK;AACjD,gBAAM,IAAI,MAAM,gCAAgC,EAAE,OAAO,IAAI,CAAC;AAAA,QAChE;AAEA;AAOA,YAAI,eAAe,aAAa;AAC9B,kBAAQ,MAAM,GAAG;AACjB;AAAA,QACF;AAGA,YAAI,eAAe,GAAG;AACpB;AAAA,QACF;AAEA,cAAM,IAAI;AAAA,UACR,0CAA0C,eAAe,QAAQ,IAAI,UAAU,GAAG;AAAA,UAClF,EAAE,OAAO,IAAI;AAAA,QACf;AAAA,MACF;AAAA,IACF;AAEA,UAAM,IAAI;AAAA,MACR;AAAA,MACA,EAAE,OAAO,IAAI,MAAM,qBAAqB,EAAE;AAAA,IAC5C;AAAA,EACF;AACF;;;AC1HA,OAAO,aAAa;AACpB,OAAO,WAAW;AAElB,IAAM,UAAU;AAChB,IAAM,mBAAmB,MAAO,KAAK,KAAK;AAEnC,IAAM,qBAAN,cAAiC,oBAAoB;AAAA,EACxC,OAAO,GAAG,mBAAmB;AAAA,EAEvC,SAAqC;AAAA,EACrC,qBACN,QAAQ,QAAQ,MAAS;AAAA,EACnB,kBAAkB;AAAA,EAClB;AAAA,EAER,YAAY,QAKT;AACD,UAAM;AACN,SAAK,iBAAiB,IAAI,eAAe;AAAA,MACvC;AAAA,MACA,QAAQ,OAAO;AAAA,MACf,YAAY,OAAO;AAAA,MACnB,SAAS,OAAO;AAAA,MAChB,aAAa,OAAO;AAAA,IACtB,CAAC;AAAA,EACH;AAAA,EAEA,MAAe,QAAQ,MAAqD;AAE1E,UAAM,CAAC,QAAQ,IAAI,MAAM,QAAQ,IAAI;AAAA,MACnC,KAAK,eAAe,QAAQ,IAAI;AAAA,MAChC,KAAK,kBAAkB,EAAE,MAAM,MAAM;AAAA,MAErC,CAAC;AAAA,IACH,CAAC;AAGD,UAAM,YAAY,KAAK,QAAQ,KAAK,KAAK,CAAC,MAAM,EAAE,OAAO,KAAK,KAAK;AACnE,QAAI,YAAgC;AACpC,QAAI,aAAiC;AAErC,QAAI,cAAc,QAAW;AAE3B,UAAI,SAAS,oBAAoB,QAAW;AAC1C,oBAAY,IAAI,QAAQ,UAAU,QAAQ,MAAM,EAC7C,IAAI,SAAS,eAAe,EAC5B,QAAQ,EAAE;AAAA,MACf;AACA,UAAI,SAAS,qBAAqB,QAAW;AAC3C,qBAAa,IAAI,QAAQ,UAAU,QAAQ,UAAU,EAClD,IAAI,SAAS,gBAAgB,EAC7B,QAAQ,EAAE;AAAA,MACf;AAAA,IACF;AAEA,WAAO;AAAA,MACL,GAAG;AAAA,MACH;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAc,oBAAoB;AAMhC,SAAK,qBAAqB,KAAK,mBAC5B,KAAK,YAAY;AAChB;AAAA;AAAA,QAEE,KAAK,WAAW;AAAA,QAEhB,KAAK,IAAI,IAAI,KAAK,kBAAkB;AAAA,QACpC;AACA,eAAO,KAAK;AAAA,MACd;AAGA,aAAO,MACJ,IAAoB,GAAG,OAAO,SAAS,EACvC,KAAK,CAAC,QAAQ,IAAI,IAAI,EACtB,KAAK,CAAC,SAAS;AAEd,eAAO;AAAA,UACL,MAAM,KAAK,KAAK;AAAA,YACd,CAAC,MACC,EAAE,aAAa,iBAAiB,SAAS,MAAM,KAC/C,EAAE,aAAa,kBAAkB,SAAS,MAAM;AAAA,YAEhD,CAAC;AAAA,cACC;AAAA,cACA;AAAA,cACA;AAAA,YACF,EAAE,SAAS,EAAE,EAAE;AAAA,UACnB;AAAA,QACF;AAEA,aAAK,SAAS;AACd,aAAK,kBAAkB,KAAK,IAAI;AAEhC,eAAO;AAAA,MACT,CAAC;AAAA,IACL,CAAC,EACA,MAAM,MAAM,MAAS;AAGxB,UAAM,KAAK;AAAA,EACb;AACF;","names":[]}

package/dist/chunk-TRNCF2BG.js

@@ -0,0 +1,35 @@
+import {
+  IdSchema
+} from "./chunk-NUEOE3K5.js";
+
+// src/schemas/extensions/response/llm.ts
+import z from "zod";
+var ExtensionLLMResponseFieldsV1 = {
+  data: z.string(),
+  modelSlug: z.string(),
+  provider: z.string(),
+  systemPromptId: IdSchema.optional(),
+  inputTokensUsed: z.number().optional(),
+  outputTokensUsed: z.number().optional(),
+  inputCost: z.string().optional(),
+  outputCost: z.string().optional()
+};
+
+// src/schemas/extensions/score/llm-as-a-judge-scorer.ts
+import z2 from "zod";
+var ExtensionLLMAsAJudgeScoreFieldsV1 = {
+  scorerAISystemPrompt: z2.string().optional(),
+  scorerAISystemPromptId: z2.string().optional(),
+  scorerAIProvider: z2.string().optional(),
+  scorerAIModelSlug: z2.string().optional(),
+  scorerAIInputTokensUsed: z2.number().optional(),
+  scorerAIOutputTokensUsed: z2.number().optional(),
+  scorerAIInputCost: z2.string().optional(),
+  scorerAIOutputCost: z2.string().optional()
+};
+
+export {
+  ExtensionLLMResponseFieldsV1,
+  ExtensionLLMAsAJudgeScoreFieldsV1
+};
+//# sourceMappingURL=chunk-TRNCF2BG.js.map

package/dist/chunk-TRNCF2BG.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/schemas/extensions/response/llm.ts","../src/schemas/extensions/score/llm-as-a-judge-scorer.ts"],"sourcesContent":["import { IdSchema } from \"@/schemas/id\";\nimport z from \"zod\";\n\n/**\n * Provides a set of fields that holds information about the LLM and its response.\n */\nexport const ExtensionLLMResponseFieldsV1 = {\n  data: z.string(),\n  modelSlug: z.string(),\n  provider: z.string(),\n  systemPromptId: IdSchema.optional(),\n\n  inputTokensUsed: z.number().optional(),\n  outputTokensUsed: z.number().optional(),\n  inputCost: z.string().optional(),\n  outputCost: z.string().optional(),\n};\n","import z from \"zod\";\n\n/**\n * Provides a set of fields that holds information about the LLM model\n * that was used to judge the response.\n */\nexport const ExtensionLLMAsAJudgeScoreFieldsV1 = {\n  scorerAISystemPrompt: z.string().optional(),\n  scorerAISystemPromptId: z.string().optional(),\n  scorerAIProvider: z.string().optional(),\n  scorerAIModelSlug: z.string().optional(),\n  scorerAIInputTokensUsed: z.number().optional(),\n  scorerAIOutputTokensUsed: z.number().optional(),\n  scorerAIInputCost: z.string().optional(),\n  scorerAIOutputCost: z.string().optional(),\n};\n"],"mappings":";;;;;AACA,OAAO,OAAO;AAKP,IAAM,+BAA+B;AAAA,EAC1C,MAAM,EAAE,OAAO;AAAA,EACf,WAAW,EAAE,OAAO;AAAA,EACpB,UAAU,EAAE,OAAO;AAAA,EACnB,gBAAgB,SAAS,SAAS;AAAA,EAElC,iBAAiB,EAAE,OAAO,EAAE,SAAS;AAAA,EACrC,kBAAkB,EAAE,OAAO,EAAE,SAAS;AAAA,EACtC,WAAW,EAAE,OAAO,EAAE,SAAS;AAAA,EAC/B,YAAY,EAAE,OAAO,EAAE,SAAS;AAClC;;;AChBA,OAAOA,QAAO;AAMP,IAAM,oCAAoC;AAAA,EAC/C,sBAAsBA,GAAE,OAAO,EAAE,SAAS;AAAA,EAC1C,wBAAwBA,GAAE,OAAO,EAAE,SAAS;AAAA,EAC5C,kBAAkBA,GAAE,OAAO,EAAE,SAAS;AAAA,EACtC,mBAAmBA,GAAE,OAAO,EAAE,SAAS;AAAA,EACvC,yBAAyBA,GAAE,OAAO,EAAE,SAAS;AAAA,EAC7C,0BAA0BA,GAAE,OAAO,EAAE,SAAS;AAAA,EAC9C,mBAAmBA,GAAE,OAAO,EAAE,SAAS;AAAA,EACvC,oBAAoBA,GAAE,OAAO,EAAE,SAAS;AAC1C;","names":["z"]}

package/dist/chunk-UHHHSYVE.js

@@ -0,0 +1,11 @@
+// src/constants.ts
+var PEERBENCH_NAMESPACE = "peerbench.ai";
+var CATEGORIES = {
+  LLM: "llm"
+};
+
+export {
+  PEERBENCH_NAMESPACE,
+  CATEGORIES
+};
+//# sourceMappingURL=chunk-UHHHSYVE.js.map

package/dist/chunk-UHHHSYVE.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/constants.ts"],"sourcesContent":["export const PEERBENCH_NAMESPACE = \"peerbench.ai\" as const;\nexport const CATEGORIES = {\n  LLM: \"llm\",\n};\n"],"mappings":";AAAO,IAAM,sBAAsB;AAC5B,IAAM,aAAa;AAAA,EACxB,KAAK;AACP;","names":[]}