peerbench 0.0.2-alpha.0 → 0.0.2-alpha.2

package/dist/storages/index.js

@@ -0,0 +1,98 @@
+import {
+  AbstractFileStorageCodec,
+  AbstractStorage,
+  FileStorage,
+  JSONFileStorage,
+  JSONFileStorageCodec
+} from "../chunk-ERALDEZY.js";
+import "../chunk-4UBK6452.js";
+import "../chunk-PZ5AY32C.js";
+
+// src/storages/http.ts
+var HttpStorage = class extends AbstractStorage {
+  url;
+  codec;
+  fetchFn;
+  constructor(config) {
+    super();
+    this.url = config.url;
+    this.codec = config.codec;
+    this.fetchFn = config.fetchFn ?? fetch;
+  }
+  async init(_params) {
+  }
+  async read(_key, _params) {
+    throw new Error("Not implemented");
+  }
+  async readAll(params) {
+    const response = await this.fetchFn(this.url, {
+      method: "GET",
+      headers: {
+        Accept: "application/json, text/plain, */*"
+      }
+    });
+    if (!response.ok) {
+      throw new Error(
+        `HTTP storage request failed: ${response.status} ${response.statusText}`
+      );
+    }
+    return await this.codec.readAll({ url: this.url, response, params });
+  }
+  async write(_key, _value, _params) {
+    throw new Error("Not implemented");
+  }
+};
+var AbstractHttpStorageCodec = class {
+};
+
+// src/storages/sqlite.ts
+import Database from "better-sqlite3";
+var SQLiteStorage = class extends AbstractStorage {
+  path;
+  codec;
+  db;
+  constructor(config) {
+    super();
+    this.path = config.path;
+    this.codec = config.codec;
+  }
+  async init(params) {
+    this.db = new Database(this.path);
+    await this.codec.init?.({ db: this.db, params });
+  }
+  async read(key, params) {
+    this.assertInitialized();
+    return await this.codec.read({ db: this.db, key, params });
+  }
+  async readAll(params) {
+    this.assertInitialized();
+    return await this.codec.readAll({ db: this.db, params });
+  }
+  async write(key, value, params) {
+    this.assertInitialized();
+    return await this.codec.write({ db: this.db, key, value, params });
+  }
+  close() {
+    this.db?.close();
+    this.db = void 0;
+  }
+  assertInitialized() {
+    if (!this.db) {
+      throw new Error("SQLite database is not initialized");
+    }
+  }
+};
+var AbstractSQLiteStorageCodec = class {
+};
+export {
+  AbstractFileStorageCodec,
+  AbstractHttpStorageCodec,
+  AbstractSQLiteStorageCodec,
+  AbstractStorage,
+  FileStorage,
+  HttpStorage,
+  JSONFileStorage,
+  JSONFileStorageCodec,
+  SQLiteStorage
+};
+//# sourceMappingURL=index.js.map

package/dist/storages/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/storages/http.ts","../../src/storages/sqlite.ts"],"sourcesContent":["import { AbstractStorage } from \"@/storages/abstract\";\n\nexport class HttpStorage<TObject> extends AbstractStorage<TObject> {\n  protected readonly url: string;\n  protected readonly codec: AbstractHttpStorageCodec<TObject>;\n  protected readonly fetchFn: typeof fetch;\n\n  constructor(config: {\n    url: string;\n    codec: AbstractHttpStorageCodec<TObject>;\n    fetchFn?: typeof fetch;\n  }) {\n    super();\n    this.url = config.url;\n    this.codec = config.codec;\n    this.fetchFn = config.fetchFn ?? fetch;\n  }\n\n  override async init(_params?: unknown): Promise<void> {\n    // Nothing to do here\n  }\n\n  override async read(\n    _key: string,\n    _params?: unknown\n  ): Promise<TObject | null> {\n    // TODO: Implement `read(key)` with codec support (e.g. query params / path templating).\n    throw new Error(\"Not implemented\");\n  }\n\n  override async readAll(params?: unknown): Promise<TObject[]> {\n    const response = await this.fetchFn(this.url, {\n      method: \"GET\",\n      headers: {\n        Accept: \"application/json, text/plain, */*\",\n      },\n    });\n\n    if (!response.ok) {\n      throw new Error(\n        `HTTP storage request failed: ${response.status} ${response.statusText}`\n      );\n    }\n\n    return await this.codec.readAll({ url: this.url, response, params });\n  }\n\n  override async write(\n    _key: string,\n    _value: TObject,\n    _params?: unknown\n  ): Promise<unknown> {\n    // TODO: Implement `write(key, value)` for HTTP storage if needed (PUT/POST/PATCH).\n    throw new Error(\"Not implemented\");\n  }\n}\n\nexport abstract class AbstractHttpStorageCodec<TObject> {\n  abstract readAll(params: {\n    url: string;\n    response: Response;\n    params?: unknown;\n  }): Promise<TObject[]>;\n}\n","import Database from \"better-sqlite3\";\nimport { AbstractStorage } from \"@/storages/abstract\";\n\nexport class SQLiteStorage<TObject> extends AbstractStorage<TObject> {\n  protected readonly path: string;\n  protected readonly codec: AbstractSQLiteStorageCodec<TObject>;\n\n  protected db: SQLiteDatabase | undefined;\n\n  constructor(config: {\n    path: string;\n    codec: AbstractSQLiteStorageCodec<TObject>;\n  }) {\n    super();\n    this.path = config.path;\n    this.codec = config.codec;\n  }\n\n  async init(params?: unknown): Promise<void> {\n    this.db = new Database(this.path);\n    await this.codec.init?.({ db: this.db, params });\n  }\n\n  override async read(key: string, params?: unknown): Promise<TObject | null> {\n    this.assertInitialized();\n    return await this.codec.read({ db: this.db, key, params });\n  }\n\n  override async readAll(params?: unknown): Promise<TObject[]> {\n    this.assertInitialized();\n    return await this.codec.readAll({ db: this.db, params });\n  }\n\n  override async write(\n    key: string,\n    value: TObject,\n    params?: unknown\n  ): Promise<unknown> {\n    this.assertInitialized();\n    return await this.codec.write({ db: this.db, key, value, params });\n  }\n\n  close(): void {\n    this.db?.close();\n    this.db = undefined;\n  }\n\n  protected assertInitialized(): asserts this is this & { db: SQLiteDatabase } {\n    if (!this.db) {\n      throw new Error(\"SQLite database is not initialized\");\n    }\n  }\n}\n\nexport abstract class AbstractSQLiteStorageCodec<TObject> {\n  init?(params: { db: SQLiteDatabase; params?: unknown }): Promise<void> | void;\n\n  abstract read(params: {\n    db: SQLiteDatabase;\n    key: string;\n    params?: unknown;\n  }): Promise<TObject | null>;\n\n  abstract readAll(params: {\n    db: SQLiteDatabase;\n    params?: unknown;\n  }): Promise<TObject[]>;\n\n  abstract write(params: {\n    db: SQLiteDatabase;\n    key: string;\n    value: TObject;\n    params?: unknown;\n  }): Promise<unknown>;\n}\n\ntype SQLiteDatabase = import(\"better-sqlite3\").Database;\n"],"mappings":";;;;;;;;;;;AAEO,IAAM,cAAN,cAAmC,gBAAyB;AAAA,EAC9C;AAAA,EACA;AAAA,EACA;AAAA,EAEnB,YAAY,QAIT;AACD,UAAM;AACN,SAAK,MAAM,OAAO;AAClB,SAAK,QAAQ,OAAO;AACpB,SAAK,UAAU,OAAO,WAAW;AAAA,EACnC;AAAA,EAEA,MAAe,KAAK,SAAkC;AAAA,EAEtD;AAAA,EAEA,MAAe,KACb,MACA,SACyB;AAEzB,UAAM,IAAI,MAAM,iBAAiB;AAAA,EACnC;AAAA,EAEA,MAAe,QAAQ,QAAsC;AAC3D,UAAM,WAAW,MAAM,KAAK,QAAQ,KAAK,KAAK;AAAA,MAC5C,QAAQ;AAAA,MACR,SAAS;AAAA,QACP,QAAQ;AAAA,MACV;AAAA,IACF,CAAC;AAED,QAAI,CAAC,SAAS,IAAI;AAChB,YAAM,IAAI;AAAA,QACR,gCAAgC,SAAS,MAAM,IAAI,SAAS,UAAU;AAAA,MACxE;AAAA,IACF;AAEA,WAAO,MAAM,KAAK,MAAM,QAAQ,EAAE,KAAK,KAAK,KAAK,UAAU,OAAO,CAAC;AAAA,EACrE;AAAA,EAEA,MAAe,MACb,MACA,QACA,SACkB;AAElB,UAAM,IAAI,MAAM,iBAAiB;AAAA,EACnC;AACF;AAEO,IAAe,2BAAf,MAAiD;AAMxD;;;AC/DA,OAAO,cAAc;AAGd,IAAM,gBAAN,cAAqC,gBAAyB;AAAA,EAChD;AAAA,EACA;AAAA,EAET;AAAA,EAEV,YAAY,QAGT;AACD,UAAM;AACN,SAAK,OAAO,OAAO;AACnB,SAAK,QAAQ,OAAO;AAAA,EACtB;AAAA,EAEA,MAAM,KAAK,QAAiC;AAC1C,SAAK,KAAK,IAAI,SAAS,KAAK,IAAI;AAChC,UAAM,KAAK,MAAM,OAAO,EAAE,IAAI,KAAK,IAAI,OAAO,CAAC;AAAA,EACjD;AAAA,EAEA,MAAe,KAAK,KAAa,QAA2C;AAC1E,SAAK,kBAAkB;AACvB,WAAO,MAAM,KAAK,MAAM,KAAK,EAAE,IAAI,KAAK,IAAI,KAAK,OAAO,CAAC;AAAA,EAC3D;AAAA,EAEA,MAAe,QAAQ,QAAsC;AAC3D,SAAK,kBAAkB;AACvB,WAAO,MAAM,KAAK,MAAM,QAAQ,EAAE,IAAI,KAAK,IAAI,OAAO,CAAC;AAAA,EACzD;AAAA,EAEA,MAAe,MACb,KACA,OACA,QACkB;AAClB,SAAK,kBAAkB;AACvB,WAAO,MAAM,KAAK,MAAM,MAAM,EAAE,IAAI,KAAK,IAAI,KAAK,OAAO,OAAO,CAAC;AAAA,EACnE;AAAA,EAEA,QAAc;AACZ,SAAK,IAAI,MAAM;AACf,SAAK,KAAK;AAAA,EACZ;AAAA,EAEU,oBAAmE;AAC3E,QAAI,CAAC,KAAK,IAAI;AACZ,YAAM,IAAI,MAAM,oCAAoC;AAAA,IACtD;AAAA,EACF;AACF;AAEO,IAAe,6BAAf,MAAmD;AAoB1D;","names":[]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "peerbench",
-  "version": "0.0.2-alpha.0",
+  "version": "0.0.2-alpha.2",
   "description": "Tooling help work with AI benchmarking data and create trustworthy AI",
   "keywords": [
     "ai",
@@ -45,7 +45,7 @@
       "types": "./dist/schemas/llm/index.d.ts",
       "import": "./dist/schemas/llm/index.js"
     },
-    "./extensions": {
+    "./schemas/extensions": {
       "types": "./dist/schemas/extensions/index.d.ts",
       "import": "./dist/schemas/extensions/index.js"
     },
@@ -53,9 +53,13 @@
       "types": "./dist/scorers/index.d.ts",
       "import": "./dist/scorers/index.js"
     },
-    "./catalogs": {
-      "types": "./dist/catalogs/index.d.ts",
-      "import": "./dist/catalogs/index.js"
+    "./storages": {
+      "types": "./dist/storages/index.d.ts",
+      "import": "./dist/storages/index.js"
+    },
+    "./aggregators": {
+      "types": "./dist/aggregators/index.d.ts",
+      "import": "./dist/aggregators/index.js"
     }
   },
   "files": [
@@ -71,10 +75,11 @@
   "dependencies": {
     "@mastra/client-js": "^0.17.2",
     "axios": "^1.11.0",
+    "better-sqlite3": "^12.6.0",
     "cheerio": "^1.1.2",
     "decimal.js": "^10.6.0",
+    "handlebars": "^4.7.8",
     "hyparquet": "^1.11.0",
-    "json-stable-stringify": "^1.3.0",
     "jsonrepair": "^3.13.0",
     "multiformats": "^13.3.2",
     "openai": "^6.15.0",
@@ -85,6 +90,7 @@
   "devDependencies": {
     "@dotenvx/dotenvx": "^1.49.0",
     "@eslint/js": "^9.24.0",
+    "@types/better-sqlite3": "^7.6.13",
     "@types/node": "^22.14.0",
     "eslint": "^9.24.0",
     "prettier": "^3.7.1",

package/dist/catalogs/index.d.ts

@@ -1,75 +0,0 @@
-import { C as ClassConstructor, A as AbstractFileLoader, a as AbstractHttpLoader } from '../utilities-BrRH32rD.js';
-import { A as AbstractProvider } from '../provider-BDjGp2y-.js';
-import { A as AbstractScorer } from '../abstract-Dec9Sc5O.js';
-import { A as AbstractDataLoader, a as AbstractLoader, L as LoaderResult, R as Runner, b as RunnerParams } from '../data-BmN5WjZ4.js';
-import { BaseResponseV1, BaseScoreV1 } from '../schemas/index.js';
-import '../index-WiPjF2AL.js';
-import 'zod';
-import 'node:fs';
-import '../system-prompt--0FdPWqK.js';
-
-declare class BaseCatalog<T> {
-    protected items: Record<string, T>;
-    register(kind: string, item: T): void;
-    deregister(kind: string): void;
-    get<K extends T>(kind: string, throwIfNotFound?: boolean): K;
-}
-
-type ClassCatalogItem<T extends ClassConstructor<unknown>> = {
-    constructor: T;
-    instantiate: (...args: any[]) => InstanceType<T>;
-};
-declare class ClassCatalog<T extends ClassConstructor<unknown>> extends BaseCatalog<ClassCatalogItem<T>> {
-    register(kind: string, item: ClassCatalogItem<T>): void;
-    instantiate(kind: string, ...args: any[]): InstanceType<T>;
-}
-
-declare class ProviderCatalog extends ClassCatalog<ClassConstructor<AbstractProvider>> {
-}
-declare class ProviderInstanceCatalog extends BaseCatalog<AbstractProvider> {
-}
-
-declare class ScorerCatalog extends ClassCatalog<ClassConstructor<AbstractScorer>> {
-}
-declare class ScorerInstanceCatalog extends BaseCatalog<AbstractScorer> {
-}
-
-declare class LoaderCatalog<T extends AbstractDataLoader | AbstractFileLoader | AbstractHttpLoader | AbstractLoader = AbstractLoader> extends BaseCatalog<T> {
-    register(kind: string, loader: T): void;
-    /**
-     * Executes all the registered loaders with the given params until one succeeds.
-     */
-    tryLoad(params: Parameters<T["loadData"]>[0]): Promise<LoaderResult<{
-        id: string;
-        kind: string;
-        schemaVersion: number;
-        metadata?: Record<string, unknown> | undefined;
-    }, {
-        id: string;
-        kind: string;
-        schemaVersion: number;
-        startedAt: number;
-        completedAt: number;
-        testCaseId: string;
-        metadata?: Record<string, unknown> | undefined;
-    }, {
-        id: string;
-        kind: string;
-        schemaVersion: number;
-        value: number;
-        responseId: string;
-        scoringMethod: "ai" | "human" | "algo";
-        explanation?: string | undefined;
-        metadata?: Record<string, unknown> | undefined;
-    }, {
-        kind: string;
-        schemaVersion: number;
-        metadata?: Record<string, unknown> | undefined;
-    }>>;
-}
-
-declare class RunnerCatalog<TRunner extends Runner = Runner> extends BaseCatalog<TRunner> {
-    register<TResponse extends BaseResponseV1 = BaseResponseV1, TScore extends BaseScoreV1 = BaseScoreV1, TParams extends RunnerParams = RunnerParams>(kind: string, runner: Runner<TResponse, TScore, TParams>): void;
-}
-
-export { BaseCatalog, ClassCatalog, type ClassCatalogItem, LoaderCatalog, ProviderCatalog, ProviderInstanceCatalog, RunnerCatalog, ScorerCatalog, ScorerInstanceCatalog };

package/dist/catalogs/index.js

@@ -1,88 +0,0 @@
-import {
-  CatalogItemNotFoundError
-} from "../chunk-VBOM2YEG.js";
-import "../chunk-PZ5AY32C.js";
-
-// src/catalogs/base.ts
-var BaseCatalog = class {
-  items = {};
-  register(kind, item) {
-    this.items[kind] = item;
-  }
-  deregister(kind) {
-    delete this.items[kind];
-  }
-  get(kind, throwIfNotFound = true) {
-    const item = this.items[kind];
-    if (throwIfNotFound && !item) {
-      throw new CatalogItemNotFoundError(kind);
-    }
-    return item;
-  }
-};
-
-// src/catalogs/class.ts
-var ClassCatalog = class extends BaseCatalog {
-  register(kind, item) {
-    super.register(kind, item);
-  }
-  instantiate(kind, ...args) {
-    const item = this.get(kind);
-    if (!item) {
-      throw new CatalogItemNotFoundError(kind);
-    }
-    return item.instantiate(...args);
-  }
-};
-
-// src/catalogs/provider.ts
-var ProviderCatalog = class extends ClassCatalog {
-};
-var ProviderInstanceCatalog = class extends BaseCatalog {
-};
-
-// src/catalogs/scorer.ts
-var ScorerCatalog = class extends ClassCatalog {
-};
-var ScorerInstanceCatalog = class extends BaseCatalog {
-};
-
-// src/catalogs/loader.ts
-var LoaderCatalog = class extends BaseCatalog {
-  register(kind, loader) {
-    super.register(kind, loader);
-  }
-  /**
-   * Executes all the registered loaders with the given params until one succeeds.
-   */
-  async tryLoad(params) {
-    for (const [, loader] of Object.entries(this.items)) {
-      try {
-        return await loader.loadData(params);
-      } catch {
-        continue;
-      }
-    }
-    throw new CatalogItemNotFoundError({
-      message: "No compatible loader found"
-    });
-  }
-};
-
-// src/catalogs/runner.ts
-var RunnerCatalog = class extends BaseCatalog {
-  register(kind, runner) {
-    super.register(kind, runner);
-  }
-};
-export {
-  BaseCatalog,
-  ClassCatalog,
-  LoaderCatalog,
-  ProviderCatalog,
-  ProviderInstanceCatalog,
-  RunnerCatalog,
-  ScorerCatalog,
-  ScorerInstanceCatalog
-};
-//# sourceMappingURL=index.js.map

package/dist/catalogs/index.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../src/catalogs/base.ts","../../src/catalogs/class.ts","../../src/catalogs/provider.ts","../../src/catalogs/scorer.ts","../../src/catalogs/loader.ts","../../src/catalogs/runner.ts"],"sourcesContent":["import { CatalogItemNotFoundError } from \"@/errors/catalog\";\n\nexport class BaseCatalog<T> {\n  protected items: Record<string, T> = {};\n\n  register(kind: string, item: T) {\n    this.items[kind] = item;\n  }\n\n  deregister(kind: string) {\n    delete this.items[kind];\n  }\n\n  get<K extends T>(kind: string, throwIfNotFound: boolean = true) {\n    const item = this.items[kind];\n\n    if (throwIfNotFound && !item) {\n      throw new CatalogItemNotFoundError(kind);\n    }\n\n    return item as K;\n  }\n}\n","import { CatalogItemNotFoundError } from \"@/errors/catalog\";\nimport { ClassConstructor } from \"@/utilities\";\nimport { BaseCatalog } from \"./base\";\n\nexport type ClassCatalogItem<T extends ClassConstructor<unknown>> = {\n  constructor: T;\n  instantiate: (...args: any[]) => InstanceType<T>;\n};\n\nexport class ClassCatalog<\n  T extends ClassConstructor<unknown>,\n> extends BaseCatalog<ClassCatalogItem<T>> {\n  override register(kind: string, item: ClassCatalogItem<T>) {\n    super.register(kind, item);\n  }\n\n  instantiate(kind: string, ...args: any[]) {\n    const item = this.get(kind);\n\n    if (!item) {\n      throw new CatalogItemNotFoundError(kind);\n    }\n\n    return item.instantiate(...args);\n  }\n}\n","import { ClassCatalog } from \"./class\";\nimport { ClassConstructor } from \"@/utilities\";\nimport { AbstractProvider } from \"@/providers/abstract/provider\";\nimport { BaseCatalog } from \"./base\";\n\nexport class ProviderCatalog extends ClassCatalog<\n  ClassConstructor<AbstractProvider>\n> {}\nexport class ProviderInstanceCatalog extends BaseCatalog<AbstractProvider> {}\n","import { ClassCatalog } from \"./class\";\nimport { ClassConstructor } from \"@/utilities\";\nimport { AbstractScorer } from \"@/scorers/abstract\";\nimport { BaseCatalog } from \"./base\";\n\nexport class ScorerCatalog extends ClassCatalog<\n  ClassConstructor<AbstractScorer>\n> {}\nexport class ScorerInstanceCatalog extends BaseCatalog<AbstractScorer> {}\n","import { CatalogItemNotFoundError } from \"@/errors/catalog\";\nimport { BaseCatalog } from \"./base\";\nimport {\n  AbstractDataLoader,\n  AbstractFileLoader,\n  AbstractHttpLoader,\n  AbstractLoader,\n} from \"@/loaders\";\n\nexport class LoaderCatalog<\n  T extends\n    | AbstractDataLoader\n    | AbstractFileLoader\n    | AbstractHttpLoader\n    | AbstractLoader = AbstractLoader,\n> extends BaseCatalog<T> {\n  override register(kind: string, loader: T) {\n    super.register(kind, loader);\n  }\n\n  /**\n   * Executes all the registered loaders with the given params until one succeeds.\n   */\n  async tryLoad(params: Parameters<T[\"loadData\"]>[0]) {\n    for (const [, loader] of Object.entries(this.items)) {\n      try {\n        return await loader.loadData(params as any);\n      } catch {\n        continue;\n      }\n    }\n\n    throw new CatalogItemNotFoundError({\n      message: \"No compatible loader found\",\n    });\n  }\n}\n","import { Runner, RunnerParams } from \"@/types\";\nimport { BaseCatalog } from \"./base\";\nimport { BaseResponseV1, BaseScoreV1 } from \"@/schemas\";\n\nexport class RunnerCatalog<\n  TRunner extends Runner = Runner,\n> extends BaseCatalog<TRunner> {\n  override register<\n    TResponse extends BaseResponseV1 = BaseResponseV1,\n    TScore extends BaseScoreV1 = BaseScoreV1,\n    TParams extends RunnerParams = RunnerParams,\n  >(kind: string, runner: Runner<TResponse, TScore, TParams>) {\n    super.register(kind, runner as unknown as TRunner);\n  }\n}\n"],"mappings":";;;;;;AAEO,IAAM,cAAN,MAAqB;AAAA,EAChB,QAA2B,CAAC;AAAA,EAEtC,SAAS,MAAc,MAAS;AAC9B,SAAK,MAAM,IAAI,IAAI;AAAA,EACrB;AAAA,EAEA,WAAW,MAAc;AACvB,WAAO,KAAK,MAAM,IAAI;AAAA,EACxB;AAAA,EAEA,IAAiB,MAAc,kBAA2B,MAAM;AAC9D,UAAM,OAAO,KAAK,MAAM,IAAI;AAE5B,QAAI,mBAAmB,CAAC,MAAM;AAC5B,YAAM,IAAI,yBAAyB,IAAI;AAAA,IACzC;AAEA,WAAO;AAAA,EACT;AACF;;;ACbO,IAAM,eAAN,cAEG,YAAiC;AAAA,EAChC,SAAS,MAAc,MAA2B;AACzD,UAAM,SAAS,MAAM,IAAI;AAAA,EAC3B;AAAA,EAEA,YAAY,SAAiB,MAAa;AACxC,UAAM,OAAO,KAAK,IAAI,IAAI;AAE1B,QAAI,CAAC,MAAM;AACT,YAAM,IAAI,yBAAyB,IAAI;AAAA,IACzC;AAEA,WAAO,KAAK,YAAY,GAAG,IAAI;AAAA,EACjC;AACF;;;ACpBO,IAAM,kBAAN,cAA8B,aAEnC;AAAC;AACI,IAAM,0BAAN,cAAsC,YAA8B;AAAC;;;ACHrE,IAAM,gBAAN,cAA4B,aAEjC;AAAC;AACI,IAAM,wBAAN,cAAoC,YAA4B;AAAC;;;ACCjE,IAAM,gBAAN,cAMG,YAAe;AAAA,EACd,SAAS,MAAc,QAAW;AACzC,UAAM,SAAS,MAAM,MAAM;AAAA,EAC7B;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,QAAQ,QAAsC;AAClD,eAAW,CAAC,EAAE,MAAM,KAAK,OAAO,QAAQ,KAAK,KAAK,GAAG;AACnD,UAAI;AACF,eAAO,MAAM,OAAO,SAAS,MAAa;AAAA,MAC5C,QAAQ;AACN;AAAA,MACF;AAAA,IACF;AAEA,UAAM,IAAI,yBAAyB;AAAA,MACjC,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AACF;;;AChCO,IAAM,gBAAN,cAEG,YAAqB;AAAA,EACpB,SAIP,MAAc,QAA4C;AAC1D,UAAM,SAAS,MAAM,MAA4B;AAAA,EACnD;AACF;","names":[]}

package/dist/chunk-22HU24QF.js

@@ -1,8 +0,0 @@
-// src/scorers/abstract.ts
-var AbstractScorer = class {
-};
-
-export {
-  AbstractScorer
-};
-//# sourceMappingURL=chunk-22HU24QF.js.map

package/dist/chunk-22HU24QF.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/scorers/abstract.ts"],"sourcesContent":["export abstract class AbstractScorer {\n  abstract readonly kind: string;\n\n  abstract score(params: any): Promise<BaseScorerResult | null>;\n}\n\nexport type BaseScorerResult = {\n  value: number;\n  explanation?: string;\n  metadata?: Record<string, unknown>;\n  [key: string]: unknown;\n};\n"],"mappings":";AAAO,IAAe,iBAAf,MAA8B;AAIrC;","names":[]}

package/dist/chunk-232PY7K3.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/schemas/benchmark-spec.ts","../src/schemas/test-case.ts","../src/schemas/score.ts"],"sourcesContent":["import z from \"zod\";\nimport { buildSchemaDefiner } from \"./schema-definer\";\n\nexport const BaseBenchmarkSpecSchemaV1 = z.object({\n  kind: z.string(),\n  schemaVersion: z.number(),\n  metadata: z.record(z.string(), z.unknown()).optional(),\n});\nexport type BaseBenchmarkSpecV1 = z.infer<typeof BaseBenchmarkSpecSchemaV1>;\n\nexport const defineBenchmarkSpecSchema =\n  buildSchemaDefiner<typeof BaseBenchmarkSpecSchemaV1.shape>();\n","import { IdSchema } from \"./id\";\nimport { z } from \"zod\";\nimport { buildSchemaDefiner } from \"./schema-definer\";\n\nexport const BaseTestCaseSchemaV1 = z.object({\n  id: IdSchema,\n  kind: z.string(),\n  schemaVersion: z.number(),\n  metadata: z.record(z.string(), z.unknown()).optional(),\n});\nexport type BaseTestCaseV1 = z.infer<typeof BaseTestCaseSchemaV1>;\n\nexport const defineTestCaseSchema =\n  buildSchemaDefiner<typeof BaseTestCaseSchemaV1.shape>();\n","import { IdSchema } from \"./id\";\nimport { ScoringMethod } from \"@/types\";\nimport { buildSchemaDefiner } from \"./schema-definer\";\nimport z from \"zod\";\n\nexport const BaseScoreSchemaV1 = z.object({\n  id: IdSchema,\n  kind: z.string(),\n  schemaVersion: z.number(),\n\n  value: z.number(),\n  responseId: IdSchema,\n  explanation: z.string().optional(),\n  metadata: z.record(z.string(), z.unknown()).optional(),\n  scoringMethod: z.enum(ScoringMethod),\n});\nexport type BaseScoreV1 = z.infer<typeof BaseScoreSchemaV1>;\n\nexport const defineScoreSchema =\n  buildSchemaDefiner<typeof BaseScoreSchemaV1.shape>();\n"],"mappings":";;;;;;;;;AAAA,OAAO,OAAO;AAGP,IAAM,4BAA4B,EAAE,OAAO;AAAA,EAChD,MAAM,EAAE,OAAO;AAAA,EACf,eAAe,EAAE,OAAO;AAAA,EACxB,UAAU,EAAE,OAAO,EAAE,OAAO,GAAG,EAAE,QAAQ,CAAC,EAAE,SAAS;AACvD,CAAC;AAGM,IAAM,4BACX,mBAA2D;;;ACV7D,SAAS,KAAAA,UAAS;AAGX,IAAM,uBAAuBC,GAAE,OAAO;AAAA,EAC3C,IAAI;AAAA,EACJ,MAAMA,GAAE,OAAO;AAAA,EACf,eAAeA,GAAE,OAAO;AAAA,EACxB,UAAUA,GAAE,OAAOA,GAAE,OAAO,GAAGA,GAAE,QAAQ,CAAC,EAAE,SAAS;AACvD,CAAC;AAGM,IAAM,uBACX,mBAAsD;;;ACVxD,OAAOC,QAAO;AAEP,IAAM,oBAAoBA,GAAE,OAAO;AAAA,EACxC,IAAI;AAAA,EACJ,MAAMA,GAAE,OAAO;AAAA,EACf,eAAeA,GAAE,OAAO;AAAA,EAExB,OAAOA,GAAE,OAAO;AAAA,EAChB,YAAY;AAAA,EACZ,aAAaA,GAAE,OAAO,EAAE,SAAS;AAAA,EACjC,UAAUA,GAAE,OAAOA,GAAE,OAAO,GAAGA,GAAE,QAAQ,CAAC,EAAE,SAAS;AAAA,EACrD,eAAeA,GAAE,KAAK,aAAa;AACrC,CAAC;AAGM,IAAM,oBACX,mBAAmD;","names":["z","z","z"]}

package/dist/chunk-7TREBPSJ.js

@@ -1,26 +0,0 @@
-import {
-  BaseResponseSchemaV1,
-  IdSchema,
-  defineResponseSchema
-} from "./chunk-IUN2IUCS.js";
-
-// src/schemas/llm/response.ts
-import z from "zod";
-var BaseLLMChatResponseSchemaV1 = defineResponseSchema({
-  baseSchema: BaseResponseSchemaV1,
-  fields: {
-    data: z.string(),
-    modelSlug: z.string(),
-    provider: z.string(),
-    systemPromptId: IdSchema.optional(),
-    inputTokensUsed: z.number().optional(),
-    outputTokensUsed: z.number().optional(),
-    inputCost: z.string().optional(),
-    outputCost: z.string().optional()
-  }
-});
-
-export {
-  BaseLLMChatResponseSchemaV1
-};
-//# sourceMappingURL=chunk-7TREBPSJ.js.map

package/dist/chunk-7TREBPSJ.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/schemas/llm/response.ts"],"sourcesContent":["import { IdSchema } from \"../id\";\nimport { BaseResponseSchemaV1, defineResponseSchema } from \"../response\";\nimport z from \"zod\";\n\n/**\n * Base response schema specialized for LLM chat responses.\n */\nexport const BaseLLMChatResponseSchemaV1 = defineResponseSchema({\n  baseSchema: BaseResponseSchemaV1,\n  fields: {\n    data: z.string(),\n    modelSlug: z.string(),\n    provider: z.string(),\n    systemPromptId: IdSchema.optional(),\n\n    inputTokensUsed: z.number().optional(),\n    outputTokensUsed: z.number().optional(),\n    inputCost: z.string().optional(),\n    outputCost: z.string().optional(),\n  },\n});\n"],"mappings":";;;;;;;AAEA,OAAO,OAAO;AAKP,IAAM,8BAA8B,qBAAqB;AAAA,EAC9D,YAAY;AAAA,EACZ,QAAQ;AAAA,IACN,MAAM,EAAE,OAAO;AAAA,IACf,WAAW,EAAE,OAAO;AAAA,IACpB,UAAU,EAAE,OAAO;AAAA,IACnB,gBAAgB,SAAS,SAAS;AAAA,IAElC,iBAAiB,EAAE,OAAO,EAAE,SAAS;AAAA,IACrC,kBAAkB,EAAE,OAAO,EAAE,SAAS;AAAA,IACtC,WAAW,EAAE,OAAO,EAAE,SAAS;AAAA,IAC/B,YAAY,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC;AACF,CAAC;","names":[]}

package/dist/chunk-DUBKY73H.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/utils/string.ts","../src/utils/sleep.ts","../src/utils/llm.ts","../src/utils/rate-limiter.ts","../src/utils/id-generator.ts"],"sourcesContent":["/**\n * Converts the given byte array to a string\n */\nexport function bufferToString(buffer: Uint8Array): string {\n  const decoder = new TextDecoder();\n  return decoder.decode(buffer);\n}\n\n/**\n * Converts the given string to a byte array\n */\nexport function stringToBuffer(str: string): Uint8Array {\n  const encoder = new TextEncoder();\n  return encoder.encode(str);\n}\n","export async function sleep(ms: number, signal?: AbortSignal) {\n  return new Promise<void>((resolve, reject) => {\n    if (signal?.aborted) {\n      return reject(new Error(\"Aborted\"));\n    }\n\n    const timeout = setTimeout(() => {\n      cleanup();\n      resolve();\n    }, ms);\n\n    const onAbort = () => {\n      cleanup();\n      reject(new Error(\"Aborted\"));\n    };\n\n    const cleanup = () => {\n      clearTimeout(timeout);\n      signal?.removeEventListener(\"abort\", onAbort);\n    };\n\n    signal?.addEventListener(\"abort\", onAbort);\n  });\n}\n","import { jsonrepair } from \"jsonrepair\";\n\n/**\n * Tries to repair and parse LLM response as a JSON object. LLM must\n * be configured to return a JSON object. This function only helps to\n * get rid out of some additional formatting (e.g. ```json) and repair\n * the JSON syntax (e.g missing comma, single quotes instead double).\n */\nexport function parseResponseAsJSON<T>(response: string) {\n  try {\n    return JSON.parse(jsonrepair(response)) as T;\n  } catch (e) {\n    if (process?.env?.PB_SDK_DEBUG) {\n      console.log(\"Original response\", JSON.stringify(response));\n      console.error(\"Error parsing response as JSON\", e);\n    }\n  }\n}\n","import { sleep } from \"./sleep\";\n\nexport interface RateLimiterOptions {\n  /**\n   * Maximum weight of calls allowed within a time window. Each call has a weight of 1 by default.\n   * Rate limiting is disabled when this is 0 or negative.\n   * @default 25\n   */\n  maxWeight?: number;\n\n  /**\n   * Time window in milliseconds for rate limiting\n   * @default 1000\n   */\n  timeWindow?: number;\n}\n\nexport interface RateLimiterCallOptions {\n  /**\n   * Weight of the call. Rate limiting is applied based on the total weight of the calls.\n   * @default 1\n   */\n  weight?: number;\n\n  /**\n   * Abort signal for cancellation\n   */\n  signal?: AbortSignal;\n}\n\n/**\n * Generic rate limiter. It can be used to limit async function calls\n * by a given number of calls within a time window.\n */\nexport class RateLimiter {\n  maxWeight: number;\n  timeWindow: number;\n\n  private timestamps: number[] = [];\n\n  constructor(options: RateLimiterOptions = {}) {\n    this.maxWeight = options.maxWeight ?? 25;\n    this.timeWindow = options.timeWindow ?? 1000;\n  }\n\n  /**\n   * Checks if rate limiting is disabled\n   */\n  isDisabled(): boolean {\n    return this.maxWeight <= 0;\n  }\n\n  /**\n   * Disables rate limiting. Set `maxWeight` to re-enable it.\n   */\n  disable() {\n    this.maxWeight = 0;\n  }\n\n  /**\n   * Returns how many weight of calls are there in the current time window\n   */\n  getCurrentCalls(): number {\n    const now = Date.now();\n    this.timestamps = this.timestamps.filter(\n      (ts) => now - ts < this.timeWindow\n    );\n    return this.timestamps.length;\n  }\n\n  /**\n   * Executes the given function with rate limiting applied\n   */\n  async execute<T = unknown>(\n    func: () => Promise<T>,\n    { weight = 1, signal }: RateLimiterCallOptions = {}\n  ): Promise<T> {\n    // If maxWeight is zero/negative, execute immediately (no rate limiting)\n    if (this.maxWeight <= 0) {\n      return await func();\n    }\n\n    await this.waitForRateLimit(weight, signal);\n    return await func();\n  }\n\n  /**\n   * Waits until rate limit allows the specified weight of calls\n   */\n  private async waitForRateLimit(\n    weight: number,\n    signal?: AbortSignal\n  ): Promise<void> {\n    const now = Date.now();\n\n    // Clear timestamps from the old time period\n    this.timestamps = this.timestamps.filter(\n      (ts) => now - ts < this.timeWindow\n    );\n\n    // Check if there are empty slots for the requests\n    if (this.timestamps.length + weight <= this.maxWeight) {\n      for (let i = 0; i < weight; i++) {\n        this.timestamps.push(now);\n      }\n      return;\n    }\n\n    // Calculate how much time to wait for the next time window\n    const earliest = this.timestamps[0];\n    const waitTime = this.timeWindow - (now - (earliest ?? 0));\n\n    await sleep(waitTime, signal);\n\n    // Retry after waiting\n    return this.waitForRateLimit(weight, signal);\n  }\n\n  /**\n   * Resets the rate limited weight of calls\n   */\n  reset(): void {\n    this.timestamps = [];\n  }\n}\n","import { IdGenerator } from \"@/types\";\nimport { v7 as uuidv7 } from \"uuid\";\n\nexport const idGeneratorUUIDv7: IdGenerator = () => {\n  return uuidv7();\n};\n"],"mappings":";AAGO,SAAS,eAAe,QAA4B;AACzD,QAAM,UAAU,IAAI,YAAY;AAChC,SAAO,QAAQ,OAAO,MAAM;AAC9B;AAKO,SAAS,eAAe,KAAyB;AACtD,QAAM,UAAU,IAAI,YAAY;AAChC,SAAO,QAAQ,OAAO,GAAG;AAC3B;;;ACdA,eAAsB,MAAM,IAAY,QAAsB;AAC5D,SAAO,IAAI,QAAc,CAAC,SAAS,WAAW;AAC5C,QAAI,QAAQ,SAAS;AACnB,aAAO,OAAO,IAAI,MAAM,SAAS,CAAC;AAAA,IACpC;AAEA,UAAM,UAAU,WAAW,MAAM;AAC/B,cAAQ;AACR,cAAQ;AAAA,IACV,GAAG,EAAE;AAEL,UAAM,UAAU,MAAM;AACpB,cAAQ;AACR,aAAO,IAAI,MAAM,SAAS,CAAC;AAAA,IAC7B;AAEA,UAAM,UAAU,MAAM;AACpB,mBAAa,OAAO;AACpB,cAAQ,oBAAoB,SAAS,OAAO;AAAA,IAC9C;AAEA,YAAQ,iBAAiB,SAAS,OAAO;AAAA,EAC3C,CAAC;AACH;;;ACvBA,SAAS,kBAAkB;AAQpB,SAAS,oBAAuB,UAAkB;AACvD,MAAI;AACF,WAAO,KAAK,MAAM,WAAW,QAAQ,CAAC;AAAA,EACxC,SAAS,GAAG;AACV,QAAI,SAAS,KAAK,cAAc;AAC9B,cAAQ,IAAI,qBAAqB,KAAK,UAAU,QAAQ,CAAC;AACzD,cAAQ,MAAM,kCAAkC,CAAC;AAAA,IACnD;AAAA,EACF;AACF;;;ACiBO,IAAM,cAAN,MAAkB;AAAA,EACvB;AAAA,EACA;AAAA,EAEQ,aAAuB,CAAC;AAAA,EAEhC,YAAY,UAA8B,CAAC,GAAG;AAC5C,SAAK,YAAY,QAAQ,aAAa;AACtC,SAAK,aAAa,QAAQ,cAAc;AAAA,EAC1C;AAAA;AAAA;AAAA;AAAA,EAKA,aAAsB;AACpB,WAAO,KAAK,aAAa;AAAA,EAC3B;AAAA;AAAA;AAAA;AAAA,EAKA,UAAU;AACR,SAAK,YAAY;AAAA,EACnB;AAAA;AAAA;AAAA;AAAA,EAKA,kBAA0B;AACxB,UAAM,MAAM,KAAK,IAAI;AACrB,SAAK,aAAa,KAAK,WAAW;AAAA,MAChC,CAAC,OAAO,MAAM,KAAK,KAAK;AAAA,IAC1B;AACA,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,QACJ,MACA,EAAE,SAAS,GAAG,OAAO,IAA4B,CAAC,GACtC;AAEZ,QAAI,KAAK,aAAa,GAAG;AACvB,aAAO,MAAM,KAAK;AAAA,IACpB;AAEA,UAAM,KAAK,iBAAiB,QAAQ,MAAM;AAC1C,WAAO,MAAM,KAAK;AAAA,EACpB;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,iBACZ,QACA,QACe;AACf,UAAM,MAAM,KAAK,IAAI;AAGrB,SAAK,aAAa,KAAK,WAAW;AAAA,MAChC,CAAC,OAAO,MAAM,KAAK,KAAK;AAAA,IAC1B;AAGA,QAAI,KAAK,WAAW,SAAS,UAAU,KAAK,WAAW;AACrD,eAAS,IAAI,GAAG,IAAI,QAAQ,KAAK;AAC/B,aAAK,WAAW,KAAK,GAAG;AAAA,MAC1B;AACA;AAAA,IACF;AAGA,UAAM,WAAW,KAAK,WAAW,CAAC;AAClC,UAAM,WAAW,KAAK,cAAc,OAAO,YAAY;AAEvD,UAAM,MAAM,UAAU,MAAM;AAG5B,WAAO,KAAK,iBAAiB,QAAQ,MAAM;AAAA,EAC7C;AAAA;AAAA;AAAA;AAAA,EAKA,QAAc;AACZ,SAAK,aAAa,CAAC;AAAA,EACrB;AACF;;;AC3HA,SAAS,MAAM,cAAc;AAEtB,IAAM,oBAAiC,MAAM;AAClD,SAAO,OAAO;AAChB;","names":[]}

package/dist/chunk-GVF4YZF3.js

@@ -1,15 +0,0 @@
-// src/schemas/extensions/score/llm-as-a-judge-scorer.ts
-import z from "zod";
-var ExtensionLLMAsAJudgeScorerFieldsV1 = {
-  scorerAIProvider: z.string().optional(),
-  scorerAIModelSlug: z.string().optional(),
-  scorerAIInputTokensUsed: z.number().optional(),
-  scorerAIOutputTokensUsed: z.number().optional(),
-  scorerAIInputCost: z.string().optional(),
-  scorerAIOutputCost: z.string().optional()
-};
-
-export {
-  ExtensionLLMAsAJudgeScorerFieldsV1
-};
-//# sourceMappingURL=chunk-GVF4YZF3.js.map

package/dist/chunk-GVF4YZF3.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/schemas/extensions/score/llm-as-a-judge-scorer.ts"],"sourcesContent":["import z from \"zod\";\n\n/**\n * Provides a set of fields that holds information about the LLM model\n * that was used to judge the response.\n */\nexport const ExtensionLLMAsAJudgeScorerFieldsV1 = {\n  scorerAIProvider: z.string().optional(),\n  scorerAIModelSlug: z.string().optional(),\n  scorerAIInputTokensUsed: z.number().optional(),\n  scorerAIOutputTokensUsed: z.number().optional(),\n  scorerAIInputCost: z.string().optional(),\n  scorerAIOutputCost: z.string().optional(),\n};\n"],"mappings":";AAAA,OAAO,OAAO;AAMP,IAAM,qCAAqC;AAAA,EAChD,kBAAkB,EAAE,OAAO,EAAE,SAAS;AAAA,EACtC,mBAAmB,EAAE,OAAO,EAAE,SAAS;AAAA,EACvC,yBAAyB,EAAE,OAAO,EAAE,SAAS;AAAA,EAC7C,0BAA0B,EAAE,OAAO,EAAE,SAAS;AAAA,EAC9C,mBAAmB,EAAE,OAAO,EAAE,SAAS;AAAA,EACvC,oBAAoB,EAAE,OAAO,EAAE,SAAS;AAC1C;","names":[]}

package/dist/chunk-HJH3SW3L.js

@@ -1,103 +0,0 @@
-import {
-  bufferToString
-} from "./chunk-DUBKY73H.js";
-
-// src/loaders/abstract/loader.ts
-var AbstractLoader = class {
-};
-
-// src/loaders/abstract/data.ts
-var AbstractDataLoader = class extends AbstractLoader {
-};
-
-// src/utils/json.ts
-import { stringify } from "safe-stable-stringify";
-function parseJSONL(str, options) {
-  return str.split("\n").filter((line) => line.trim() !== "").map((line) => {
-    const obj = tryParseJson(line);
-    if (options?.errorOnInvalid && !obj) {
-      throw new Error(`Invalid JSON line: ${line}`);
-    }
-    return obj;
-  }).filter((obj) => obj !== void 0);
-}
-function tryParseJson(content) {
-  try {
-    return JSON.parse(content);
-  } catch {
-  }
-}
-function stableStringify(value) {
-  return stringify(value);
-}
-
-// src/loaders/generic-array.ts
-var GenericJSONArrayDataLoader = class extends AbstractDataLoader {
-  async responseBuilder(data, context) {
-    return void 0;
-  }
-  async scoreBuilder(data, context) {
-    return void 0;
-  }
-  async loadData(params) {
-    const contentStr = bufferToString(params.content);
-    let data = tryParseJson(contentStr);
-    if (!data) {
-      data = parseJSONL(contentStr);
-    }
-    if (!data || !Array.isArray(data) || data.length === 0) {
-      throw new Error(
-        "Invalid data: content must be a non-empty JSON or JSONL array"
-      );
-    }
-    return this.transformArrayToResult(data);
-  }
-  async transformArrayToResult(data) {
-    const includedTestCaseIds = /* @__PURE__ */ new Set();
-    const includedResponseIds = /* @__PURE__ */ new Set();
-    const includedScoreIds = /* @__PURE__ */ new Set();
-    const result = {
-      testCases: [],
-      responses: [],
-      scores: []
-    };
-    for (const item of data) {
-      const testCase = await this.testCaseBuilder(item, { result });
-      const response = await this.responseBuilder(item, { result });
-      const score = await this.scoreBuilder(item, { result });
-      if (testCase) {
-        const testCaseIdStr = String(testCase.id);
-        if (!includedTestCaseIds.has(testCaseIdStr)) {
-          includedTestCaseIds.add(testCaseIdStr);
-          result.testCases.push(testCase);
-        }
-      }
-      if (response) {
-        const responseIdStr = String(response.id);
-        if (!includedResponseIds.has(responseIdStr)) {
-          includedResponseIds.add(responseIdStr);
-          result.responses.push(response);
-        }
-      }
-      if (score) {
-        const scoreIdStr = String(score.id);
-        if (!includedScoreIds.has(scoreIdStr)) {
-          includedScoreIds.add(scoreIdStr);
-          result.scores.push(score);
-        }
-      }
-      if (!testCase && !response && !score) {
-        throw new Error("Incompatible object format");
-      }
-    }
-    return result;
-  }
-};
-
-export {
-  AbstractLoader,
-  AbstractDataLoader,
-  stableStringify,
-  GenericJSONArrayDataLoader
-};
-//# sourceMappingURL=chunk-HJH3SW3L.js.map

package/dist/chunk-HJH3SW3L.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/loaders/abstract/loader.ts","../src/loaders/abstract/data.ts","../src/utils/json.ts","../src/loaders/generic-array.ts"],"sourcesContent":["import { BaseResponseV1, BaseScoreV1, BaseTestCaseV1 } from \"@/schemas\";\nimport { BaseBenchmarkSpecV1 } from \"@/schemas/benchmark-spec\";\nimport { MaybePromise } from \"@/types\";\n\nexport type LoaderResult<\n  TTestCase extends BaseTestCaseV1 = BaseTestCaseV1,\n  TResponse extends BaseResponseV1 = BaseResponseV1,\n  TScore extends BaseScoreV1 = BaseScoreV1,\n  TBenchmarkSpec extends BaseBenchmarkSpecV1 = BaseBenchmarkSpecV1,\n> = {\n  testCases: TTestCase[];\n  responses: TResponse[];\n  scores: TScore[];\n\n  benchmarkSpec?: TBenchmarkSpec;\n};\n\nexport abstract class AbstractLoader {\n  abstract readonly kind: string;\n\n  abstract loadData(params: unknown): MaybePromise<LoaderResult>;\n  abstract loadBenchmarkSpec(\n    params: unknown\n  ): MaybePromise<BaseBenchmarkSpecV1>;\n}\n","import { MaybePromise } from \"@/types\";\nimport { AbstractLoader, LoaderResult } from \"./loader\";\nimport { BaseBenchmarkSpecV1 } from \"@/schemas/benchmark-spec\";\n\nexport abstract class AbstractDataLoader extends AbstractLoader {\n  abstract loadData(params: {\n    content: Uint8Array;\n  }): MaybePromise<LoaderResult>;\n\n  abstract loadBenchmarkSpec(params: {\n    content: Uint8Array;\n  }): MaybePromise<BaseBenchmarkSpecV1>;\n}\n","import { stringify } from \"safe-stable-stringify\";\n\n/**\n * Parses JSONL formatted string into an array\n * @returns An array of parsed JSON lines\n */\nexport function parseJSONL<T>(\n  str: string,\n  options?: { errorOnInvalid?: boolean }\n): T[] {\n  return str\n    .split(\"\\n\") // Split per line\n    .filter((line) => line.trim() !== \"\") // Filter empty lines\n    .map((line) => {\n      const obj = tryParseJson(line);\n      if (options?.errorOnInvalid && !obj) {\n        throw new Error(`Invalid JSON line: ${line}`);\n      }\n      return obj;\n    }) // Parse line (parse as undefined if it is invalid)\n    .filter((obj) => obj !== undefined); // Filter invalid lines\n}\n\n/**\n * Tries to parse the given string as JSON.\n * Returns `undefined` if it is not a valid JSON entity.\n */\nexport function tryParseJson<T = any>(content: string): T | undefined {\n  try {\n    return JSON.parse(content);\n  } catch {\n    // Invalid JSON\n  }\n}\n\n/**\n * Stringifies the given JSON value using `safe-stable-stringify` in a stable manner.\n * This stable method generates the same string output for the same input value (including objects).\n */\nexport function stableStringify(value: any) {\n  return stringify(value);\n}\n","import { BaseTestCaseV1 } from \"@/schemas/test-case\";\nimport { BaseResponseV1 } from \"@/schemas/response\";\nimport { BaseScoreV1 } from \"@/schemas/score\";\nimport { MaybePromise } from \"@/types\";\nimport { AbstractDataLoader } from \"@/loaders/abstract/data\";\nimport { LoaderResult } from \"@/loaders/abstract/loader\";\nimport { tryParseJson, parseJSONL } from \"@/utils/json\";\nimport { bufferToString } from \"@/utils/string\";\n\nexport type GenericJSONArrayLoaderResult<\n  TTestCase extends BaseTestCaseV1 = BaseTestCaseV1,\n  TResponse extends BaseResponseV1 = BaseResponseV1,\n  TScore extends BaseScoreV1 = BaseScoreV1,\n> = LoaderResult<TTestCase, TResponse, TScore>;\n\nexport abstract class GenericJSONArrayDataLoader<\n  TTestCase extends BaseTestCaseV1 = BaseTestCaseV1,\n  TResponse extends BaseResponseV1 = BaseResponseV1,\n  TScore extends BaseScoreV1 = BaseScoreV1,\n> extends AbstractDataLoader {\n  protected abstract testCaseBuilder(\n    data: any,\n    context: {\n      result: GenericJSONArrayLoaderResult<TTestCase, TResponse, TScore>;\n    }\n  ): MaybePromise<TTestCase | undefined>;\n\n  protected async responseBuilder(\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    data: any,\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    context: {\n      result: GenericJSONArrayLoaderResult<TTestCase, TResponse, TScore>;\n    }\n  ): Promise<TResponse | undefined> {\n    return undefined;\n  }\n\n  protected async scoreBuilder(\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    data: any,\n    // eslint-disable-next-line @typescript-eslint/no-unused-vars\n    context: {\n      result: GenericJSONArrayLoaderResult<TTestCase, TResponse, TScore>;\n    }\n  ): Promise<TScore | undefined> {\n    return undefined;\n  }\n\n  async loadData(params: {\n    content: Uint8Array;\n  }): Promise<GenericJSONArrayLoaderResult<TTestCase, TResponse, TScore>> {\n    const contentStr = bufferToString(params.content);\n    let data: unknown[] | undefined = tryParseJson<unknown[]>(contentStr);\n\n    if (!data) {\n      data = parseJSONL<unknown>(contentStr);\n    }\n\n    if (!data || !Array.isArray(data) || data.length === 0) {\n      throw new Error(\n        \"Invalid data: content must be a non-empty JSON or JSONL array\"\n      );\n    }\n\n    return this.transformArrayToResult(data);\n  }\n\n  private async transformArrayToResult(\n    data: unknown[]\n  ): Promise<GenericJSONArrayLoaderResult<TTestCase, TResponse, TScore>> {\n    const includedTestCaseIds: Set<string> = new Set();\n    const includedResponseIds: Set<string> = new Set();\n    const includedScoreIds: Set<string> = new Set();\n    const result: GenericJSONArrayLoaderResult<TTestCase, TResponse, TScore> = {\n      testCases: [],\n      responses: [],\n      scores: [],\n    };\n\n    for (const item of data) {\n      const testCase = await this.testCaseBuilder(item, { result });\n      const response = await this.responseBuilder(item, { result });\n      const score = await this.scoreBuilder(item, { result });\n\n      if (testCase) {\n        const testCaseIdStr = String(testCase.id);\n        if (!includedTestCaseIds.has(testCaseIdStr)) {\n          includedTestCaseIds.add(testCaseIdStr);\n          result.testCases.push(testCase);\n        }\n      }\n\n      if (response) {\n        const responseIdStr = String(response.id);\n        if (!includedResponseIds.has(responseIdStr)) {\n          includedResponseIds.add(responseIdStr);\n          result.responses.push(response);\n        }\n      }\n\n      if (score) {\n        const scoreIdStr = String(score.id);\n        if (!includedScoreIds.has(scoreIdStr)) {\n          includedScoreIds.add(scoreIdStr);\n          result.scores.push(score);\n        }\n      }\n\n      if (!testCase && !response && !score) {\n        throw new Error(\"Incompatible object format\");\n      }\n    }\n\n    return result;\n  }\n}\n"],"mappings":";;;;;AAiBO,IAAe,iBAAf,MAA8B;AAOrC;;;ACpBO,IAAe,qBAAf,cAA0C,eAAe;AAQhE;;;ACZA,SAAS,iBAAiB;AAMnB,SAAS,WACd,KACA,SACK;AACL,SAAO,IACJ,MAAM,IAAI,EACV,OAAO,CAAC,SAAS,KAAK,KAAK,MAAM,EAAE,EACnC,IAAI,CAAC,SAAS;AACb,UAAM,MAAM,aAAa,IAAI;AAC7B,QAAI,SAAS,kBAAkB,CAAC,KAAK;AACnC,YAAM,IAAI,MAAM,sBAAsB,IAAI,EAAE;AAAA,IAC9C;AACA,WAAO;AAAA,EACT,CAAC,EACA,OAAO,CAAC,QAAQ,QAAQ,MAAS;AACtC;AAMO,SAAS,aAAsB,SAAgC;AACpE,MAAI;AACF,WAAO,KAAK,MAAM,OAAO;AAAA,EAC3B,QAAQ;AAAA,EAER;AACF;AAMO,SAAS,gBAAgB,OAAY;AAC1C,SAAO,UAAU,KAAK;AACxB;;;AC1BO,IAAe,6BAAf,cAIG,mBAAmB;AAAA,EAQ3B,MAAgB,gBAEd,MAEA,SAGgC;AAChC,WAAO;AAAA,EACT;AAAA,EAEA,MAAgB,aAEd,MAEA,SAG6B;AAC7B,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,SAAS,QAEyD;AACtE,UAAM,aAAa,eAAe,OAAO,OAAO;AAChD,QAAI,OAA8B,aAAwB,UAAU;AAEpE,QAAI,CAAC,MAAM;AACT,aAAO,WAAoB,UAAU;AAAA,IACvC;AAEA,QAAI,CAAC,QAAQ,CAAC,MAAM,QAAQ,IAAI,KAAK,KAAK,WAAW,GAAG;AACtD,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,WAAO,KAAK,uBAAuB,IAAI;AAAA,EACzC;AAAA,EAEA,MAAc,uBACZ,MACqE;AACrE,UAAM,sBAAmC,oBAAI,IAAI;AACjD,UAAM,sBAAmC,oBAAI,IAAI;AACjD,UAAM,mBAAgC,oBAAI,IAAI;AAC9C,UAAM,SAAqE;AAAA,MACzE,WAAW,CAAC;AAAA,MACZ,WAAW,CAAC;AAAA,MACZ,QAAQ,CAAC;AAAA,IACX;AAEA,eAAW,QAAQ,MAAM;AACvB,YAAM,WAAW,MAAM,KAAK,gBAAgB,MAAM,EAAE,OAAO,CAAC;AAC5D,YAAM,WAAW,MAAM,KAAK,gBAAgB,MAAM,EAAE,OAAO,CAAC;AAC5D,YAAM,QAAQ,MAAM,KAAK,aAAa,MAAM,EAAE,OAAO,CAAC;AAEtD,UAAI,UAAU;AACZ,cAAM,gBAAgB,OAAO,SAAS,EAAE;AACxC,YAAI,CAAC,oBAAoB,IAAI,aAAa,GAAG;AAC3C,8BAAoB,IAAI,aAAa;AACrC,iBAAO,UAAU,KAAK,QAAQ;AAAA,QAChC;AAAA,MACF;AAEA,UAAI,UAAU;AACZ,cAAM,gBAAgB,OAAO,SAAS,EAAE;AACxC,YAAI,CAAC,oBAAoB,IAAI,aAAa,GAAG;AAC3C,8BAAoB,IAAI,aAAa;AACrC,iBAAO,UAAU,KAAK,QAAQ;AAAA,QAChC;AAAA,MACF;AAEA,UAAI,OAAO;AACT,cAAM,aAAa,OAAO,MAAM,EAAE;AAClC,YAAI,CAAC,iBAAiB,IAAI,UAAU,GAAG;AACrC,2BAAiB,IAAI,UAAU;AAC/B,iBAAO,OAAO,KAAK,KAAK;AAAA,QAC1B;AAAA,MACF;AAEA,UAAI,CAAC,YAAY,CAAC,YAAY,CAAC,OAAO;AACpC,cAAM,IAAI,MAAM,4BAA4B;AAAA,MAC9C;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AACF;","names":[]}

package/dist/chunk-IUN2IUCS.js

@@ -1,58 +0,0 @@
-// src/schemas/id.ts
-import z from "zod";
-var IdSchema = z.string();
-
-// src/schemas/schema-definer.ts
-import z2 from "zod";
-function buildSchemaDefiner() {
-  return function(config) {
-    const schema = config.baseSchema.omit({ kind: true, schemaVersion: true }).extend({
-      ...config.fields ?? {},
-      kind: config.kind !== void 0 ? z2.literal(config.kind) : config.baseSchema.shape.kind,
-      schemaVersion: config.schemaVersion !== void 0 ? z2.literal(config.schemaVersion) : config.baseSchema.shape.schemaVersion
-    });
-    return Object.assign(schema, {
-      new(input) {
-        return schema.parse({
-          ...input,
-          kind: config.kind,
-          schemaVersion: config.schemaVersion
-        });
-      },
-      async newWithId(input, generator) {
-        const obj = schema.parse({
-          ...input,
-          id: "",
-          kind: config.kind,
-          schemaVersion: config.schemaVersion
-        });
-        const id = await generator(obj);
-        return {
-          ...obj,
-          id
-        };
-      }
-    });
-  };
-}
-
-// src/schemas/response.ts
-import z3 from "zod";
-var BaseResponseSchemaV1 = z3.object({
-  id: IdSchema,
-  kind: z3.string(),
-  schemaVersion: z3.number(),
-  startedAt: z3.number(),
-  completedAt: z3.number(),
-  testCaseId: IdSchema,
-  metadata: z3.record(z3.string(), z3.unknown()).optional()
-});
-var defineResponseSchema = buildSchemaDefiner();
-
-export {
-  buildSchemaDefiner,
-  IdSchema,
-  BaseResponseSchemaV1,
-  defineResponseSchema
-};
-//# sourceMappingURL=chunk-IUN2IUCS.js.map

package/dist/chunk-IUN2IUCS.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/schemas/id.ts","../src/schemas/schema-definer.ts","../src/schemas/response.ts"],"sourcesContent":["import z from \"zod\";\n\nexport const IdSchema = z.string();\n","import { IdGenerator } from \"@/types\";\nimport z from \"zod\";\n\nexport function buildSchemaDefiner<\n  TBaseShape extends {\n    kind: z.ZodType;\n    schemaVersion: z.ZodType;\n    [key: string]: z.ZodType;\n  },\n>() {\n  return function <\n    TBaseSchema extends z.ZodObject<\n      Omit<TBaseShape, \"kind\" | \"schemaVersion\"> & {\n        kind: z.ZodString | z.ZodLiteral<string>;\n        schemaVersion: z.ZodNumber | z.ZodLiteral<number>;\n      }\n    >,\n    TKind extends string | undefined = undefined,\n    TSchemaVersion extends number | undefined = undefined,\n    TFields extends z.ZodRawShape = {},\n  >(config: {\n    baseSchema: TBaseSchema;\n    kind?: TKind;\n    schemaVersion?: TSchemaVersion;\n    fields?: TFields;\n  }) {\n    const schema = config.baseSchema\n      .omit({ kind: true, schemaVersion: true })\n      .extend({\n        ...((config.fields ?? {}) as TFields),\n        kind: (config.kind !== undefined\n          ? z.literal(config.kind)\n          : config.baseSchema.shape.kind) as TKind extends undefined\n          ? TBaseSchema[\"shape\"][\"kind\"]\n          : z.ZodLiteral<TKind>,\n        schemaVersion: (config.schemaVersion !== undefined\n          ? z.literal(config.schemaVersion)\n          : config.baseSchema.shape\n              .schemaVersion) as TSchemaVersion extends undefined\n          ? TBaseSchema[\"shape\"][\"schemaVersion\"]\n          : z.ZodLiteral<TSchemaVersion>,\n      });\n\n    type SchemaType =\n      TBaseSchema extends z.ZodObject<infer U>\n        ? z.ZodObject<\n            Omit<U, \"kind\" | \"schemaVersion\"> &\n              TFields & {\n                kind: TKind extends undefined ? U[\"kind\"] : z.ZodLiteral<TKind>;\n                schemaVersion: TSchemaVersion extends undefined\n                  ? U[\"schemaVersion\"]\n                  : z.ZodLiteral<TSchemaVersion>;\n              }\n          >\n        : never;\n\n    return Object.assign(schema, {\n      new(input: Omit<z.infer<typeof schema>, \"kind\" | \"schemaVersion\">) {\n        return schema.parse({\n          ...input,\n          kind: config.kind,\n          schemaVersion: config.schemaVersion,\n        });\n      },\n      async newWithId(\n        input: Omit<z.infer<typeof schema>, \"kind\" | \"schemaVersion\" | \"id\">,\n        generator: IdGenerator\n      ) {\n        const obj = schema.parse({\n          ...input,\n          id: \"\",\n          kind: config.kind,\n          schemaVersion: config.schemaVersion,\n        });\n        const id = await generator(obj);\n\n        return {\n          ...obj,\n          id,\n        };\n      },\n    }) as unknown as SchemaType & {\n      /**\n       * Creates a new object with the given input. Uses the `kind` and\n       * `schemaVersion` from the schema config. Although the `input` is already typed,\n       * it still uses `.parse()` to validate it.\n       * @param input\n       * @returns The object that follows the schema\n       */\n      new: (\n        input: Omit<z.infer<SchemaType>, \"kind\" | \"schemaVersion\">\n      ) => z.infer<SchemaType>;\n\n      newWithId(\n        input: Omit<z.infer<SchemaType>, \"id\" | \"kind\" | \"schemaVersion\">,\n        generator: IdGenerator\n      ): Promise<z.infer<SchemaType>>;\n    };\n  };\n}\n","import { IdSchema } from \"./id\";\nimport { buildSchemaDefiner } from \"./schema-definer\";\nimport z from \"zod\";\n\nexport const BaseResponseSchemaV1 = z.object({\n  id: IdSchema,\n  kind: z.string(),\n  schemaVersion: z.number(),\n\n  startedAt: z.number(),\n  completedAt: z.number(),\n  testCaseId: IdSchema,\n  metadata: z.record(z.string(), z.unknown()).optional(),\n});\nexport type BaseResponseV1 = z.infer<typeof BaseResponseSchemaV1>;\n\nexport const defineResponseSchema =\n  buildSchemaDefiner<typeof BaseResponseSchemaV1.shape>();\n"],"mappings":";AAAA,OAAO,OAAO;AAEP,IAAM,WAAW,EAAE,OAAO;;;ACDjC,OAAOA,QAAO;AAEP,SAAS,qBAMZ;AACF,SAAO,SAUL,QAKC;AACD,UAAM,SAAS,OAAO,WACnB,KAAK,EAAE,MAAM,MAAM,eAAe,KAAK,CAAC,EACxC,OAAO;AAAA,MACN,GAAK,OAAO,UAAU,CAAC;AAAA,MACvB,MAAO,OAAO,SAAS,SACnBA,GAAE,QAAQ,OAAO,IAAI,IACrB,OAAO,WAAW,MAAM;AAAA,MAG5B,eAAgB,OAAO,kBAAkB,SACrCA,GAAE,QAAQ,OAAO,aAAa,IAC9B,OAAO,WAAW,MACf;AAAA,IAGT,CAAC;AAeH,WAAO,OAAO,OAAO,QAAQ;AAAA,MAC3B,IAAI,OAA+D;AACjE,eAAO,OAAO,MAAM;AAAA,UAClB,GAAG;AAAA,UACH,MAAM,OAAO;AAAA,UACb,eAAe,OAAO;AAAA,QACxB,CAAC;AAAA,MACH;AAAA,MACA,MAAM,UACJ,OACA,WACA;AACA,cAAM,MAAM,OAAO,MAAM;AAAA,UACvB,GAAG;AAAA,UACH,IAAI;AAAA,UACJ,MAAM,OAAO;AAAA,UACb,eAAe,OAAO;AAAA,QACxB,CAAC;AACD,cAAM,KAAK,MAAM,UAAU,GAAG;AAE9B,eAAO;AAAA,UACL,GAAG;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EAiBH;AACF;;;ACjGA,OAAOC,QAAO;AAEP,IAAM,uBAAuBA,GAAE,OAAO;AAAA,EAC3C,IAAI;AAAA,EACJ,MAAMA,GAAE,OAAO;AAAA,EACf,eAAeA,GAAE,OAAO;AAAA,EAExB,WAAWA,GAAE,OAAO;AAAA,EACpB,aAAaA,GAAE,OAAO;AAAA,EACtB,YAAY;AAAA,EACZ,UAAUA,GAAE,OAAOA,GAAE,OAAO,GAAGA,GAAE,QAAQ,CAAC,EAAE,SAAS;AACvD,CAAC;AAGM,IAAM,uBACX,mBAAsD;","names":["z","z"]}