peerbench 0.0.1 → 0.0.2-alpha-dev.0

package/dist/index.js

@@ -1,1091 +1,22 @@
-// src/types.ts
-import { z as z2 } from "zod";
-
-// src/validation/did.ts
-import { z } from "zod";
-
-// src/utils/json.ts
-import { stringify } from "safe-stable-stringify";
-function parseJSONL(str, options) {
-  return str.split("\n").filter((line) => line.trim() !== "").map((line) => {
-    const obj = tryParseJson(line);
-    if (options?.errorOnInvalid && !obj) {
-      throw new Error(`Invalid JSON line: ${line}`);
-    }
-    return obj;
-  }).filter((obj) => obj !== void 0);
-}
-function tryParseJson(content) {
-  try {
-    return JSON.parse(content);
-  } catch {
-  }
-}
-function stableStringify(value) {
-  return stringify(value);
-}
-
-// src/utils/cid.ts
-import { CID } from "multiformats/cid";
-import { sha256 } from "multiformats/hashes/sha2";
-import * as json from "multiformats/codecs/json";
-async function calculateCID(data) {
-  const bytes = json.encode(data);
-  const hash = await sha256.digest(bytes);
-  const cid = CID.create(1, json.code, hash);
-  return cid;
-}
-
-// src/utils/sleep.ts
-async function sleep(ms) {
-  return await new Promise((res) => setTimeout(res, ms));
-}
-
-// src/utils/file.ts
-async function readFile(path) {
-  if (typeof window === "undefined") {
-    const { readFileSync, statSync } = await import("node:fs");
-    if (!statSync(path, { throwIfNoEntry: false })?.isFile()) {
-      throw new Error(`File doesn't exist: ${path}`);
-    }
-    return readFileSync(path);
-  } else {
-    throw new Error(
-      "File system operations are not supported in browser environment. Use readFromContent instead."
-    );
-  }
-}
-
-// src/utils/prompt.ts
-function preparePrompt(question, options = {}) {
-  if (options && Object.keys(options).length === 0) {
-    return question;
-  }
-  let fullPrompt = `${question}
-
-`;
-  for (const [letter, answer] of Object.entries(options)) {
-    fullPrompt += `${letter}: ${answer}
-`;
-  }
-  return fullPrompt;
-}
-
-// src/utils/sha256.ts
-async function calculateSHA256(input) {
-  if (typeof window !== "undefined" && window.crypto) {
-    let data;
-    if (input instanceof Uint8Array) {
-      data = input;
-    } else {
-      const encoder = new TextEncoder();
-      data = encoder.encode(input);
-    }
-    const buffer = await window.crypto.subtle.digest("SHA-256", data);
-    const hashArray = Array.from(new Uint8Array(buffer));
-    return hashArray.map((b) => b.toString(16).padStart(2, "0")).join("");
-  } else {
-    const { createHash } = await import("crypto");
-    if (input instanceof Uint8Array) {
-      return createHash("sha256").update(Buffer.from(input)).digest("hex");
-    }
-    return createHash("sha256").update(input).digest("hex");
-  }
-}
-
-// src/utils/did.ts
-function removeDIDPrefix(did) {
-  if (!did.startsWith("did:")) {
-    return did;
-  }
-  const parts = did.split(":");
-  if (parts.length < 3) {
-    return did;
-  }
-  return parts.slice(2).join(":");
-}
-
-// src/utils/string.ts
-function formatMs(ms, options) {
-  const {
-    full = false,
-    include = ["millisecond", "second", "minute", "hour", "day"]
-  } = options ?? {};
-  const totalSeconds = ms / 1e3;
-  if (full) {
-    const days = Math.floor(totalSeconds / 86400);
-    const hours = Math.floor(totalSeconds % 86400 / 3600);
-    const minutes = Math.floor(totalSeconds % 3600 / 60);
-    const seconds = Math.floor(totalSeconds % 60);
-    const milliseconds = ms % 1e3;
-    const parts = [];
-    if (days > 0 && include.includes("day")) {
-      parts.push(days + (days === 1 ? " day" : " days"));
-    }
-    if (hours > 0 && include.includes("hour")) {
-      parts.push(hours + (hours === 1 ? " hour" : " hours"));
-    }
-    if (minutes > 0 && include.includes("minute")) {
-      parts.push(minutes + (minutes === 1 ? " minute" : " minutes"));
-    }
-    if (seconds > 0 && include.includes("second")) {
-      parts.push(seconds + (seconds === 1 ? " second" : " seconds"));
-    }
-    if (milliseconds > 0 && include.includes("millisecond")) {
-      parts.push(milliseconds + "ms");
-    }
-    return parts.join(", ");
-  }
-  if (totalSeconds >= 86400) {
-    const days = Math.floor(totalSeconds / 86400);
-    return days.toFixed(2) + (days === 1 ? " day" : " days");
-  } else if (totalSeconds >= 3600) {
-    const hours = Math.floor(totalSeconds / 3600);
-    return hours.toFixed(0) + (hours === 1 ? " hour" : " hours");
-  } else if (totalSeconds >= 60) {
-    const minutes = Math.floor(totalSeconds / 60);
-    return minutes.toFixed(0) + (minutes === 1 ? " minute" : " minutes");
-  } else if (totalSeconds >= 1) {
-    return totalSeconds.toFixed(0) + (totalSeconds === 1 ? " second" : " seconds");
-  } else {
-    return ms.toFixed(0) + "ms";
-  }
-}
-function bufferToString(buffer) {
-  const decoder = new TextDecoder();
-  return decoder.decode(buffer);
-}
-function stringToBuffer(str) {
-  const encoder = new TextEncoder();
-  return encoder.encode(str);
-}
-function formatString(str, values) {
-  return str.replace(/\{(\w+)\}/g, (_, key) => values[key] ?? `{${key}}`);
-}
-
-// src/utils/llm.ts
-import { jsonrepair } from "jsonrepair";
-function parseResponseAsJSON(response) {
-  try {
-    const json2 = extractJSONFromResponse(response);
-    if (!json2) {
-      throw new Error("No JSON found in the response");
-    }
-    return JSON.parse(jsonrepair(json2));
-  } catch (e) {
-    if (process?.env?.PB_SDK_DEBUG) {
-      console.log("Original response", JSON.stringify(response));
-      console.error("Error parsing response as JSON", e);
-    }
-  }
-}
-function extractJSONFromResponse(response) {
-  const jsonRegex = /```(json)?\n*(?<content>.*?)\n*```/s;
-  const jsonMatch = response.match(jsonRegex);
-  return jsonMatch?.groups?.content;
-}
-
-// src/utils/validation.ts
-function parseValidationError(safeParseReturn, path) {
-  path ??= "";
-  if (safeParseReturn?.error) {
-    const firstError = safeParseReturn.error.errors[0];
-    if (path) {
-      path = `${path}: `;
-    }
-    path = firstError.path.length > 0 ? `"${firstError.path.join(".")}": ` : path;
-    return `${path}${firstError.message}`;
-  }
-}
-function checkValidationError(safeParseReturn, path) {
-  if (safeParseReturn?.error) {
-    throw new Error(parseValidationError(safeParseReturn, path));
-  }
-  return safeParseReturn.data;
-}
-
-// src/utils/builder.ts
-import { v7 as uuidv7 } from "uuid";
-async function buildPrompt(params) {
-  return PromptSchema.parse({
-    did: params.did ?? uuidv7(),
-    options: params.options ?? void 0,
-    question: {
-      data: params.question,
-      cid: await calculateCID(params.question).then((c) => c.toString()),
-      sha256: await calculateSHA256(params.question)
-    },
-    fullPrompt: {
-      data: params.fullPrompt ?? params.question,
-      cid: await calculateCID(params.fullPrompt ?? params.question).then(
-        (c) => c.toString()
-      ),
-      sha256: await calculateSHA256(params.fullPrompt ?? params.question)
-    },
-    type: params.type,
-    answer: params.answer ?? void 0,
-    answerKey: params.answerKey ?? void 0,
-    metadata: params.metadata ?? void 0,
-    scorers: params.scorers ?? void 0
-  });
-}
-async function buildResponse(params) {
-  return PromptResponseSchema.parse({
-    did: params.did ?? uuidv7(),
-    runId: params.runId ?? uuidv7(),
-    data: params.forwardResponse.data,
-    sha256: await calculateSHA256(params.forwardResponse.data),
-    cid: await calculateCID(params.forwardResponse.data).then(
-      (c) => c.toString()
-    ),
-    startedAt: params.forwardResponse.startedAt.getTime(),
-    finishedAt: params.forwardResponse.completedAt.getTime(),
-    prompt: params.prompt,
-    metadata: params.metadata,
-    provider: params.provider,
-    modelId: params.modelId,
-    modelName: params.modelName || "unknown",
-    modelOwner: params.modelOwner || "unknown",
-    modelHost: params.modelHost || "auto",
-    inputTokensUsed: params.forwardResponse.inputTokensUsed,
-    inputCost: params.forwardResponse.inputCost,
-    outputTokensUsed: params.forwardResponse.outputTokensUsed,
-    outputCost: params.forwardResponse.outputCost
-  });
-}
-async function buildScore(params) {
-  return PromptScoreSchema.parse({
-    ...params,
-    scoreDID: params.scoreDID ?? uuidv7()
-  });
-}
-
-// src/validation/did.ts
-function DIDSchema(schema) {
-  return z.string().transform((val) => removeDIDPrefix(val)).pipe(schema);
-}
-var DIDasUUIDSchema = DIDSchema(
-  z.string().uuid({ message: "Invalid DID" })
-);
-
-// src/types.ts
-var PromptTypes = {
-  MultipleChoice: "multiple-choice",
-  OrderSentences: "order-sentences",
-  TextReplacement: "text-replacement",
-  Typo: "typo",
-  OpenEnded: "open-ended",
-  OpenEndedWithDocs: "open-ended-with-docs"
-};
-var ScoringMethods = {
-  /**
-   * Scored by a human
-   */
-  human: "human",
-  /**
-   * Scored using an AI model
-   */
-  ai: "ai",
-  /**
-   * Scored using an algorithm
-   */
-  algo: "algo"
-};
-var PromptSchema = z2.object({
-  /**
-   * Decentralized identifier of the Prompt
-   */
-  did: DIDasUUIDSchema,
-  /**
-   * The question that is going to be asked to the model
-   */
-  question: z2.object({
-    /**
-     * Question data itself
-     */
-    data: z2.string(),
-    /**
-     * CID v1 calculation of the question string
-     */
-    cid: z2.string(),
-    /**
-     * SHA256 hash of the question string
-     */
-    sha256: z2.string()
-  }),
-  /**
-   * Multiple choice answers for the question where the keys are letters and the values are the answers.
-   */
-  options: z2.record(z2.string(), z2.string()).optional(),
-  /**
-   * Full prompt that is going to be sent to the model
-   */
-  fullPrompt: z2.object({
-    /**
-     * Full prompt itself
-     */
-    data: z2.string(),
-    /**
-     * CID v1 calculation of the full prompt string
-     */
-    cid: z2.string(),
-    /**
-     * SHA256 hash of the full prompt string
-     */
-    sha256: z2.string()
-  }),
-  /**
-   * Type of the Prompt
-   */
-  type: z2.nativeEnum(PromptTypes),
-  /**
-   * Expected option value for the question
-   */
-  answer: z2.string().optional(),
-  /**
-   * Expected letter of the answer (e.g "A", "B" or "C")
-   */
-  answerKey: z2.string().optional(),
-  /**
-   * Additional metadata related to the Prompt
-   */
-  metadata: z2.record(z2.string(), z2.any()).optional(),
-  /**
-   * Expected Scorer identifiers that can be used to
-   * score the Responses for this Prompt
-   */
-  scorers: z2.array(z2.string()).optional()
-}).transform((prompt, ctx) => {
-  if (prompt.type === PromptTypes.MultipleChoice) {
-    if (Object.keys(prompt.options || {}).length === 0) {
-      ctx.addIssue({
-        code: z2.ZodIssueCode.custom,
-        message: "No options provided for multiple choice question"
-      });
-      return z2.NEVER;
-    }
-    if (Object.values(prompt.options || {}).some(
-      (value) => value?.trim() === ""
-    )) {
-      ctx.addIssue({
-        code: z2.ZodIssueCode.custom,
-        message: "Multiple choice options cannot be empty"
-      });
-      return z2.NEVER;
-    }
-    if (!prompt.answerKey) {
-      ctx.addIssue({
-        code: z2.ZodIssueCode.custom,
-        message: "Correct answer key cannot be empty"
-      });
-      return z2.NEVER;
-    }
-    if (!prompt.answer) {
-      ctx.addIssue({
-        code: z2.ZodIssueCode.custom,
-        message: "Correct answer value cannot be empty"
-      });
-      return z2.NEVER;
-    }
-  }
-  return prompt;
-});
-var TaskSchema = z2.object({
-  /**
-   * Decentralized identifier of the Task
-   */
-  did: z2.string().startsWith("did:task:"),
-  /**
-   * The Prompts that the Task has
-   */
-  prompts: z2.array(PromptSchema),
-  /**
-   * CID v1 calculation of the Task file
-   */
-  cid: z2.string(),
-  /**
-   * SHA256 calculation of the Task file
-   */
-  sha256: z2.string(),
-  /**
-   * Basename of the Task file
-   */
-  fileName: z2.string(),
-  /**
-   * Full path of the Task file
-   */
-  path: z2.string()
-});
-var PromptResponseSchema = z2.object({
-  /**
-   * Unique identifier of the Response
-   */
-  did: DIDasUUIDSchema,
-  /**
-   * Name of the Provider that the Response comes from
-   */
-  provider: z2.string(),
-  /**
-   * ID of the Model that was used by the Provider
-   */
-  modelId: z2.string(),
-  /**
-   * Known name of the model by peerBench
-   */
-  modelName: z2.string(),
-  /**
-   * Owner of the model
-   */
-  modelOwner: z2.string(),
-  /**
-   * The entity that responsible for hosting the model
-   */
-  modelHost: z2.string(),
-  /**
-   * The Prompt that used to achieve this Response.
-   */
-  prompt: PromptSchema,
-  /**
-   * CID v1 calculation of the Response data.
-   */
-  cid: z2.string(),
-  /**
-   * SHA256 calculation of the Response data.
-   */
-  sha256: z2.string(),
-  /**
-   * Response data itself.
-   */
-  data: z2.string(),
-  /**
-   * Timestamp when the Prompt sent to the Model
-   */
-  startedAt: z2.number(),
-  /**
-   * Timestamp when the Model responded this particular Prompt.
-   */
-  finishedAt: z2.number(),
-  /**
-   * Unique identifier of which run this Response belongs to
-   */
-  runId: z2.string(),
-  inputTokensUsed: z2.number().optional(),
-  outputTokensUsed: z2.number().optional(),
-  inputCost: z2.string().optional(),
-  outputCost: z2.string().optional(),
-  metadata: z2.record(z2.string(), z2.any()).optional()
-});
-var PromptScoreSchema = PromptResponseSchema.extend({
-  prompt: (
-    // Modify some of the fields of the Prompt in case
-    // if the Score object doesn't want to include original Prompt data
-    PromptSchema.sourceType().extend({
-      options: PromptSchema.sourceType().shape.options.optional(),
-      question: PromptSchema.sourceType().shape.question.extend({
-        data: PromptSchema.sourceType().shape.question.shape.data.optional()
-      }),
-      fullPrompt: PromptSchema.sourceType().shape.fullPrompt.extend({
-        data: PromptSchema.sourceType().shape.fullPrompt.shape.data.optional()
-      }),
-      type: PromptSchema.sourceType().shape.type.optional(),
-      answer: PromptSchema.sourceType().shape.answer.optional(),
-      answerKey: PromptSchema.sourceType().shape.answerKey.optional()
-    }).optional()
-  ),
-  data: z2.string().optional(),
-  /**
-   * Unique identifier of this Scoring result. This is named
-   * like this because `did` field represents the Response ID since
-   * the Score object inherits from the Response object.
-   */
-  scoreDID: DIDasUUIDSchema,
-  /**
-   * Additional metadata about the Scoring result. This is named
-   * like this because `metadata` field represents the Response metadata since
-   * the Score object inherits from the Response object.
-   */
-  scoreMetadata: z2.record(z2.string(), z2.any()).optional(),
-  score: z2.number().min(0).max(1),
-  method: z2.nativeEnum(ScoringMethods),
-  /**
-   * Explanation about how the score was calculated.
-   */
-  explanation: z2.string().optional(),
-  // Only presented if the scoring method is `ai`
-  scorerAI: z2.object({
-    provider: z2.string(),
-    modelName: z2.string(),
-    modelHost: z2.string(),
-    modelOwner: z2.string(),
-    modelId: z2.string(),
-    inputTokensUsed: z2.number().optional(),
-    outputTokensUsed: z2.number().optional(),
-    inputCost: z2.string().optional(),
-    outputCost: z2.string().optional()
-  }).optional()
-});
-
-// src/collectors/abstract/abstract-collector.ts
-var AbstractCollector = class {
-  /**
-   * Initializes the collector (depends on the implementation)
-   */
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  async initialize(...args) {
-  }
-};
-
-// src/collectors/abstract/abstract-rss-collector.ts
-import { XMLParser } from "fast-xml-parser";
-var AbstractRSSCollector = class extends AbstractCollector {
-  parser;
-  constructor() {
-    super();
-    this.parser = new XMLParser({});
-  }
-  /**
-   * Helper function to fetch the feed from the URL
-   * @param url The URL of the feed
-   * @returns Raw XML string of the feed
-   */
-  async fetchFeed(url) {
-    const response = await fetch(url);
-    if (!response.ok) {
-      throw new Error(`HTTP error! status: ${response.status}`);
-    }
-    return response.text();
-  }
-  /**
-   * Parses the raw XML string and validates it
-   * against the defined `FeedStructure` schema
-   * @param xml The raw XML string
-   * @returns Structured data of the RSS feed
-   */
-  async parseFeedXML(xml) {
-    const parsedData = this.parser.parse(xml);
-    return await this.feedSchema.parseAsync(parsedData);
-  }
-};
-
-// src/collectors/pubmed-collector.ts
-import { z as z3 } from "zod";
-import * as cheerio from "cheerio";
-var PubMedCollector = class extends AbstractRSSCollector {
-  identifier = "pubmed";
-  feedSchema = z3.object({
-    rss: z3.object({
-      channel: z3.object({
-        title: z3.string(),
-        link: z3.string(),
-        description: z3.string(),
-        pubDate: z3.string(),
-        lastBuildDate: z3.string().optional(),
-        language: z3.string().optional(),
-        item: z3.any().transform((data) => Array.isArray(data) ? data : [data]).pipe(
-          z3.array(
-            z3.object({
-              title: z3.string(),
-              link: z3.string(),
-              description: z3.string(),
-              guid: z3.string(),
-              pubDate: z3.string(),
-              "content:encoded": z3.string()
-            })
-          )
-        )
-      })
-    })
-  });
-  async collect(url) {
-    const feed = await this.parseFeedXML(await this.fetchFeed(url));
-    const parsed = [];
-    const tags = /* @__PURE__ */ new Set();
-    tags.add("pubmed");
-    for (const article of feed.rss.channel.item) {
-      const $ = cheerio.load(article["content:encoded"]);
-      const elements = $("p");
-      const texts = [];
-      const paragraphs = {};
-      elements.each((i, el) => {
-        if (i === 0) return;
-        const text = $(el).text().trim();
-        if (text === "" || text === "ABSTRACT" || text.startsWith("PMID:") || text.startsWith("DOI:")) {
-          return;
-        }
-        texts.push(text);
-      });
-      if (this.parseParagraphTitle(texts[0])) {
-        tags.add(feed.rss.channel.title);
-        for (const text of texts) {
-          const match = this.parseParagraphTitle(text);
-          if (match) {
-            paragraphs[match[1].trim()] = match[2].trim();
-          }
-        }
-        parsed.push({
-          pmid: article.guid,
-          title: article.title,
-          paragraphs,
-          tags: Array.from(tags)
-        });
-      }
-    }
-    return parsed;
-  }
-  /**
-   * Parses the title of a paragraph which
-   * is in the format of "TITLE:"
-   */
-  parseParagraphTitle(text) {
-    const paragraphRegex = /^([A-Z|\s]+):\s*(.*)/;
-    const match = text.match(paragraphRegex);
-    return match;
-  }
-};
-
-// src/collectors/simple-general-rss-collector.ts
-import { z as z4 } from "zod";
-import * as cheerio2 from "cheerio";
-var SimpleGeneralRSSCollector = class extends AbstractRSSCollector {
-  identifier = "simple-general-rss";
-  feedSchema = z4.object({
-    // Handle both RSS and RDF formats
-    rss: z4.object({
-      channel: z4.object({
-        title: z4.string().optional(),
-        link: z4.string().optional(),
-        description: z4.string().optional(),
-        pubDate: z4.string().optional(),
-        lastBuildDate: z4.string().optional(),
-        language: z4.string().optional(),
-        item: z4.any().transform((data) => Array.isArray(data) ? data : [data]).pipe(
-          z4.array(
-            z4.object({
-              title: z4.string(),
-              link: z4.string().optional(),
-              description: z4.string().optional(),
-              guid: z4.union([z4.string(), z4.object({}).passthrough()]).optional(),
-              pubDate: z4.string().optional(),
-              "content:encoded": z4.string().optional(),
-              // Handle various RSS namespaces and extensions
-              "dc:creator": z4.union([z4.string(), z4.array(z4.string())]).optional(),
-              "dc:date": z4.string().optional(),
-              "dc:title": z4.string().optional(),
-              "dc:subject": z4.string().optional(),
-              "dc:publisher": z4.string().optional(),
-              "dc:identifier": z4.union([z4.string(), z4.array(z4.string())]).optional(),
-              "dc:rights": z4.string().optional(),
-              "dc:source": z4.string().optional(),
-              "prism:publicationDate": z4.string().optional(),
-              "prism:section": z4.string().optional(),
-              "prism:volume": z4.string().optional(),
-              "prism:number": z4.string().optional(),
-              "prism:startingPage": z4.string().optional(),
-              "prism:endingPage": z4.string().optional(),
-              "arxiv:announce_type": z4.string().optional(),
-              category: z4.union([z4.string(), z4.array(z4.string())]).optional()
-            })
-          )
-        )
-      })
-    }).optional(),
-    // Handle RDF format (like medRxiv)
-    rdf: z4.object({
-      channel: z4.object({
-        title: z4.string().optional(),
-        link: z4.string().optional(),
-        description: z4.string().optional(),
-        items: z4.object({
-          "rdf:Seq": z4.object({
-            "rdf:li": z4.any().optional()
-          }).optional()
-        }).optional(),
-        item: z4.any().transform((data) => Array.isArray(data) ? data : [data]).pipe(
-          z4.array(
-            z4.object({
-              title: z4.string(),
-              link: z4.string().optional(),
-              description: z4.string().optional(),
-              "dc:creator": z4.union([z4.string(), z4.array(z4.string())]).optional(),
-              "dc:date": z4.string().optional(),
-              "dc:title": z4.string().optional(),
-              "dc:subject": z4.string().optional(),
-              "dc:publisher": z4.string().optional(),
-              "dc:identifier": z4.union([z4.string(), z4.array(z4.string())]).optional(),
-              "dc:rights": z4.string().optional(),
-              "dc:source": z4.string().optional(),
-              "prism:publicationDate": z4.string().optional(),
-              "prism:section": z4.string().optional(),
-              "prism:volume": z4.string().optional(),
-              "prism:number": z4.string().optional(),
-              "prism:startingPage": z4.string().optional(),
-              "prism:endingPage": z4.string().optional(),
-              category: z4.union([z4.string(), z4.array(z4.string())]).optional()
-            })
-          )
-        ).optional()
-      }).optional()
-    }).optional(),
-    // Handle Atom feeds
-    feed: z4.object({
-      title: z4.string().optional(),
-      link: z4.string().optional(),
-      entry: z4.any().transform((data) => Array.isArray(data) ? data : [data]).pipe(
-        z4.array(
-          z4.object({
-            title: z4.string(),
-            link: z4.string().optional(),
-            summary: z4.string().optional(),
-            content: z4.string().optional(),
-            author: z4.string().optional(),
-            published: z4.string().optional(),
-            id: z4.string().optional()
-          })
-        )
-      )
-    }).optional()
-    // Allow any other structure and try to extract items dynamically
-  });
-  async collect(url) {
-    const rawXml = await this.fetchFeed(url);
-    const feed = await this.parseFeedXML(rawXml);
-    const parsed = [];
-    const tags = /* @__PURE__ */ new Set();
-    let items = [];
-    let channelTitle = "";
-    if (feed.rss?.channel?.item) {
-      items = feed.rss.channel.item;
-      channelTitle = feed.rss.channel.title || "";
-    } else if (feed.rdf?.channel?.item) {
-      items = feed.rdf.channel.item;
-      channelTitle = feed.rdf.channel.title || "";
-    } else if (feed.rdf?.channel?.items?.["rdf:Seq"]?.["rdf:li"]) {
-      console.log("RDF 1.0 format detected - items are references, not content");
-      items = [];
-      channelTitle = feed.rdf.channel.title || "";
-    } else if (feed.feed?.entry) {
-      items = feed.feed.entry;
-      channelTitle = feed.feed.title || "";
-    } else {
-      items = this.findItemsDynamically(feed);
-      channelTitle = this.findChannelTitle(feed) || "";
-    }
-    if (items.length === 0) {
-      console.log("No items found with standard parsing, trying regex fallback...");
-      items = this.findItemsWithRegex(rawXml);
-      if (!channelTitle) {
-        channelTitle = this.findChannelTitleWithRegex(rawXml) || "";
-      }
-    }
-    console.log(`Items found: ${items.length}`);
-    if (channelTitle) {
-      tags.add(channelTitle.toLowerCase().replace(/\s+/g, "-"));
-    }
-    tags.add("rss-feed");
-    for (const item of items) {
-      const mainText = this.extractMainText(item);
-      if (mainText.length < 200) {
-        console.log(`Skipping item with short content (${mainText.length} chars): ${item.title?.substring(0, 50)}...`);
-        continue;
-      }
-      const metadata = this.extractMetadata(item);
-      const itemTags = this.extractTags(item, tags);
-      parsed.push({
-        title: item.title,
-        link: item.link,
-        mainText,
-        // Already cleaned in extractMainText
-        rawXml: item,
-        // Keep the raw XML data for future use
-        metadata,
-        tags: itemTags,
-        pubDate: item.pubDate || item.published,
-        guid: this.extractGuidValue(item.guid) || item.id
-      });
-    }
-    return parsed;
-  }
-  /**
-   * Extracts GUID value from either a string or object
-   */
-  extractGuidValue(guid) {
-    if (typeof guid === "string") {
-      return guid;
-    }
-    if (guid && typeof guid === "object") {
-      if (guid._text) return guid._text;
-      if (guid.text) return guid.text;
-      if (guid.value) return guid.value;
-      if (guid.content) return guid.content;
-      if (guid.id) return guid.id;
-      try {
-        return JSON.stringify(guid);
-      } catch {
-        return void 0;
-      }
-    }
-    return void 0;
-  }
-  /**
-   * Dynamically searches for items in any XML structure
-   */
-  findItemsDynamically(feed) {
-    const items = [];
-    const searchForItems = (obj, path = []) => {
-      if (Array.isArray(obj)) {
-        if (obj.length > 0 && obj[0] && typeof obj[0] === "object") {
-          const firstItem = obj[0];
-          if (firstItem.title || firstItem["dc:title"] || firstItem.name || firstItem.link) {
-            items.push(...obj);
-            return;
-          }
-        }
-      }
-      if (obj && typeof obj === "object") {
-        for (const [key, value] of Object.entries(obj)) {
-          if (key.toLowerCase().includes("item") || key.toLowerCase().includes("entry") || key.toLowerCase().includes("article") || key.toLowerCase().includes("post")) {
-            if (Array.isArray(value)) {
-              items.push(...value);
-              return;
-            }
-          }
-          searchForItems(value, [...path, key]);
-        }
-      }
-    };
-    searchForItems(feed);
-    return items;
-  }
-  /**
-   * Fallback method using regex to find items when standard parsing fails
-   */
-  findItemsWithRegex(xmlString) {
-    try {
-      console.log("Using regex fallback to find items...");
-      console.log(`XML length: ${xmlString.length} characters`);
-      const firstItemMatch = xmlString.match(/<item[^>]*>/i);
-      if (firstItemMatch) {
-        console.log(`Found first item tag: ${firstItemMatch[0]}`);
-      }
-      const titleMatches = xmlString.match(/<title[^>]*>.*?<\/title>/gi);
-      if (titleMatches) {
-        console.log(`Found ${titleMatches.length} title tags`);
-        console.log(`First few titles: ${titleMatches.slice(0, 3).map((t) => t.substring(0, 100))}`);
-      }
-      const itemRegex = /<item[^>]*>([\s\S]*?)<\/item>/gi;
-      const items = [];
-      let match;
-      while ((match = itemRegex.exec(xmlString)) !== null) {
-        const itemXml = match[1];
-        const titleMatch = itemXml.match(/<title[^>]*>[\s\S]*?<!\[CDATA\[([\s\S]*?)\]\]>[\s\S]*?<\/title>/i) || itemXml.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
-        const title = titleMatch ? this.cleanText(titleMatch[1]) : "";
-        const descMatch = itemXml.match(/<description[^>]*>[\s\S]*?<!\[CDATA\[([\s\S]*?)\]\]>[\s\S]*?<\/description>/i) || itemXml.match(/<description[^>]*>([\s\S]*?)<\/description>/i);
-        const description = descMatch ? this.cleanText(descMatch[1]) : "";
-        const linkMatch = itemXml.match(/<link[^>]*>([\s\S]*?)<\/link>/i);
-        const link = linkMatch ? this.cleanText(linkMatch[1]) : "";
-        if (title && description && description.length > 100) {
-          items.push({
-            title,
-            link,
-            description,
-            rawXml: itemXml
-          });
-        }
-      }
-      console.log(`Regex fallback found ${items.length} items`);
-      return items;
-    } catch (error) {
-      console.log(`Regex fallback failed: ${error}`);
-      return [];
-    }
-  }
-  /**
-  * Dynamically searches for channel title in any XML structure
-  */
-  findChannelTitle(feed) {
-    const searchForTitle = (obj) => {
-      if (obj && typeof obj === "object") {
-        if (obj.title) return obj.title;
-        if (obj.name) return obj.name;
-        for (const value of Object.values(obj)) {
-          const result = searchForTitle(value);
-          if (result) return result;
-        }
-      }
-      return void 0;
-    };
-    return searchForTitle(feed);
-  }
-  /**
-   * Find channel title using regex
-   */
-  findChannelTitleWithRegex(xmlString) {
-    try {
-      const titleMatch = xmlString.match(/<channel[^>]*>[\s\S]*?<title[^>]*>([\s\S]*?)<\/title>/i);
-      if (titleMatch) {
-        return this.cleanText(titleMatch[1]);
-      }
-      const feedTitleMatch = xmlString.match(/<feed[^>]*>[\s\S]*?<title[^>]*>([\s\S]*?)<\/title>/i);
-      if (feedTitleMatch) {
-        return this.cleanText(feedTitleMatch[1]);
-      }
-      return void 0;
-    } catch (error) {
-      console.log(`Regex channel title search failed: ${error}`);
-      return void 0;
-    }
-  }
-  /**
-   * Intelligently extracts the main text content from an RSS item.
-   * Finds the element with the most text and appends other text-rich elements.
-   * Returns cleaned, combined text content.
-   */
-  extractMainText(item) {
-    const textFields = [
-      "content:encoded",
-      "description",
-      "dc:description",
-      "summary",
-      "excerpt",
-      "abstract",
-      "content",
-      "body"
-    ];
-    let primaryText = "";
-    let primaryField = "";
-    let maxLength = 0;
-    for (const field of textFields) {
-      if (item[field] && typeof item[field] === "string") {
-        const cleanLength = this.cleanText(item[field]).length;
-        if (cleanLength > maxLength) {
-          maxLength = cleanLength;
-          primaryText = item[field];
-          primaryField = field;
-        }
-      }
-    }
-    if (!primaryText) {
-      return item.title || "";
-    }
-    const additionalTexts = [];
-    for (const field of textFields) {
-      if (field !== primaryField && item[field] && typeof item[field] === "string") {
-        const cleanText = this.cleanText(item[field]);
-        if (cleanText.length > 20 && !this.isDuplicateContent(primaryText, cleanText)) {
-          additionalTexts.push(cleanText);
-        }
-      }
-    }
-    for (const [key, value] of Object.entries(item)) {
-      if (typeof value === "string" && !textFields.includes(key) && !key.startsWith("dc:") && !key.startsWith("prism:") && !key.startsWith("arxiv:") && key !== "title" && key !== "link" && key !== "guid" && key !== "pubDate") {
-        const cleanText = this.cleanText(value);
-        if (cleanText.length > 20 && !this.isDuplicateContent(primaryText, cleanText)) {
-          additionalTexts.push(cleanText);
-        }
-      }
-    }
-    let combinedText = primaryText;
-    if (additionalTexts.length > 0) {
-      combinedText += "\n\n" + additionalTexts.join("\n\n");
-    }
-    return this.cleanText(combinedText);
-  }
-  /**
-   * Checks if two text contents are substantially similar to avoid duplication
-   */
-  isDuplicateContent(primaryText, additionalText) {
-    const primary = primaryText.toLowerCase().trim();
-    const additional = additionalText.toLowerCase().trim();
-    if (primary.includes(additional) || additional.includes(primary)) {
-      return true;
-    }
-    const primaryWords = primary.split(/\s+/).filter((word) => word.length > 3);
-    const additionalWords = additional.split(/\s+/).filter((word) => word.length > 3);
-    if (primaryWords.length === 0 || additionalWords.length === 0) {
-      return false;
-    }
-    const commonWords = primaryWords.filter((word) => additionalWords.includes(word));
-    const similarity = commonWords.length / Math.max(primaryWords.length, additionalWords.length);
-    return similarity > 0.7;
-  }
-  /**
-   * Cleans text of HTML tags, special characters, and formatting.
-   * Preserves readable text while removing markup.
-   */
-  cleanText(text) {
-    if (!text) return "";
-    const $ = cheerio2.load(text);
-    $("script, style").remove();
-    let cleaned = $.text();
-    cleaned = cleaned.replace(/\s+/g, " ").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&amp;/g, "&").replace(/&quot;/g, '"').replace(/&#39;/g, "'").replace(/&nbsp;/g, " ").replace(/\[CDATA\[/g, "").replace(/\]\]>/g, "").replace(/\n+/g, " ").replace(/\t+/g, " ").trim();
-    return cleaned;
-  }
-  /**
-   * Extracts structured metadata from RSS item fields.
-   */
-  extractMetadata(item) {
-    const metadata = {};
-    if (item["dc:creator"]) metadata.creator = item["dc:creator"];
-    if (item["dc:date"]) metadata.date = item["dc:date"];
-    if (item["dc:subject"]) metadata.subject = item["dc:subject"];
-    if (item["dc:publisher"]) metadata.publisher = item["dc:publisher"];
-    if (item["dc:identifier"]) metadata.identifier = item["dc:identifier"];
-    if (item["dc:rights"]) metadata.rights = item["dc:rights"];
-    if (item["dc:source"]) metadata.source = item["dc:source"];
-    if (item["prism:publicationDate"]) metadata.publicationDate = item["prism:publicationDate"];
-    if (item["prism:section"]) metadata.section = item["prism:section"];
-    if (item["prism:volume"]) metadata.volume = item["prism:volume"];
-    if (item["prism:number"]) metadata.number = item["prism:number"];
-    if (item["prism:startingPage"]) metadata.startingPage = item["prism:startingPage"];
-    if (item["prism:endingPage"]) metadata.endingPage = item["prism:endingPage"];
-    if (item["arxiv:announce_type"]) metadata.announceType = item["arxiv:announce_type"];
-    if (item.category) {
-      if (Array.isArray(item.category)) {
-        metadata.categories = item.category;
-      } else {
-        metadata.categories = [item.category];
-      }
-    }
-    return metadata;
-  }
-  /**
-   * Extracts tags from the item and combines with feed-level tags.
-   */
-  extractTags(item, baseTags) {
-    const tags = new Set(baseTags);
-    if (item.category) {
-      if (Array.isArray(item.category)) {
-        item.category.forEach((cat) => tags.add(cat.toLowerCase().replace(/\s+/g, "-")));
-      } else {
-        tags.add(item.category.toLowerCase().replace(/\s+/g, "-"));
-      }
-    }
-    if (item["dc:subject"]) {
-      tags.add(item["dc:subject"].toLowerCase().replace(/\s+/g, "-"));
-    }
-    if (item["prism:section"]) {
-      tags.add(item["prism:section"].toLowerCase().replace(/\s+/g, "-"));
-    }
-    if (item["dc:source"]) {
-      tags.add(item["dc:source"].toLowerCase().replace(/\s+/g, "-"));
-    }
-    if (item["dc:publisher"]) {
-      tags.add(item["dc:publisher"].toLowerCase().replace(/\s+/g, "-"));
-    }
-    return Array.from(tags);
-  }
-};
-
-// src/collectors/string-collector.ts
-var StringCollector = class extends AbstractCollector {
-  identifier = "string";
-  async collect(source) {
-    if (typeof source === "string") {
-      return source;
-    }
-    return void 0;
-  }
-};
+import {
+  defineRunner
+} from "./chunk-QY5MPNNB.js";
+import {
+  ScoringMethod
+} from "./chunk-HMQYGCKI.js";
+import {
+  CATEGORIES,
+  PEERBENCH_NAMESPACE
+} from "./chunk-UHHHSYVE.js";
+import {
+  RateLimiter,
+  bufferToString,
+  idGeneratorUUIDv7,
+  parseResponseAsJSON,
+  sleep,
+  stringToBuffer
+} from "./chunk-4UBK6452.js";
+import "./chunk-PZ5AY32C.js";
 
 // src/errors/polyfill.ts
 function captureStackTrace(error, constructor) {
@@ -1096,2472 +27,27 @@ function captureStackTrace(error, constructor) {
   }
 }
 
-// src/errors/provider.ts
-var ForwardError = class extends Error {
-  startedAt;
+// src/errors/peerbench.ts
+var PeerbenchError = class extends Error {
   code;
   constructor(message, options) {
     super(message, options);
-    this.startedAt = options?.startedAt || /* @__PURE__ */ new Date();
-    this.code = options?.code;
+    this.code = options?.code ?? 0;
     captureStackTrace(this, this.constructor);
   }
 };
-
-// src/errors/parser.ts
-var InvalidDataError = class extends Error {
-  constructor(message) {
-    super(message || "Data is in an invalid format");
-    this.name = "InvalidDataError";
-    captureStackTrace(this, this.constructor);
-  }
-};
-var ParserIsNotCompatibleError = class extends InvalidDataError {
-  constructor() {
-    super("Parser is not compatible with the data");
-    this.name = "ParserIsNotCompatibleError";
-    captureStackTrace(this, this.constructor);
-  }
-};
-
-// src/errors/codes.ts
-var PEERBENCH_ERROR_CODES = {
-  PROVIDER_FORWARD_FAILED: "PROVIDER_FORWARD_FAILED",
-  PROVIDER_UNAUTHORIZED: "PROVIDER_UNAUTHORIZED",
-  PROVIDER_MAX_RETRIES_REACHED: "PROVIDER_MAX_RETRIES_REACHED"
-};
-
-// src/generators/abstract/abstract-generator.ts
-import { v7 as uuidv72 } from "uuid";
-var AbstractGenerator = class {
-  /**
-   * Generate prompt from the collected source data
-   * @param input - Raw input data that will be validated against inputSchema
-   * @param options - Optional configuration options
-   * @returns Promise resolving to an array of prompts
-   */
-  async generate(input, options) {
-    const validatedInput = this.inputSchema.parse(input);
-    return this.generatePrompts(validatedInput, options);
-  }
-  /**
-   * Checks whether the generator can handle the given input
-   */
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  canHandle(input) {
-    return true;
-  }
-  /**
-   * Initializes the generator (depends on the implementation)
-   */
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  async initialize(...args) {
-  }
-  async buildPrompt(params) {
-    const questionCID = await calculateCID(params.question).then(
-      (cid) => cid.toString()
-    );
-    const questionSHA256 = await calculateSHA256(params.question);
-    const fullPromptCID = await calculateCID(params.fullPrompt).then(
-      (cid) => cid.toString()
-    );
-    const fullPromptSHA256 = await calculateSHA256(params.fullPrompt);
-    return {
-      did: uuidv72(),
-      question: {
-        data: params.question,
-        cid: questionCID,
-        sha256: questionSHA256
-      },
-      // If the Prompt is a Multiple Choice, the answer data should be
-      // the value of the correct answer key
-      answer: params.options === void 0 || Object.keys(params.options).length === 0 ? params.correctAnswer : params.options[params.correctAnswer],
-      // If the options is provided then the correctAnswer points to the letter of the correct answer
-      // Answer key is only valid when the options are provided
-      // which means the Prompt is a multiple choice question
-      answerKey: params.options === void 0 || Object.keys(params.options).length === 0 ? "" : params.correctAnswer,
-      options: params.options ?? {},
-      fullPrompt: {
-        data: params.fullPrompt,
-        cid: fullPromptCID,
-        sha256: fullPromptSHA256
-      },
-      type: params.type,
-      metadata: {
-        generatorIdentifier: this.identifier,
-        ...params.metadata || {}
-      },
-      scorers: params.scorers ?? []
-    };
-  }
-};
-
-// src/generators/pubmed/trp-generator.ts
-import { z as z6 } from "zod";
-
-// src/validation/enum.ts
-import { z as z5 } from "zod";
-function EnumSchema(_enum, lookFor = "value") {
-  return z5.string().transform((value, ctx) => {
-    let enumValue;
-    if (lookFor === "key") {
-      enumValue = _enum[value];
-    } else {
-      const values = Object.values(_enum);
-      enumValue = values.find((v) => v === value);
-    }
-    if (!enumValue) {
-      ctx.addIssue({
-        code: z5.ZodIssueCode.custom,
-        message: `Invalid enum value: ${value}`
-      });
-      return z5.NEVER;
-    }
-    return enumValue;
-  });
-}
-
-// src/generators/pubmed/helpers/paragraph-merge.ts
-function paragraphMerge(paragraphs, strategy) {
-  switch (strategy) {
-    case ParagraphMergeStrategy.WithoutTitles:
-      return Object.entries(paragraphs).map(([, value]) => value).join("\n");
-    case ParagraphMergeStrategy.TitlesAsSentences:
-      return Object.entries(paragraphs).map(([key, value]) => `${key}. ${value}`).join("\n");
-    case ParagraphMergeStrategy.TitlesWithinSentences:
-      return Object.entries(paragraphs).map(([key, value]) => `${key}: ${value}`).join("\n");
-    default:
-      throw new Error(`Invalid paragraph merge strategy: ${strategy}`);
-  }
-}
-var ParagraphMergeStrategy = {
-  WithoutTitles: "without-titles",
-  TitlesAsSentences: "with-titles-as-sentences",
-  TitlesWithinSentences: "titles-within-sentences"
-};
-
-// src/generators/pubmed/helpers/replace-entities.ts
-function replaceEntities(text, entities, placeholder = "{}") {
-  if (!entities || entities.length === 0) return text;
-  const esc = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-  const pattern = entities.map(esc).sort((a, b) => b.length - a.length).join("|");
-  const re = new RegExp(`\\b(?:${pattern})\\b`, "g");
-  return text.replace(re, placeholder);
-}
-
-// src/generators/pubmed/helpers/crypto-random.ts
-function cryptoRandom() {
-  return crypto.getRandomValues(new Uint32Array(1))[0] / 2 ** 32;
-}
-
-// src/providers/abstract/abstract-provider.ts
-var AbstractProvider = class {
-};
-
-// src/providers/llm/base-llm-provider.ts
-import OpenAI, { APIError } from "openai";
-var BaseLLMProvider = class extends AbstractProvider {
-  timestamps = [];
-  timeout;
-  rateLimit;
-  rateLimitTimeWindow;
-  client;
-  maxRetries;
-  /**
-   * Initialize a new LLM provider
-   * @param options Options for the provider
-   */
-  constructor(options) {
-    super();
-    this.rateLimit = options.rateLimit ?? 20;
-    this.rateLimitTimeWindow = options.rateLimitTimeWindow ?? 3e3;
-    this.timeout = options.timeout ?? 5 * 6e4;
-    this.maxRetries = options.maxRetries ?? 5;
-    this.client = new OpenAI({
-      baseURL: options.baseURL,
-      apiKey: options.apiKey,
-      maxRetries: options.maxRetries,
-      timeout: options.timeout,
-      dangerouslyAllowBrowser: true
-    });
-  }
-  async enforceRateLimit() {
-    const now = Date.now();
-    this.timestamps = this.timestamps.filter(
-      (ts) => now - ts < this.rateLimitTimeWindow
-    );
-    if (this.timestamps.length < this.rateLimit) {
-      this.timestamps.push(now);
-      return;
-    }
-    const earliest = this.timestamps[0];
-    const waitTime = this.rateLimitTimeWindow - (now - earliest);
-    await sleep(waitTime);
-    return this.enforceRateLimit();
-  }
-  /**
-   * Fetch all supported models from the provider
-   * @returns Array of model information
-   */
-  async getSupportedModels() {
-    await this.enforceRateLimit();
-    try {
-      const response = await this.client.models.list();
-      const models = response.data;
-      const parsedModels = await Promise.all(
-        models.map(async (model) => {
-          const parsed = await this.parseModelInfo(model);
-          if (!parsed) {
-            return;
-          }
-          return {
-            ...parsed,
-            metadata: {
-              // These fields might not be available in all models
-              contextWindow: model.context_window,
-              maxTokens: model.max_tokens,
-              pricing: model.pricing ? {
-                input: model.pricing.input,
-                output: model.pricing.output
-              } : void 0
-            }
-          };
-        })
-      );
-      return parsedModels.filter((model) => model !== void 0);
-    } catch (error) {
-      throw new Error(
-        `Failed to fetch supported models: ${error instanceof Error ? error.message : "Unknown error"}`
-      );
-    }
-  }
-  async forward(input, options) {
-    await this.enforceRateLimit();
-    let retryCount = this.maxRetries;
-    while (retryCount > 0) {
-      const startedAt = /* @__PURE__ */ new Date();
-      try {
-        const messages = [];
-        if (Array.isArray(input)) {
-          messages.push(...input);
-        } else {
-          if (options?.system) {
-            messages.push({
-              role: "system",
-              content: options.system
-            });
-          }
-          messages.push({
-            role: "user",
-            content: input
-          });
-        }
-        const response = await this.client.chat.completions.create(
-          {
-            model: options.model,
-            messages,
-            temperature: options.temperature
-          },
-          { signal: options?.abortSignal, timeout: this.timeout }
-        );
-        if ("error" in response) {
-          const err = response.error;
-          throw new Error(
-            `${err.message} - Code ${err.code} - ${JSON.stringify(err)}`
-          );
-        }
-        return {
-          data: response?.choices?.[0]?.message?.content || "",
-          inputTokensUsed: response?.usage?.prompt_tokens,
-          outputTokensUsed: response?.usage?.completion_tokens,
-          startedAt,
-          completedAt: /* @__PURE__ */ new Date()
-        };
-      } catch (err) {
-        if (err instanceof APIError && err.status === 401) {
-          throw new ForwardError(
-            `Invalid credentials provided for the Provider`,
-            {
-              cause: err,
-              startedAt,
-              code: PEERBENCH_ERROR_CODES.PROVIDER_UNAUTHORIZED
-            }
-          );
-        }
-        retryCount--;
-        if (err instanceof SyntaxError) {
-          console.debug(err);
-          continue;
-        }
-        if (retryCount !== 0) {
-          continue;
-        }
-        throw new ForwardError(
-          `Failed to forward prompt to the model: ${err instanceof Error ? err.message : err}`,
-          {
-            cause: err,
-            startedAt,
-            code: PEERBENCH_ERROR_CODES.PROVIDER_FORWARD_FAILED
-          }
-        );
-      }
-    }
-    throw new ForwardError(
-      `Failed to forward prompt to the model: Max retries reached`,
-      {
-        startedAt: /* @__PURE__ */ new Date(),
-        code: PEERBENCH_ERROR_CODES.PROVIDER_MAX_RETRIES_REACHED
-      }
-    );
-  }
-};
-var LargeLanguageModelOwner = {
-  Meta: "meta",
-  OpenAI: "openai",
-  Deepseek: "deepseek",
-  Qwen: "qwen",
-  Google: "google",
-  XAI: "x-ai",
-  Anthropic: "anthropic",
-  Mistral: "mistral"
-};
-var MetaModels = {
-  Llama_4_Maverick: "llama-4-maverick",
-  Llama_4_Scout: "llama-4-scout",
-  Llama_3_3_70b_Instruct: "llama-3.3-70b-instruct",
-  Llama_3_1_8b_Instruct: "llama-3.1-8b-instruct",
-  Llama_3_1_70b_Instruct: "llama-3.1-70b-instruct"
-};
-var QwenModels = {
-  QwQ_32b: "qwq-32b"
-};
-var DeepSeekModels = {
-  V3: "deepseek-v3",
-  V3_0324: "deepseek-v3-0324"
-};
-var XAIModels = {
-  Grok3_Beta: "grok-3-beta",
-  Grok4: "grok-4"
-};
-var GoogleModels = {
-  Gemini_2_0_Flash: "gemini-2.0-flash",
-  Gemini_2_5_Flash_Lite: "gemini-2.5-flash-lite",
-  Gemini_2_5_Pro: "gemini-2.5-pro"
-};
-var AnthropicModels = {
-  Claude_3_7_Sonnet: "claude-3.7-sonnet",
-  Claude_Sonnet_4_5: "claude-sonnet-4.5"
-};
-var OpenAIModels = {
-  ChatGPT_4o: "chatgpt-4o-latest",
-  GPT_4o: "gpt-4o",
-  GPT_4o_Mini: "gpt-4o-mini",
-  GPT_5: "gpt-5"
-};
-var MistralModels = {
-  Ministral_8B: "ministral-8b"
-};
-var LargeLanguageModel = {
-  [LargeLanguageModelOwner.Meta]: MetaModels,
-  [LargeLanguageModelOwner.Deepseek]: DeepSeekModels,
-  [LargeLanguageModelOwner.Qwen]: QwenModels,
-  [LargeLanguageModelOwner.Google]: GoogleModels,
-  [LargeLanguageModelOwner.XAI]: XAIModels,
-  [LargeLanguageModelOwner.OpenAI]: OpenAIModels,
-  [LargeLanguageModelOwner.Anthropic]: AnthropicModels,
-  [LargeLanguageModelOwner.Mistral]: MistralModels
-};
-
-// src/providers/llm/openrouter.ts
-import axios from "axios";
-import Decimal from "decimal.js";
-var baseURL = "https://openrouter.ai/api/v1";
-var MODELS_CACHE_TTL = 1e3 * 60 * 60 * 24;
-var OpenRouterProvider = class extends BaseLLMProvider {
-  models = void 0;
-  modelsCachePromise = Promise.resolve(void 0);
-  modelsUpdatedAt = 0;
-  identifier = "openrouter.ai";
-  constructor(options) {
-    super({
-      ...options,
-      baseURL
-    });
-  }
-  /**
-   * Updates the cache that holds information about OpenRouter models
-   * including pricing information. It will be valid for 24 hours as
-   * long as the instance of this Provider object is alive.
-   */
-  async updateModelsCache() {
-    this.modelsCachePromise = this.modelsCachePromise.then(async () => {
-      if (
-        // The data presented in the cache
-        this.models !== void 0 && // The cache is still valid
-        Date.now() - this.modelsUpdatedAt < MODELS_CACHE_TTL
-      ) {
-        return this.models;
-      }
-      return axios.get(`${baseURL}/models`).then((res) => res.data).then((data) => {
-        data = {
-          data: data.data.filter(
-            (m) => m.architecture.input_modalities.includes("text") && m.architecture.output_modalities.includes("text")
-          )
-        };
-        this.models = data;
-        this.modelsUpdatedAt = Date.now();
-        return data;
-      });
-    }).catch(() => void 0);
-    await this.modelsCachePromise;
-  }
-  async getModelDetails(modelId) {
-    await this.updateModelsCache();
-    if (modelId === void 0) {
-      return this.models?.data;
-    }
-    return this.models?.data.find((model) => model.id === modelId);
-  }
-  /**
-   * Returns the credits that the account has on openrouter.ai
-   */
-  async getCredits() {
-    const response = await axios.get(`${baseURL}/credits`, {
-      headers: {
-        Authorization: `Bearer ${this.client.apiKey}`
-      }
-    });
-    return response.data;
-  }
-  /**
-   * Makes a request to a protected endpoint to validate the API key.
-   */
-  async validateApiKey() {
-    await axios.get(`${baseURL}/models/user`, {
-      headers: {
-        Authorization: `Bearer ${this.client.apiKey}`
-      }
-    });
-    return true;
-  }
-  async forward(input, options) {
-    const [response] = await Promise.all([
-      super.forward(input, options),
-      this.updateModelsCache()
-      // Concurrently update the cache for cost info
-    ]);
-    const modelInfo = this.models?.data.find(
-      (model) => model.id === options.model
-    );
-    let inputCost = void 0;
-    let outputCost = void 0;
-    if (modelInfo !== void 0) {
-      if (response.inputTokensUsed !== void 0) {
-        inputCost = new Decimal(modelInfo.pricing.prompt).mul(response.inputTokensUsed).toFixed(10);
-      }
-      if (response.outputTokensUsed !== void 0) {
-        outputCost = new Decimal(modelInfo.pricing.completion).mul(response.outputTokensUsed).toFixed(10);
-      }
-    }
-    return {
-      ...response,
-      inputCost,
-      outputCost
-    };
-  }
-  parseModelInfo(modelOrId) {
-    const id = typeof modelOrId === "string" ? modelOrId : modelOrId.id;
-    const [, modelName] = id.split("/");
-    if (!modelName) {
-      return;
-    }
-    let name;
-    let owner;
-    switch (modelName) {
-      case "ministral-8b":
-        owner = LargeLanguageModelOwner.Mistral;
-        name = LargeLanguageModel[owner].Ministral_8B;
-        break;
-      case "chatgpt-4o-latest":
-        owner = LargeLanguageModelOwner.OpenAI;
-        name = LargeLanguageModel[owner].ChatGPT_4o;
-        break;
-      case "gpt-4o-mini":
-        owner = LargeLanguageModelOwner.OpenAI;
-        name = LargeLanguageModel[owner].GPT_4o_Mini;
-        break;
-      case "deepseek-chat-v3-0324":
-        owner = LargeLanguageModelOwner.Deepseek;
-        name = LargeLanguageModel[owner].V3_0324;
-        break;
-      case "gpt-4o":
-        owner = LargeLanguageModelOwner.OpenAI;
-        name = LargeLanguageModel[owner].GPT_4o;
-        break;
-      case "gpt-5":
-        owner = LargeLanguageModelOwner.OpenAI;
-        name = LargeLanguageModel[owner].GPT_5;
-        break;
-      case "claude-3.7-sonnet":
-        owner = LargeLanguageModelOwner.Anthropic;
-        name = LargeLanguageModel[owner].Claude_3_7_Sonnet;
-        break;
-      case "claude-sonnet-4.5":
-        owner = LargeLanguageModelOwner.Anthropic;
-        name = LargeLanguageModel[owner].Claude_Sonnet_4_5;
-        break;
-      case "llama-3.3-70b-instruct":
-        owner = LargeLanguageModelOwner.Meta;
-        name = LargeLanguageModel[owner].Llama_3_3_70b_Instruct;
-        break;
-      case "llama-3.1-70b-instruct":
-        owner = LargeLanguageModelOwner.Meta;
-        name = LargeLanguageModel[owner].Llama_3_1_70b_Instruct;
-        break;
-      case "llama-3.1-8b-instruct":
-        owner = LargeLanguageModelOwner.Meta;
-        name = LargeLanguageModel[owner].Llama_3_1_8b_Instruct;
-        break;
-      case "deepseek-chat":
-        owner = LargeLanguageModelOwner.Deepseek;
-        name = LargeLanguageModel[owner].V3;
-        break;
-      case "qwq-32b":
-        owner = LargeLanguageModelOwner.Qwen;
-        name = LargeLanguageModel[owner].QwQ_32b;
-        break;
-      case "gemini-2.0-flash-001":
-        owner = LargeLanguageModelOwner.Google;
-        name = LargeLanguageModel[owner].Gemini_2_0_Flash;
-        break;
-      case "gemini-2.5-flash-lite":
-        owner = LargeLanguageModelOwner.Google;
-        name = LargeLanguageModel[owner].Gemini_2_5_Flash_Lite;
-        break;
-      case "gemini-2.5-pro":
-        owner = LargeLanguageModelOwner.Google;
-        name = LargeLanguageModel[owner].Gemini_2_5_Pro;
-        break;
-      case "grok-3-beta":
-      case "grok-3":
-        owner = LargeLanguageModelOwner.XAI;
-        name = LargeLanguageModel[owner].Grok3_Beta;
-        break;
-      case "grok-4":
-        owner = LargeLanguageModelOwner.XAI;
-        name = LargeLanguageModel[owner].Grok4;
-        break;
-      case "llama-4-maverick":
-        owner = LargeLanguageModelOwner.Meta;
-        name = LargeLanguageModel[owner].Llama_4_Maverick;
-        break;
-      case "llama-4-scout":
-        owner = LargeLanguageModelOwner.Meta;
-        name = LargeLanguageModel[owner].Llama_4_Scout;
-        break;
-      default:
-        return;
-    }
-    return {
-      id,
-      name,
-      owner,
-      provider: this.identifier.toLowerCase()
-    };
-  }
-};
-
-// src/providers/llm/nearai.ts
-var NearAIProvider = class extends BaseLLMProvider {
-  identifier = "near.ai";
-  constructor(options) {
-    super({
-      ...options,
-      baseURL: "https://api.near.ai/v1"
-    });
-  }
-  parseModelInfo(modelOrId) {
-    const id = typeof modelOrId === "string" ? modelOrId : modelOrId.id;
-    let modelName;
-    let host;
-    if (id.startsWith("fireworks::")) {
-      const splitResult = id.split("/");
-      modelName = splitResult.pop();
-      host = "fireworks";
-    } else if (id.startsWith("hyperbolic::")) {
-      const splitResult = id.split("/");
-      modelName = splitResult.pop();
-      host = "hyperbolic";
-    }
-    if (!modelName) {
-      return;
-    }
-    let name;
-    let owner;
-    switch (modelName) {
-      case "llama4-maverick-instruct-basic":
-        owner = LargeLanguageModelOwner.Meta;
-        name = LargeLanguageModel[owner].Llama_4_Maverick;
-        break;
-      case "llama4-scout-instruct-basic":
-        owner = LargeLanguageModelOwner.Meta;
-        name = LargeLanguageModel[owner].Llama_4_Scout;
-        break;
-      case "llama-v3p3-70b-instruct":
-      case "Llama-3.3-70B-Instruct":
-        owner = LargeLanguageModelOwner.Meta;
-        name = LargeLanguageModel[owner].Llama_3_3_70b_Instruct;
-        break;
-      case "llama-v3p1-8b-instruct":
-        owner = LargeLanguageModelOwner.Meta;
-        name = LargeLanguageModel[owner].Llama_3_1_8b_Instruct;
-        break;
-      case "deepseek-v3":
-      case "DeepSeek-V3":
-        owner = LargeLanguageModelOwner.Deepseek;
-        name = LargeLanguageModel[owner].V3;
-        break;
-      case "qwq-32b":
-        owner = LargeLanguageModelOwner.Qwen;
-        name = LargeLanguageModel[owner].QwQ_32b;
-        break;
-      default:
-        return;
-    }
-    return {
-      id,
-      name,
-      owner,
-      host,
-      provider: this.identifier.toLowerCase()
-    };
-  }
-};
-
-// src/generators/pubmed/trp-generator.ts
-var TRPGenerator = class extends AbstractGenerator {
-  identifier = "trp";
-  inputSchema = z6.array(
-    z6.object({
-      pmid: z6.string(),
-      title: z6.string(),
-      paragraphs: z6.record(z6.string(), z6.string()),
-      tags: z6.array(z6.string())
-    })
-  );
-  optionsSchema = z6.object(
-    {
-      openRouterApiKey: z6.string(),
-      paragraphMergeStrategy: EnumSchema(ParagraphMergeStrategy).default(
-        ParagraphMergeStrategy.TitlesWithinSentences
-      ),
-      model: z6.string(),
-      placeholder: z6.string().default("{}"),
-      nerPrompt: z6.string().optional().default(
-        `You are a Named Entity Recognition model which is specialized on medical relevant texts. Your task is extracting all medical related entities. Your output strictly forced to be a JSON array of strings where each item represents a single entity that you've extracted. Markdown formatting is forbidden. JSON output must be minified.`
-      )
-    },
-    { message: "No options provided" }
-  );
-  async generatePrompts(input, options) {
-    const parsedOptions = this.optionsSchema.parse(options);
-    const provider = new OpenRouterProvider({
-      apiKey: parsedOptions.openRouterApiKey
-    });
-    const generatedPrompts = await Promise.all(
-      input.map(
-        (article) => this.generatePromptFromArticle(article, provider, parsedOptions)
-      )
-    );
-    return generatedPrompts.filter((prompt) => prompt !== null);
-  }
-  async generatePromptFromArticle(article, provider, options) {
-    const tags = [
-      `generator-${this.identifier}`,
-      "perform-ner",
-      `merge-paragraphs-${options.paragraphMergeStrategy}`,
-      `ner-for-medical-related-entities`
-    ];
-    const text = `${article.title}
-
-${paragraphMerge(
-      article.paragraphs,
-      options.paragraphMergeStrategy
-    )}`;
-    const entities = await this.doNER({
-      text,
-      provider,
-      model: options.model,
-      systemPrompt: options.nerPrompt
-    });
-    if (!entities) {
-      return null;
-    }
-    const modifiedText = replaceEntities(text, entities, options.placeholder);
-    const fullPrompt = `TEXT:
-${modifiedText}
-
-ENTITIES:
-${entities.sort(() => cryptoRandom() - 0.5).map((e) => `"${e}"`).join(", ")}`;
-    return await this.buildPrompt({
-      question: modifiedText,
-      fullPrompt,
-      correctAnswer: text,
-      // Original formatted text is the correct answer
-      type: PromptTypes.TextReplacement,
-      metadata: {
-        articleTags: article.tags,
-        articleId: article.pmid,
-        paragraphMergeStrategy: options.paragraphMergeStrategy,
-        entityTypes: ["medical-related-entities"],
-        entities,
-        brainModel: options.model,
-        generatorTags: tags,
-        tags: [...tags, ...article.tags]
-      }
-    });
-  }
-  async doNER(params) {
-    const { text, provider, model, systemPrompt } = params;
-    const { data } = await provider.forward(text, {
-      model,
-      system: systemPrompt
-    });
-    return parseResponseAsJSON(data);
-  }
-};
-
-// src/generators/mcq-generator.ts
-import { z as z9 } from "zod";
-
-// src/utils/debug.ts
-function debugLog(...args) {
-  if (process?.env?.PB_SDK_DEBUG) {
-    console.log(`[${(/* @__PURE__ */ new Date()).toISOString()}] PB_SDK_DEBUG:`, ...args);
-  }
-}
-
-// src/scorers/abstract/abstract-scorer.ts
-var AbstractScorer = class {
-};
-
-// src/scorers/multiple-choice-scorer.ts
-import { v7 as uuidv73 } from "uuid";
-var MultipleChoiceScorer = class extends AbstractScorer {
-  identifier = "multiple-choice";
-  EXPLANATION_TEXT = `This scorer searches for multiple choice answers using the following patterns (in order):
-1) "<!NO ANSWER!>" (special marker indicating model's inability to answer),
-2) "Answer is $\\boxed{answer text}$" (full answer text in LaTeX boxed format),
-3) "Answer is answer text" (full answer text),
-4) "Answer is **answer text**" (full answer text in bold),
-5) "Answer is $\\boxed{A}$" or "Answer is $\\boxed{A}$." (single letter in LaTeX boxed format, optional period),
-6) "Answer is A" (single letter),
-7) "Answer is **A**" (single letter in bold),
-8) "A: ..." (letter followed by colon),
-9) "A) ..." (letter followed by closing parenthesis and optional text),
-10) "A)" (letter followed by closing parenthesis).
-The scorer extracts the answer from the last matching pattern (if multiple matches exist) and compares it with the expected answer key (or the answer text itself).`;
-  /**
-   * Score a multiple choice response
-   */
-  async scoreOne(response) {
-    if (!await this.canScore(response)) {
-      return void 0;
-    }
-    const { data, prompt } = response;
-    let score = 0;
-    if (data.trim() === prompt.answerKey?.trim()) {
-      score = 1;
-    }
-    let extractedAnswer = this.lookForAnswer(data, prompt.answerKey);
-    if (extractedAnswer === prompt.answerKey) {
-      score = 1;
-    } else {
-      const answerOption = Object.entries(prompt.options).find(
-        ([, value]) => value.trim() === extractedAnswer?.trim()
-      );
-      if (answerOption && answerOption[0] === prompt.answerKey) {
-        score = 1;
-        extractedAnswer = answerOption[0];
-      }
-    }
-    return PromptScoreSchema.parse({
-      ...response,
-      score,
-      scoreDID: uuidv73(),
-      method: ScoringMethods.algo,
-      prompt: response.prompt,
-      scoreMetadata: {
-        scorerIdentifier: this.identifier,
-        extractedAnswer
-      },
-      explanation: this.EXPLANATION_TEXT
-    });
-  }
-  async canScore(response) {
-    return response.data !== void 0 && response.prompt !== void 0 && // TODO: Enable this condition once we are sure the structure of the Prompt and whether to include the type in there
-    // response.prompt.type === PromptTypes.MultipleChoice &&
-    response.prompt.options !== void 0 && Object.keys(response.prompt.options).length > 0 && Boolean(response.prompt.answerKey) && // not undefined or empty string
-    Boolean(response.prompt.answer);
-  }
-  /**
-   * Extracts answer from Response text using regex patterns
-   */
-  lookForAnswer(response, answer) {
-    const patterns = [
-      {
-        // "<!NO ANSWER!>"
-        regex: /<!NO ANSWER!>/g,
-        // Pattern matches, but no group is specified in the regex.
-        // So the final value will be `undefined`.
-        answerGroupIndex: 1
-      },
-      {
-        // "Answer is $\boxed{answer text}$"
-        regex: new RegExp(
-          `[Aa]nswer is \\$\\\\boxed\\{(${this.escapeRegex(answer)})\\}\\$`,
-          "g"
-        ),
-        answerGroupIndex: 1
-      },
-      {
-        // "Answer is answer text"
-        regex: new RegExp(`[Aa]nswer is\\s+(${this.escapeRegex(answer)})`, "g"),
-        answerGroupIndex: 1
-      },
-      {
-        // "Answer is **answer text**"
-        regex: new RegExp(
-          `[Aa]nswer is\\s+\\**(${this.escapeRegex(answer)})\\**`,
-          "g"
-        ),
-        answerGroupIndex: 1
-      },
-      {
-        // "Answer is $\boxed{A}$."
-        regex: /[Aa]nswer is \$\\boxed\{([A-Z])\}\$\.?/g,
-        answerGroupIndex: 1
-      },
-      {
-        // "Answer is A"
-        regex: /[Aa]nswer is\s+([A-Z])/g,
-        answerGroupIndex: 1
-      },
-      {
-        // "Answer is **A**"
-        regex: /[Aa]nswer is\s+\**([A-Z])\**/g,
-        answerGroupIndex: 1
-      },
-      {
-        // "A: answer text"
-        regex: /([A-Z]):.+/g,
-        answerGroupIndex: 1
-      },
-      {
-        // "A) answer text"
-        regex: /([A-Z])\)\s*.+/g,
-        answerGroupIndex: 1
-      },
-      {
-        // "A)"
-        regex: /([A-Z])\)/g,
-        answerGroupIndex: 1
-      }
-    ];
-    for (const pattern of patterns) {
-      const matches = Array.from(response.matchAll(pattern.regex));
-      const match = matches.at(-1);
-      if (match) {
-        return match[pattern.answerGroupIndex];
-      }
-    }
-  }
-  escapeRegex(str) {
-    return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-  }
-};
-
-// src/scorers/exact-match-scorer.ts
-import { v7 as uuidv74 } from "uuid";
-var ExactMatchScorer = class extends AbstractScorer {
-  identifier = "exact-match";
-  async scoreOne(response) {
-    if (!await this.canScore(response)) {
-      return void 0;
-    }
-    let score = 0;
-    if (response.prompt.options !== void 0 && Object.keys(response.prompt.options).length > 0) {
-      score = response.data?.trim() === response.prompt.answerKey?.trim() ? 1 : 0;
-    } else {
-      score = response.data?.trim() === response.prompt.answer?.trim() ? 1 : 0;
-    }
-    return PromptScoreSchema.parse({
-      ...response,
-      score,
-      scoreDID: uuidv74(),
-      method: ScoringMethods.algo,
-      prompt: response.prompt,
-      scoreMetadata: {
-        scorerIdentifier: this.identifier,
-        extractedAnswer: response.data?.trim()
-      },
-      explanation: void 0
-    });
-  }
-  async canScore(response) {
-    return response.data !== void 0 && response.prompt !== void 0;
-  }
-};
-
-// src/scorers/ref-answer-equality-llm-judge-scorer.ts
-import { z as z7 } from "zod";
-import { v7 as uuidv75 } from "uuid";
-var RefAnswerEqualityLLMJudgeScorer = class extends AbstractScorer {
-  identifier = "ref-answer-equality-llm-judge-scorer";
-  optionsSchema = z7.object({
-    openRouterApiKey: z7.string().optional(),
-    provider: z7.instanceof(BaseLLMProvider).optional(),
-    model: z7.string(),
-    prompt: z7.string().default(`
-Judge whether the following [response] to [question] is correct or not based on the precise and unambiguous [correctAnswer] OR [correctAnswerKey] (if the question was a multiple choice question) below.
-
-[question]: {question}
-
-[response]: {response}
-
-[correctAnswer]: {correctAnswer}
-[correctAnswerKey]: {correctAnswerKey}
-
-Your judgement must be in the format and criteria specified below:
-\`\`\`json
-{
-  "extractedFinalAnswer": "The final exact answer extracted from the [response]. Put the extracted answer as 'None' if there is no exact, final answer to extract from the response.",
-  "reasoning": "Explain why the extractedFinalAnswer is correct or incorrect based on [correctAnswer], focusing only on if there are meaningful differences between [correctAnswer] and the extractedFinalAnswer. Do not comment on any background to the problem, do not attempt to solve the problem, do not argue for any answer different than [correctAnswer], focus only on whether the answers match.",
-  "correct": "Answer 'yes' if extracted_final_answer matches the [correctAnswer] given above, or is within a small margin of error for numerical problems. Answer 'no' otherwise, i.e. if there if there is any inconsistency, ambiguity, non-equivalency, or if the extracted answer is incorrect.",
-  "confidence": "The extracted confidence score between 0% and 100% from [response]. Put 100 if there is no confidence score available."
-}
-\`\`\``),
-    promptSuffix: z7.string().default(""),
-    promptPrefix: z7.string().default("")
-  }).transform((options, ctx) => {
-    if (options.provider !== void 0) {
-      return options;
-    }
-    if (options.openRouterApiKey !== void 0) {
-      return {
-        ...options,
-        provider: new OpenRouterProvider({
-          apiKey: options.openRouterApiKey
-        })
-      };
-    }
-    ctx.addIssue({
-      code: z7.ZodIssueCode.custom,
-      message: "No provider or openRouterApiKey provided"
-    });
-    return z7.NEVER;
-  });
-  async scoreOne(response, options) {
-    if (!await this.canScore(response)) {
-      return void 0;
-    }
-    const parsedOptions = this.optionsSchema.parse(options);
-    const provider = options?.provider ?? new OpenRouterProvider({
-      apiKey: parsedOptions.openRouterApiKey
-    });
-    const prompt = formatString(
-      parsedOptions.promptPrefix + parsedOptions.prompt + parsedOptions.promptSuffix,
-      {
-        question: response.prompt.question.data,
-        response: response.data,
-        correctAnswer: response.prompt.answer,
-        correctAnswerKey: (
-          // If the Prompt is a multiple choice question, then the model might have answered
-          // with the correct answer key rather than the actual answer text so consider that
-          // the answer key is also the correct answer
-          response.prompt.answerKey || "CORRECT ANSWER KEY IS NOT AVAILABLE"
-        )
-      }
-    );
-    const judge = await provider.forward(prompt, {
-      model: parsedOptions.model
-    });
-    const extractedJSON = parseResponseAsJSON(judge.data);
-    debugLog("Extracted JSON from Judge response:", extractedJSON);
-    debugLog("-------------------------------------");
-    let score = 0;
-    if (extractedJSON && typeof extractedJSON === "object") {
-      score = extractedJSON.correct.toLowerCase() === "yes" ? 1 : 0;
-    }
-    const modelInfo = await provider.parseModelInfo(parsedOptions.model);
-    return PromptScoreSchema.parse({
-      ...response,
-      score,
-      scoreDID: uuidv75(),
-      prompt: response.prompt,
-      method: ScoringMethods.ai,
-      scorerAI: {
-        provider: provider.identifier,
-        modelName: modelInfo?.name ?? "unknown",
-        modelHost: modelInfo?.host ?? "auto",
-        modelOwner: modelInfo?.owner ?? "unknown",
-        modelId: parsedOptions.model,
-        inputTokensUsed: judge.inputTokensUsed,
-        outputTokensUsed: judge.outputTokensUsed,
-        inputCost: judge.inputCost,
-        outputCost: judge.outputCost
-      },
-      scoreMetadata: {
-        scorerIdentifier: this.identifier,
-        extractedAnswer: extractedJSON?.extractedFinalAnswer,
-        reasoning: extractedJSON?.reasoning,
-        confidence: extractedJSON?.confidence
-      },
-      explanation: extractedJSON?.reasoning ?? void 0
-    });
-  }
-  async canScore(response) {
-    return response.data !== void 0 && response.prompt !== void 0 && response.prompt.answer !== void 0;
-  }
-};
-
-// src/scorers/similarity-scorer.ts
-import tokenizer from "sbd";
-import { v7 as uuidv76 } from "uuid";
-var SimilarityScorer = class extends AbstractScorer {
-  identifier = "similarity";
-  async scoreOne(response, options) {
-    if (!await this.canScore(response)) {
-      return void 0;
-    }
-    const originalSentences = options?.ignoreCase ? tokenizer.sentences(response.prompt.answer.toLowerCase()) : tokenizer.sentences(response.prompt.answer);
-    const responseSentences = options?.ignoreCase ? tokenizer.sentences(response.data.toLowerCase()) : tokenizer.sentences(response.data);
-    let score = 0;
-    for (let i = 0; i < originalSentences.length; i++) {
-      const originalSentence = originalSentences[i];
-      const responseSentence = responseSentences[i];
-      if (originalSentence === responseSentence) {
-        score += 1;
-      }
-    }
-    return PromptScoreSchema.parse({
-      ...response,
-      scoreDID: uuidv76(),
-      // Calculate the accuracy
-      score: score / originalSentences.length,
-      method: ScoringMethods.algo,
-      prompt: response.prompt,
-      scoreMetadata: {
-        scorerIdentifier: this.identifier,
-        correctPositionedSentences: score,
-        totalSentencesInAnswer: originalSentences.length,
-        totalSentencesInResponse: responseSentences.length
-      },
-      explanation: void 0
-    });
-  }
-  async canScore(response) {
-    return response.data !== void 0 && response.prompt !== void 0 && response.prompt.answer !== void 0;
-  }
-};
-
-// src/scorers/llm-judge-scorer.ts
-import { z as z8 } from "zod";
-import { v7 as uuidv77 } from "uuid";
-var LLMJudgeScorer = class extends AbstractScorer {
-  identifier = "llm-judge";
-  optionsSchema = z8.object({
-    openRouterApiKey: z8.string().optional(),
-    provider: z8.instanceof(BaseLLMProvider).optional(),
-    model: z8.string().default("openai/gpt-4o-mini"),
-    mode: z8.enum(["pointwise", "pairwise"]).default("pointwise"),
-    criteria: z8.array(CriterionSchema).min(1),
-    meta: z8.record(z8.any(), z8.any()).optional(),
-    temperature: z8.number().min(0).max(2).default(0),
-    promptPrefix: z8.string().default(""),
-    promptSuffix: z8.string().default(""),
-    responseB: PromptResponseSchema.optional()
-  }).transform((options, ctx) => {
-    if (options.provider !== void 0) {
-      return options;
-    }
-    if (options.openRouterApiKey !== void 0) {
-      return {
-        ...options,
-        provider: new OpenRouterProvider({
-          apiKey: options.openRouterApiKey
-        })
-      };
-    }
-    ctx.addIssue({
-      code: z8.ZodIssueCode.custom,
-      message: "No provider or openRouterApiKey provided"
-    });
-    return z8.NEVER;
-  });
-  async scoreOne(response, options) {
-    if (!await this.canScore(response, options)) {
-      return void 0;
-    }
-    const parsedOptions = this.optionsSchema.parse(options);
-    const provider = parsedOptions.provider ?? new OpenRouterProvider({
-      apiKey: parsedOptions.openRouterApiKey
-    });
-    if (parsedOptions.mode === "pointwise") {
-      return this.scorePointwise(response, parsedOptions, provider);
-    } else {
-      if (!parsedOptions.responseB) {
-        throw new Error(
-          "Pairwise mode requires responseB to be provided in options."
-        );
-      }
-      return this.scorePairwise(response, parsedOptions, provider);
-    }
-  }
-  systemPrompt() {
-    return [
-      "You are a strict, fair evaluation judge.",
-      "Only use information provided in the task and candidate answers.",
-      "For each criterion, return an integer score within the provided scale and a very brief justification (\u22642 sentences).",
-      "Return only JSON that conforms to the requested schema.",
-      "Do not include chain-of-thought or internal reasoning; just concise justifications."
-    ].join(" ");
-  }
-  async scorePointwise(response, options, provider) {
-    let task = response.prompt.fullPrompt.data;
-    if (response.prompt.type === PromptTypes.MultipleChoice) {
-      task += `
-Correct answer: ${response.prompt.answerKey} - ${response.prompt.answer}`;
-    } else if (response.prompt.answer) {
-      task += `
-Expected answer: ${response.prompt.answer}`;
-    }
-    const norm = this.normalizeWeights(options.criteria);
-    const user = [
-      `TASK:
-${task}`,
-      options.meta ? `
-ADDITIONAL CONTEXT (may include references, constraints, expected behavior):
-${JSON.stringify(options.meta, null, 2)}` : "",
-      `
-RUBRIC:
-${this.renderCriteria(norm)}`,
-      `
-CANDIDATE ANSWER:
-${response.data}`,
-      `
-RESPONSE FORMAT (strict JSON):`,
-      JSON.stringify(
-        {
-          perCriterion: [
-            {
-              id: "<string>",
-              score: "<integer within scale>",
-              justification: "<\u22642 sentences>"
-            }
-          ],
-          overall: "<0..100 integer>",
-          verdict: "<one of: 'strong-pass' | 'pass' | 'borderline' | 'fail'>",
-          notes: "<optional, \u22643 short bullet points>"
-        },
-        null,
-        2
-      )
-    ].join("");
-    const scorePrompt = [
-      user,
-      "\nINSTRUCTIONS:",
-      "- Compute per-criterion integer scores within each scale.",
-      "- Compute weighted overall as a 0-100 integer: normalize weights, map each score to 0-100 by its scale, then weighted average.",
-      "- Choose verdict thresholds: \u226585 strong-pass, 70-84 pass, 60-69 borderline, <60 fail.",
-      "- Output valid JSON only."
-    ].join("\n");
-    const messages = [
-      { role: "system", content: this.systemPrompt() },
-      {
-        role: "user",
-        content: scorePrompt
-      }
-    ];
-    const llmResponse = await provider.forward(messages, {
-      model: options.model
-    });
-    debugLog("LLM Response:", llmResponse.data);
-    const json2 = extractFirstJSON(llmResponse.data);
-    if (!Array.isArray(json2.perCriterion)) {
-      throw new Error("Model did not return perCriterion array.");
-    }
-    debugLog("Extracted JSON from Judge response:", json2);
-    const computedOverall = this.computeOverallScore(json2.perCriterion, norm);
-    const overall = Number.isFinite(Number(json2.overall)) && Number(json2.overall) > 0 ? Number(json2.overall) : computedOverall;
-    const score = Math.min(1, Math.max(0, overall / 100));
-    let explanation = "";
-    if (json2.notes && json2.notes.length > 0) {
-      explanation = json2.notes.join(". ");
-    }
-    for (const criterion of json2.perCriterion) {
-      explanation += [
-        `Criteria: ${criterion.id}`,
-        `Score: ${criterion.score}`,
-        `Justification:
-${criterion.justification}
-`
-      ].join("\n");
-    }
-    return PromptScoreSchema.parse({
-      ...response,
-      score,
-      scoreDID: uuidv77(),
-      prompt: response.prompt,
-      method: ScoringMethods.ai,
-      scorerAI: {
-        provider: provider.identifier,
-        modelId: options.model,
-        modelHost: "auto",
-        modelName: "unknown",
-        modelOwner: "unknown",
-        inputTokensUsed: llmResponse.inputTokensUsed,
-        outputTokensUsed: llmResponse.outputTokensUsed,
-        inputCost: llmResponse.inputCost,
-        outputCost: llmResponse.outputCost
-      },
-      scoreMetadata: {
-        overall,
-        scorerIdentifier: this.identifier,
-        mode: "pointwise",
-        perCriterion: json2.perCriterion,
-        verdict: json2.verdict,
-        scorePrompt,
-        systemPrompt: this.systemPrompt()
-      },
-      explanation: explanation || void 0
-    });
-  }
-  async scorePairwise(response, options, provider) {
-    if (!options.responseB || !response.data || !options.responseB.data) {
-      return void 0;
-    }
-    const norm = this.normalizeWeights(options.criteria);
-    const user = [
-      `TASK:
-${response.prompt.fullPrompt.data}`,
-      options.meta ? `
-ADDITIONAL CONTEXT:
-${JSON.stringify(options.meta, null, 2)}` : "",
-      `
-RUBRIC:
-${this.renderCriteria(norm)}`,
-      `
-CANDIDATE A:
-${response.data}`,
-      `
-CANDIDATE B:
-${options.responseB.data}`,
-      `
-RESPONSE FORMAT (strict JSON):`,
-      JSON.stringify(
-        {
-          winner: "<'A' | 'B' | 'tie'>",
-          confidence: "<integer 1..5>",
-          rationale: "<\u22643 sentences>",
-          perCriterion: [
-            {
-              id: "<string>",
-              better: "<'A'|'B'|'tie'>",
-              justification: "<\u22642 sentences>"
-            }
-          ]
-        },
-        null,
-        2
-      )
-    ].join("");
-    const systemPrompt = [
-      this.systemPrompt(),
-      "When comparing, do not reward verbosity or length. Prefer factual accuracy, adherence to instructions, safety, and clarity."
-    ].join(" ");
-    const messages = [
-      {
-        role: "system",
-        content: systemPrompt
-      },
-      { role: "user", content: user }
-    ];
-    const content = await provider.forward(messages, {
-      model: options.model
-    });
-    const json2 = extractFirstJSON(content.data);
-    debugLog("Original Pairwise Judge response:", content.data);
-    debugLog("Extracted JSON from Pairwise Judge response:", json2);
-    if (!json2 || !["A", "B", "tie"].includes(json2.winner)) {
-      throw new Error("Model returned invalid pairwise result.");
-    }
-    const confidence = Math.max(1, Math.min(5, Number(json2.confidence) || 3));
-    let score = 0.5;
-    if (json2.winner === "A") {
-      score = 1;
-    } else if (json2.winner === "B") {
-      score = 0;
-    }
-    let explanation = "";
-    if (json2.rationale) {
-      explanation = json2.rationale;
-    }
-    for (const criterion of json2.perCriterion) {
-      explanation += [
-        `Criteria: ${criterion.id}`,
-        `Better: ${criterion.better}`,
-        `Justification:
-${criterion.justification}
-`
-      ].join("\n");
-    }
-    return PromptScoreSchema.parse({
-      ...response,
-      score,
-      scoreDID: uuidv77(),
-      prompt: response.prompt,
-      method: ScoringMethods.ai,
-      scorerAI: {
-        provider: provider.identifier,
-        modelId: options.model,
-        modelHost: "auto",
-        modelName: "unknown",
-        modelOwner: "unknown",
-        inputTokensUsed: content.inputTokensUsed,
-        outputTokensUsed: content.outputTokensUsed,
-        inputCost: content.inputCost,
-        outputCost: content.outputCost
-      },
-      scoreMetadata: {
-        scorerIdentifier: this.identifier,
-        mode: "pairwise",
-        winner: json2.winner,
-        confidence,
-        rationale: json2.rationale,
-        perCriterion: json2.perCriterion,
-        responseB: options.responseB.data,
-        systemPrompt,
-        scorePrompt: user
-      },
-      explanation: explanation || void 0
-    });
-  }
-  /**
-   * Build the pairwise evaluation prompt
-   */
-  buildPairwisePrompt(task, answerA, answerB, criteria, meta, prefix = "", suffix = "") {
-    const rubric = this.renderCriteria(criteria);
-    const userPrompt = [
-      `TASK:
-${task}`,
-      meta ? `
-ADDITIONAL CONTEXT:
-${JSON.stringify(meta, null, 2)}` : "",
-      `
-RUBRIC:
-${rubric}`,
-      `
-CANDIDATE A:
-${answerA}`,
-      `
-CANDIDATE B:
-${answerB}`,
-      `
-RESPONSE FORMAT (strict JSON):`,
-      JSON.stringify(
-        {
-          winner: "<'A' | 'B' | 'tie'>",
-          confidence: "<integer 1..5>",
-          rationale: "<\u22643 sentences>",
-          perCriterion: [
-            {
-              id: "<string>",
-              better: "<'A'|'B'|'tie'>",
-              justification: "<\u22642 sentences>"
-            }
-          ]
-        },
-        null,
-        2
-      )
-    ].join("\n");
-    return `${prefix}${userPrompt}${suffix}`;
-  }
-  /**
-   * Normalize criterion weights to sum to 1
-   */
-  normalizeWeights(criteria) {
-    const sum = criteria.reduce((a, c) => a + (c.weight ?? 1), 0) || 1;
-    return criteria.map((c) => ({ ...c, weight: (c.weight ?? 1) / sum }));
-  }
-  /**
-   * Render criteria as a formatted string
-   */
-  renderCriteria(criteria) {
-    return criteria.map((c, i) => {
-      const mn = c.scale?.min ?? 0;
-      const mx = c.scale?.max ?? 5;
-      return `${i + 1}. id="${c.id}" (weight=${c.weight}, scale=${mn}..${mx}) \u2014 ${c.description}`;
-    }).join("\n");
-  }
-  /**
-   * Compute overall score from per-criterion scores
-   */
-  computeOverallScore(perCriterion, criteria) {
-    let total = 0;
-    for (const pc of perCriterion) {
-      const criterion = criteria.find((c) => c.id === pc.id);
-      const { min, max, weight } = {
-        min: criterion?.scale?.min ?? 0,
-        max: criterion?.scale?.max ?? 5,
-        weight: criterion?.weight ?? 0
-      };
-      const score = Number(pc.score);
-      if (!Number.isFinite(score)) continue;
-      const clamped = Math.max(min, Math.min(max, score));
-      const normalized100 = max === min ? 0 : (clamped - min) / (max - min) * 100;
-      total += normalized100 * weight;
-    }
-    return Math.round(total);
-  }
-  /**
-   * Extract criteria IDs from prompt text (fallback)
-   */
-  extractCriteria(promptText) {
-    const ids = [];
-    const re = /\d+\. id="([^"]+)"/g;
-    let m;
-    while ((m = re.exec(promptText)) !== null) {
-      ids.push(m[1]);
-    }
-    return ids.length ? ids : ["correctness", "instruction_following", "clarity"];
-  }
-  async canScore(response, options) {
-    const hasValidResponse = response.data !== void 0 && response.prompt !== void 0;
-    if (!hasValidResponse) {
-      return false;
-    }
-    if (options?.mode === "pairwise") {
-      return options.responseB !== void 0 && options.responseB.data !== void 0 && options.responseB.prompt !== void 0;
-    }
-    return true;
-  }
-};
-function extractFirstJSON(maybeJSON) {
-  try {
-    return JSON.parse(maybeJSON);
-  } catch {
-  }
-  const start = maybeJSON.indexOf("{");
-  const end = maybeJSON.lastIndexOf("}");
-  if (start >= 0 && end > start) {
-    const slice = maybeJSON.slice(start, end + 1);
-    try {
-      return JSON.parse(slice);
-    } catch {
-    }
-  }
-  throw new Error("Failed to parse model response as JSON.");
-}
-var CriterionSchema = z8.object({
-  id: z8.string(),
-  description: z8.string(),
-  weight: z8.number().min(0).max(1).default(1),
-  scale: z8.object({
-    min: z8.number().default(0),
-    max: z8.number().default(5)
-  }).default({ min: 0, max: 5 })
-});
-
-// src/generators/mcq-generator.ts
-var MCQGenerator = class extends AbstractGenerator {
-  identifier = "mcq";
-  inputSchema = z9.array(z9.any());
-  optionsSchema = z9.object(
-    {
-      /**
-       * The API key for the OpenRouter provider. Won't be used if `provider` is given.
-       */
-      openRouterApiKey: z9.string().optional(),
-      /**
-       * The Provider that is going to be used to generate the Prompts.
-       */
-      provider: z9.instanceof(BaseLLMProvider).optional(),
-      /**
-       * The model that is going to be used to generate the prompts
-       */
-      model: z9.string(),
-      systemPromptPrefix: z9.string().optional().default(""),
-      /**
-       * Additional rules that will be appended to the system prompt.
-       */
-      systemPromptRules: z9.array(z9.string()).default([]),
-      systemPrompt: z9.string().optional().default(`Take the input text and find a difficult question about the content. The question should test understanding of the key concepts, facts or relationships described in the text. The question should include enough context to answer it without the need to read the text. The question must not refer to the text in any way. The question must not want a very specific values such as p-values or time frames like when effect was on day. Each option must not be longer than 5-7 words.
-
-You need to generate a multiple choice question with at least 8 options. Some of the options can be real terms from the domain, others can be plausible-sounding terms that you invent to sound authentic to the subject matter.
-{rules}
-Your output strictly forced to be a JSON object which applies the following schema:
-\`\`\`json
-{
-  "question": "Question text",
-  "options": {
-    "A": "Option A text",
-    "B": "Option B text",
-    "C": "Option C text",
-    "D": "Option D text"
-    // ....
-  },
-  "answerKey": "C",
-  "answer": "Option C text"
-}
-\`\`\`
-`),
-      systemPromptSuffix: z9.string().optional().default(""),
-      /**
-       * The main function that parses the input value to a string.
-       */
-      parseInput: z9.function().args(z9.any()).returns(z9.string()),
-      /**
-       * Whether to include the original data as a field inside the metadata.
-       */
-      includeOriginalInputAsMetadata: z9.boolean().optional().default(false),
-      /**
-       * Additional metadata that is going to be added to the each Prompt.
-       * If a function is given then that function takes an argument which
-       * is the input value for each item and generates the additional metadata.
-       */
-      additionalMetadata: z9.union([
-        z9.function().args(z9.any(), z9.object({ systemPrompt: z9.string() })).returns(z9.record(z9.string(), z9.any())),
-        z9.record(z9.string(), z9.any())
-      ]).optional()
-    },
-    { message: "No options provided" }
-  ).transform((options, ctx) => {
-    if (options.provider !== void 0) {
-      return options;
-    }
-    if (options.openRouterApiKey !== void 0) {
-      return {
-        ...options,
-        provider: new OpenRouterProvider({
-          apiKey: options.openRouterApiKey
-        })
-      };
-    }
-    ctx.addIssue({
-      code: z9.ZodIssueCode.custom,
-      message: "No provider or openRouterApiKey provided"
-    });
-    return z9.NEVER;
-  });
-  async generatePrompts(input, options) {
-    const parsedOptions = this.optionsSchema.parse(options);
-    const systemPrompt = formatString(
-      parsedOptions.systemPromptPrefix + parsedOptions.systemPrompt + parsedOptions.systemPromptSuffix,
-      {
-        rules: parsedOptions.systemPromptRules.length > 0 ? `
-Also strictly follow the rules below:
-${parsedOptions.systemPromptRules.join("\n")}
-` : ""
-      }
-    );
-    const results = await Promise.all(
-      input.map(
-        (item) => this.generateFromInput(
-          item,
-          parsedOptions.provider,
-          parsedOptions,
-          systemPrompt
-        )
-      )
-    );
-    return results.filter((result) => result !== null);
-  }
-  async generateFromInput(input, provider, options, systemPrompt) {
-    const inputText = options.parseInput(input);
-    const response = await provider.forward(inputText, {
-      model: options.model,
-      system: systemPrompt
-    });
-    const mcq = parseResponseAsJSON(response.data);
-    if (!mcq) {
-      debugLog("Couldn't parsed the LLM response as a JSON:", response.data);
-      return null;
-    }
-    const fullPrompt = preparePrompt(mcq.question, mcq.options);
-    const additionalMetadata = typeof options.additionalMetadata === "function" ? options.additionalMetadata(input, { systemPrompt }) : options.additionalMetadata || {};
-    return await this.buildPrompt({
-      correctAnswer: mcq.answerKey || mcq.answer,
-      question: mcq.question,
-      options: mcq.options || {},
-      fullPrompt,
-      type: PromptTypes.MultipleChoice,
-      metadata: {
-        tags: [
-          `generator-${this.identifier}`,
-          // Expand the tags field if it is provided within the additional metadata
-          ...Array.isArray(additionalMetadata.tags) ? additionalMetadata.tags : []
-        ],
-        model: options.model,
-        generatorIdentifier: this.identifier,
-        generatorClassName: this.constructor.name,
-        // Input might be an object but input text will be a string
-        originalSourceInput: options.includeOriginalInputAsMetadata ? input : void 0,
-        originalSourceInputText: inputText,
-        ...additionalMetadata
-      },
-      scorers: [
-        // TODO: We should be able to access identifier without instantiating an object
-        new MultipleChoiceScorer().identifier,
-        new RefAnswerEqualityLLMJudgeScorer().identifier,
-        new ExactMatchScorer().identifier
-      ]
-    });
-  }
-};
-
-// src/generators/open-ended-generator.ts
-import { z as z10 } from "zod";
-var OpenEndedGenerator = class extends AbstractGenerator {
-  identifier = "open-ended";
-  inputSchema = z10.array(z10.any());
-  optionsSchema = z10.object(
-    {
-      /**
-       * The API key for the OpenRouter provider. Won't be used if `provider` is given.
-       */
-      openRouterApiKey: z10.string().optional(),
-      /**
-       * The Provider that is going to be used to generate the Prompts.
-       */
-      provider: z10.instanceof(BaseLLMProvider).optional(),
-      /**
-       * The model that is going to be used to generate the prompts
-       */
-      model: z10.string(),
-      /**
-       * Adds an additional clue to the end of the question such as "Answer with a positive number"
-       * which is relevant with the answer type expected.
-       */
-      addClueToQuestion: z10.boolean().optional().default(true),
-      /**
-       * Additional rules that will be appended to the system prompt.
-       */
-      systemPromptRules: z10.array(z10.string()).default([]),
-      systemPromptPrefix: z10.string().optional().default(""),
-      systemPrompt: z10.string().optional().default(`Take the input text and find a difficult question about the content. The question should test understanding of the key concepts, facts or relationships described in the text. The question should include enough context to answer it without the need to read the text. The question must not refer to the text in any way. The question must not want a very specific values such as p-values or time frames like when effect was on day. Each option must not be longer than 5-7 words. {clue}
-{rules}
-Your output strictly forced to be a JSON object which applies the following schema:
-\`\`\`json
-{
-  "question": "Question text",
-  "answer": "Correct answer"
-}
-\`\`\`
-  `),
-      systemPromptSuffix: z10.string().optional().default(""),
-      /**
-       * The main function that parses the input value to a string.
-       */
-      parseInput: z10.function().args(z10.any()).returns(z10.string()),
-      /**
-       * Whether to include the original as a field inside the metadata.
-       */
-      includeOriginalInputAsMetadata: z10.boolean().optional().default(false),
-      /**
-       * Additional metadata that is going to be added to the each Prompt.
-       * If a function is given then that function takes an argument which
-       * is the input value for each item and generates the additional metadata.
-       */
-      additionalMetadata: z10.union([
-        z10.function().args(z10.any(), z10.object({ systemPrompt: z10.string() })).returns(z10.record(z10.string(), z10.any())),
-        z10.record(z10.string(), z10.any())
-      ]).optional()
-    },
-    { message: "No options provided" }
-  ).transform((options, ctx) => {
-    if (options.provider !== void 0) {
-      return options;
-    }
-    if (options.openRouterApiKey !== void 0) {
-      return {
-        ...options,
-        provider: new OpenRouterProvider({
-          apiKey: options.openRouterApiKey
-        })
-      };
-    }
-    ctx.addIssue({
-      code: z10.ZodIssueCode.custom,
-      message: "No provider or openRouterApiKey provided"
-    });
-    return z10.NEVER;
-  });
-  async generatePrompts(input, options) {
-    const parsedOptions = this.optionsSchema.parse(options);
-    const provider = options?.provider ?? new OpenRouterProvider({
-      apiKey: parsedOptions.openRouterApiKey
-    });
-    const systemPrompt = formatString(
-      parsedOptions.systemPromptPrefix + parsedOptions.systemPrompt + parsedOptions.systemPromptSuffix,
-      {
-        clue: parsedOptions.addClueToQuestion ? `The question must ends with a sentence that says "Answer with ..." where "..." represents the type of answer expected.` : "",
-        rules: parsedOptions.systemPromptRules.length > 0 ? `
-Also strictly follow the rules below:
-${parsedOptions.systemPromptRules.join("\n")}
-` : ""
-      }
-    );
-    const results = await Promise.all(
-      input.map(
-        (item) => this.generateFromInput(item, provider, parsedOptions, systemPrompt)
-      )
-    );
-    return results.filter((result) => result !== null);
-  }
-  async generateFromInput(input, provider, options, systemPrompt) {
-    const inputText = options.parseInput(input);
-    const response = await provider.forward(inputText, {
-      model: options.model,
-      system: systemPrompt
-    });
-    const q = parseResponseAsJSON(response.data);
-    if (!q) {
-      debugLog("Couldn't parsed the LLM response as a JSON:", response.data);
-      return null;
-    }
-    const additionalMetadata = typeof options.additionalMetadata === "function" ? options.additionalMetadata(input, { systemPrompt }) : options.additionalMetadata || {};
-    return this.buildPrompt({
-      correctAnswer: q.answer,
-      question: q.question,
-      options: {},
-      fullPrompt: q.question,
-      type: PromptTypes.OpenEnded,
-      metadata: {
-        tags: [
-          `generator-${this.identifier}`,
-          // Expand the tags field if it is provided within the additional metadata
-          ...Array.isArray(additionalMetadata.tags) ? additionalMetadata.tags : []
-        ],
-        brainModel: options.model,
-        model: options.model,
-        promptGenClassName: this.constructor.name,
-        promptGenJSClassName: this.constructor.name,
-        // Input might be an object but input text will be a string
-        originalSourceInput: options.includeOriginalInputAsMetadata ? input : void 0,
-        originalSourceInputText: inputText,
-        ...additionalMetadata
-      },
-      scorers: [
-        // TODO: We should be able to access identifier without instantiating an object
-        new RefAnswerEqualityLLMJudgeScorer().identifier
-      ]
-    });
-  }
-};
-
-// src/registries/abstract/abstract-registry.ts
-var AbstractRegistry = class {
-};
-
-// src/registries/peerbench-registry.ts
-import { createClient } from "@supabase/supabase-js";
-import axios2 from "axios";
-var PeerBenchRegistry = class _PeerBenchRegistry extends AbstractRegistry {
-  // TODO: Find a better way to force sub classes to define a static identifier
-  // Static accessor for the identifier
-  static identifier = "peerbench";
-  identifier = _PeerBenchRegistry.identifier;
-  token;
-  supabaseClient;
-  apiURL;
-  session = null;
-  email;
-  password;
-  authMethod;
-  refreshTokenInterval;
-  isRefreshingToken = false;
-  isClosed = false;
-  isInitialized = false;
-  tokenRefresher;
-  constructor(options) {
-    super();
-    this.tokenRefresher = options.tokenRefresher ?? false;
-    this.authMethod = options.authMethod ?? "token";
-    this.apiURL = options.peerbenchApiURL;
-    if (options.authMethod === "token" || options.authMethod === void 0) {
-      this.supabaseClient = createClient(
-        options.peerbenchSupabaseURL,
-        options.peerbenchSupabaseAnonKey
-      );
-      this.email = options.email;
-      this.password = options.password;
-    }
-  }
-  async uploadPrompts(prompts, options) {
-    if (prompts.length === 0) {
-      return 0;
-    }
-    await this.init();
-    const account = options?.account;
-    if (account && account.signMessage) {
-      prompts = await Promise.all(
-        prompts.map(async (prompt) => {
-          const promptWithSignature = {
-            ...prompt
-          };
-          const promptStringified = stableStringify(prompt);
-          const hash = await calculateCID(promptStringified).then(
-            (c) => c.toString()
-          );
-          const signature = await account.signMessage({
-            message: hash
-          });
-          promptWithSignature.signature = signature;
-          promptWithSignature.publicKey = account.address;
-          promptWithSignature.signatureType = "cid";
-          promptWithSignature.keyType = "secp256k1n";
-          return promptWithSignature;
-        })
-      );
-    }
-    const res = await axios2.post(
-      `${this.apiURL}/api/v2/prompts`,
-      {
-        promptSetId: options.promptSetId,
-        prompts
-      },
-      {
-        withCredentials: this.authMethod === "cookie",
-        headers: {
-          Authorization: this.authMethod === "token" ? `Bearer ${this.token}` : void 0,
-          "Content-Type": "application/json"
-        }
-      }
-    );
-    if (res.status !== 200) {
-      throw new Error(
-        `Failed to upload Prompts: ${res?.data?.message || JSON.stringify(res?.data || "No response available")}`
-      );
-    }
-    return prompts.length;
-  }
-  async uploadResponses(responses, options) {
-    if (responses.length === 0) {
-      return 0;
-    }
-    await this.init();
-    const account = options?.account;
-    if (account && account.signMessage) {
-      responses = await Promise.all(
-        responses.map(async (response) => {
-          const responseWithSignature = {
-            ...response
-          };
-          const responseStringified = stableStringify(response);
-          const hash = await calculateCID(responseStringified).then(
-            (c) => c.toString()
-          );
-          const signature = await account.signMessage({
-            message: hash
-          });
-          responseWithSignature.signature = signature;
-          responseWithSignature.publicKey = account.address;
-          responseWithSignature.signatureType = "cid";
-          responseWithSignature.keyType = "secp256k1n";
-          return responseWithSignature;
-        })
-      );
-    }
-    const res = await axios2.post(
-      `${this.apiURL}/api/v2/responses`,
-      { responses },
-      {
-        withCredentials: this.authMethod === "cookie",
-        headers: {
-          Authorization: this.authMethod === "token" ? `Bearer ${this.token}` : void 0,
-          "Content-Type": "application/json"
-        }
-      }
-    );
-    if (res.status !== 200) {
-      throw new Error(
-        `Failed to upload Responses: ${res?.data?.message || JSON.stringify(res?.data || "No response available")}`
-      );
-    }
-    return responses.length;
-  }
-  async uploadScores(scores, options) {
-    if (scores.length === 0) {
-      return 0;
-    }
-    await this.init();
-    scores = await Promise.all(
-      scores.map(async (score) => {
-        if (!score.prompt) {
-          throw new Error("Score must have a prompt for hash registration");
-        }
-        if (!score.data) {
-          throw new Error("Score must have data for hash registration");
-        }
-        const promptStringified = stableStringify(score.prompt);
-        const promptHashSha256Registration = await calculateSHA256(promptStringified);
-        const promptHashCIDRegistration = await calculateCID(
-          promptStringified
-        ).then((c) => c.toString());
-        const responseObject = PromptResponseSchema.parse(score);
-        const responseStringified = stableStringify(responseObject);
-        const responseHashSha256Registration = await calculateSHA256(responseStringified);
-        const responseHashCIDRegistration = await calculateCID(
-          responseStringified
-        ).then((c) => c.toString());
-        const scoreWithRegistration = {
-          ...score,
-          responseHashSha256Registration,
-          responseHashCIDRegistration,
-          promptHashSha256Registration,
-          promptHashCIDRegistration
-        };
-        if (options?.account && options.account.signMessage) {
-          const scoreStringified = stableStringify(score);
-          const hash = await calculateCID(scoreStringified).then(
-            (c) => c.toString()
-          );
-          const signature = await options.account.signMessage({
-            message: hash
-          });
-          scoreWithRegistration.signature = signature;
-          scoreWithRegistration.publicKey = options.account.address;
-          scoreWithRegistration.signatureType = "cid";
-          scoreWithRegistration.keyType = "secp256k1n";
-        }
-        return scoreWithRegistration;
-      })
-    );
-    const res = await axios2.post(
-      `${this.apiURL}/api/v2/scores`,
-      { scores },
-      {
-        withCredentials: this.authMethod === "cookie",
-        headers: {
-          Authorization: this.authMethod === "token" ? `Bearer ${this.token}` : void 0,
-          "Content-Type": "application/json"
-        }
-      }
-    );
-    if (res.status !== 200) {
-      throw new Error(
-        `Failed to upload Scores: ${res?.data?.message || JSON.stringify(res?.data || "No response available")}`
-      );
-    }
-    return scores.length;
-  }
-  /**
-   * Clears the interval execution for refreshing the token.
-   */
-  async clearRefreshInterval() {
-    this.isClosed = true;
-    clearInterval(this.refreshTokenInterval);
-  }
-  async init() {
-    if (this.isClosed || this.isInitialized || this.authMethod === "cookie") {
-      return;
-    }
-    const authData = await this.login(this.email, this.password);
-    this.session = authData?.session || null;
-    this.token = this.session?.access_token;
-    if (!this.token) {
-      throw new Error(
-        `Failed authentication with peerBench: No token received`
-      );
-    }
-    if (this.tokenRefresher) {
-      this.refreshTokenInterval = setInterval(
-        () => this.refreshToken(),
-        (this.session.expires_in - 15 * 60) * 1e3
-      );
-    }
-    this.isInitialized = true;
-  }
-  /**
-   * Refreshes the token if it is about to expire.
-   */
-  async refreshToken() {
-    if (this.isClosed || this.authMethod === "cookie") {
-      return;
-    }
-    if (this.isRefreshingToken) {
-      return;
-    }
-    this.isRefreshingToken = true;
-    while (!this.isClosed) {
-      try {
-        const { data, error } = await this.supabaseClient.auth.refreshSession(
-          this.session || void 0
-        );
-        if (error) {
-          throw new Error(error.message);
-        }
-        this.session = data.session;
-        this.token = this.session?.access_token;
-        break;
-      } catch (err) {
-        console.error(`PeerBenchRegistry: Failed to refresh token`, err);
-        console.error(`PeerBenchRegistry: Retrying in 10 seconds`);
-        await sleep(1e4);
-      }
-    }
-    this.isRefreshingToken = false;
-  }
-  async login(email, password) {
-    const { data, error } = await this.supabaseClient.auth.signInWithPassword({
-      email,
-      password
-    });
-    if (error) {
-      throw new Error(`Failed login to peerBench: ${error.message}`);
-    }
-    if (!data.session) {
-      throw new Error(`No session returned from peerBench authentication`);
-    }
-    return data;
-  }
-};
-
-// src/parsers/abstract-parser.ts
-var AbstractParser = class {
-  /**
-   * String identifier of the parser. Must be
-   * set to distunguish the parser from the others.
-   */
-  static identifier = "";
-  /**
-   * Returns the identifier of the parser.
-   */
-  getIdentifier() {
-    return this.constructor?.identifier || this?.identifier;
-  }
-};
-
-// src/parsers/generic-array.ts
-var GenericArrayParser = class extends AbstractParser {
-  static identifier = "generic-array";
-  promptBuilder;
-  responseBuilder;
-  scoreBuilder;
-  constructor(options) {
-    super();
-    this.promptBuilder = options.promptBuilder;
-    this.responseBuilder = options.responseBuilder;
-    this.scoreBuilder = options.scoreBuilder;
-  }
-  async parseFile(path) {
-    const content = await readFile(path);
-    return this.parseContent(content);
-  }
-  async parseContent(content) {
-    let data;
-    const contentString = typeof content === "string" ? content : bufferToString(content);
-    data = tryParseJson(contentString);
-    if (!data) {
-      data = parseJSONL(contentString);
-    }
-    if (!data || !Array.isArray(data) || data.length === 0) {
-      throw new InvalidDataError();
-    }
-    const includedPromptCIDs = {};
-    const includedResponseDIDs = {};
-    const includedScoreDIDs = {};
-    const result = {
-      prompts: [],
-      responses: [],
-      scores: []
-    };
-    for (const item of data) {
-      const prompt = await this.promptBuilder(item, { result });
-      let response;
-      let score;
-      if (prompt) {
-        if (!includedPromptCIDs[prompt.fullPrompt.cid]) {
-          includedPromptCIDs[prompt.fullPrompt.cid] = prompt.did;
-          result.prompts.push(prompt);
-        }
-      }
-      if (this.responseBuilder) {
-        response = await this.responseBuilder(item, { result, prompt });
-        if (response) {
-          if (includedPromptCIDs[response.prompt.fullPrompt.cid]) {
-            const cid = response.prompt.fullPrompt.cid;
-            response.prompt = result.prompts.find(
-              (p) => p.fullPrompt.cid === cid
-            );
-          }
-          if (!includedResponseDIDs[response.did]) {
-            includedResponseDIDs[response.did] = response.did;
-            result.responses.push(response);
-          }
-        }
-      }
-      if (this.scoreBuilder) {
-        score = await this.scoreBuilder(item, { result, prompt, response });
-        if (score) {
-          if (score.prompt && includedPromptCIDs[score.prompt.fullPrompt.cid]) {
-            const cid = score.prompt.fullPrompt.cid;
-            score.prompt = result.prompts.find(
-              (p) => p.fullPrompt.cid === cid
-            );
-          }
-          if (!includedScoreDIDs[score.did]) {
-            includedScoreDIDs[score.did] = score.did;
-            result.scores.push(score);
-          }
-        }
-      }
-    }
-    return result;
-  }
-};
-
-// src/parsers/medqa.ts
-import { parquetReadObjects } from "hyparquet";
-import { z as z11 } from "zod";
-var MedQAPromptObjectSchema = z11.object({
-  question: z11.string(),
-  options: z11.record(z11.string(), z11.string()),
-  answer: z11.string(),
-  answer_idx: z11.string(),
-  meta_info: z11.string().optional()
-});
-var MedQAParser = class extends AbstractParser {
-  static identifier = "medqa";
-  async parseFile(path) {
-    const content = await readFile(path);
-    return this.parseContent(content);
-  }
-  async parseContent(content) {
-    let data;
-    if (content instanceof ArrayBuffer) {
-      try {
-        data = await parquetReadObjects({ file: content });
-      } catch {
-      }
-    }
-    if (!data) {
-      const contentString = typeof content === "string" ? content : bufferToString(content);
-      data = tryParseJson(contentString);
-      if (!data) {
-        data = parseJSONL(contentString);
-      }
-    }
-    if (!data || data.length == 0) {
-      throw new InvalidDataError();
-    }
-    const prompts = [];
-    const validation = z11.array(MedQAPromptObjectSchema).safeParse(data);
-    if (!validation.success) {
-      throw new ParserIsNotCompatibleError();
-    }
-    const validatedData = validation.data;
-    let rowNumber = 0;
-    for (const rawPrompt of validatedData) {
-      const fullPrompt = preparePrompt(rawPrompt.question, rawPrompt.options);
-      prompts.push(
-        await buildPrompt({
-          options: rawPrompt.options,
-          question: rawPrompt.question,
-          type: PromptTypes.MultipleChoice,
-          answer: rawPrompt.answer,
-          answerKey: rawPrompt.answer_idx,
-          fullPrompt,
-          metadata: {
-            medqaCategory: rawPrompt.meta_info,
-            rowNumberInSource: rowNumber
-          }
-        })
-      );
-      rowNumber++;
-    }
-    return {
-      prompts,
-      responses: [],
-      scores: []
-    };
-  }
-};
-
-// src/parsers/mmlu-pro.ts
-import { parquetReadObjects as parquetReadObjects2 } from "hyparquet";
-import { z as z12 } from "zod";
-var MMLUProTaskZodSchema = z12.object({
-  question_id: z12.coerce.number(),
-  question: z12.string(),
-  options: z12.array(z12.string()),
-  answer: z12.string(),
-  answer_index: z12.coerce.number(),
-  cot_content: z12.string(),
-  category: z12.string(),
-  src: z12.string()
-});
-var MMLUProParser = class extends AbstractParser {
-  static identifier = "mmlu-pro";
-  async parseFile(path) {
-    const content = await readFile(path);
-    return this.parseContent(content);
-  }
-  async parseContent(content) {
-    let data;
-    if (content instanceof ArrayBuffer) {
-      try {
-        data = await parquetReadObjects2({ file: content });
-      } catch {
-      }
-    }
-    if (!data) {
-      const contentString = typeof content === "string" ? content : bufferToString(content);
-      data = tryParseJson(contentString);
-      if (!data) {
-        data = parseJSONL(contentString);
-      }
-    }
-    if (!data || data.length == 0) {
-      throw new InvalidDataError();
-    }
-    const prompts = [];
-    const validation = z12.array(MMLUProTaskZodSchema).safeParse(data);
-    if (!validation.success) {
-      throw new ParserIsNotCompatibleError();
-    }
-    const validatedData = validation.data;
-    let rowNumber = 0;
-    for (const rawPrompt of validatedData) {
-      const options = {};
-      let answerKey = "";
-      for (let i = 0; i < rawPrompt.options.length; i++) {
-        const option = rawPrompt.options[i];
-        const letter = String.fromCharCode(65 + i);
-        options[letter] = option;
-        if (i === rawPrompt.answer_index) {
-          answerKey = letter;
-        }
-      }
-      const fullPrompt = preparePrompt(rawPrompt.question, options);
-      prompts.push(
-        await buildPrompt({
-          options,
-          question: rawPrompt.question,
-          answerKey,
-          answer: rawPrompt.options[rawPrompt.answer_index],
-          fullPrompt,
-          type: PromptTypes.MultipleChoice,
-          metadata: {
-            mmluProCategory: rawPrompt.category,
-            rowNumberInSource: rowNumber,
-            mmluProQuestionId: rawPrompt.question_id,
-            mmluProCotContent: rawPrompt.cot_content,
-            mmluProSource: rawPrompt.src
-          }
-        })
-      );
-      rowNumber++;
-    }
-    return {
-      prompts,
-      responses: [],
-      scores: []
-    };
-  }
-};
-
-// src/parsers/pb.ts
-var PBParser = class extends GenericArrayParser {
-  static identifier = "pb";
-  constructor() {
-    super({
-      promptBuilder: (data) => {
-        const promptValidation = PromptSchema.safeParse(data);
-        if (!promptValidation.success) {
-          const responseValidation = PromptResponseSchema.safeParse(data);
-          if (!responseValidation.success) {
-            return;
-          }
-          return responseValidation.data.prompt;
-        }
-        return promptValidation.data;
-      },
-      responseBuilder: (data) => PromptResponseSchema.safeParse(data).data,
-      scoreBuilder: (data, context) => {
-        const validation = PromptScoreSchema.safeParse(data);
-        if (!validation.success) {
-          if (!context.prompt && !context.response) {
-            throw new InvalidDataError();
-          }
-          return;
-        }
-        return validation.data;
-      }
-    });
-  }
-};
-
-// src/parsers/data-parser.ts
-var DataParser = class {
-  static parsers = /* @__PURE__ */ new Map();
-  static {
-    const medqa = new MedQAParser();
-    this.parsers.set(MedQAParser.identifier, medqa);
-    const mmluPro = new MMLUProParser();
-    this.parsers.set(MMLUProParser.identifier, mmluPro);
-    const pb = new PBParser();
-    this.parsers.set(PBParser.identifier, pb);
-  }
-  /**
-   * Tries to parse data from a file path with one of the available parsers.
-   * @example
-   * ```typescript
-   * const { result, parser } = await DataParser.parseFile("path/to/file.json");
-   * ```
-   */
-  static async parseFile(path) {
-    if (typeof window === "undefined") {
-      const { statSync } = await import("node:fs");
-      if (!statSync(path, { throwIfNoEntry: false })?.isFile()) {
-        throw new Error(`File does not exist: ${path}`);
-      }
-      for (const parser of this.parsers.values()) {
-        try {
-          return {
-            result: await parser.parseFile(path),
-            parser
-          };
-        } catch {
-          continue;
-        }
-      }
-      throw new Error(`No parser could parse the file: ${path}`);
-    } else {
-      throw new Error(
-        "File system operations are not supported in browser environment. Use `parseContent` instead."
-      );
-    }
-  }
-  /**
-   * Tries to parse data from a string or ArrayBuffer content with one of the available parsers.
-   * @example
-   * ```typescript
-   * // With string content
-   * const { result, parser } = await DataParser.parseContent('{"question": "What is...", ...}');
-   *
-   * // With file upload in browser
-   * const file = event.target.files[0];
-   * const content = await file.arrayBuffer();
-   * const { result, parser } = await DataParser.parseContent(content);
-   * ```
-   */
-  static async parseContent(content) {
-    for (const parser of this.parsers.values()) {
-      try {
-        return {
-          result: await parser.parseContent(content),
-          parser
-        };
-      } catch {
-        continue;
-      }
-    }
-    throw new Error("No parser could parse the content");
-  }
-};
-
-// src/constants/system-prompts.ts
-var MULTIPLE_CHOICE_SYSTEM_PROMPT = `Your explanation can't be longer than 400 tokens. The last sentence must be formatted as one of the following:
-- The answer is <answer letter>
-- The answer is **<answer letter>**
-- <answer letter>: ...
-- <answer letter>) ...
-Replace <answer letter> with the letter of your chosen answer.
-
-Use the following string as your last sentence if you are not capable of answering the question:
-<!NO ANSWER!>`;
-var SENTENCE_REORDER_SYSTEM_PROMPT = "Your task is ordering the given sentences (each line is a sentence) in a correct order. Your output must be formatted as the input but with the sentences in the correct order. Markdown formatting is forbidden.";
-var TEXT_REPLACEMENT_SYSTEM_PROMPT = "Your task is placing all the entities that are provided in the ENTITIES section to the input text in a correct order. Your output only and only includes the modified text, nothing else. It is forbidden to modify anything else from the input text. Markdown formatting is forbidden too.";
-var TYPO_SYSTEM_PROMPT = "Your task is to find all the typos in the given text. Your output must include the corrected text, nothing else.";
-var OPEN_ENDED_SYSTEM_PROMPT = "You are a knowledgeable expert. Please provide a clear, accurate, short and well-reasoned answer to the following question. Be concise but comprehensive in your response. Your answer must be short and clear with less than 20 words";
 export {
-  AbstractCollector,
-  AbstractGenerator,
-  AbstractParser,
-  AbstractProvider,
-  AbstractRSSCollector,
-  AbstractRegistry,
-  AbstractScorer,
-  AnthropicModels,
-  BaseLLMProvider,
-  DataParser,
-  DeepSeekModels,
-  EnumSchema,
-  ExactMatchScorer,
-  ForwardError,
-  GenericArrayParser,
-  GoogleModels,
-  InvalidDataError,
-  LLMJudgeScorer,
-  LargeLanguageModel,
-  LargeLanguageModelOwner,
-  MCQGenerator,
-  MMLUProParser,
-  MULTIPLE_CHOICE_SYSTEM_PROMPT,
-  MedQAParser,
-  MetaModels,
-  MistralModels,
-  MultipleChoiceScorer,
-  NearAIProvider,
-  OPEN_ENDED_SYSTEM_PROMPT,
-  OpenAIModels,
-  OpenEndedGenerator,
-  OpenRouterProvider,
-  PBParser,
-  PEERBENCH_ERROR_CODES,
-  ParagraphMergeStrategy,
-  ParserIsNotCompatibleError,
-  PeerBenchRegistry,
-  PromptResponseSchema,
-  PromptSchema,
-  PromptScoreSchema,
-  PromptTypes,
-  PubMedCollector,
-  QwenModels,
-  RefAnswerEqualityLLMJudgeScorer,
-  SENTENCE_REORDER_SYSTEM_PROMPT,
-  ScoringMethods,
-  SimilarityScorer,
-  SimpleGeneralRSSCollector,
-  StringCollector,
-  TEXT_REPLACEMENT_SYSTEM_PROMPT,
-  TRPGenerator,
-  TYPO_SYSTEM_PROMPT,
-  TaskSchema,
-  XAIModels,
+  CATEGORIES,
+  PEERBENCH_NAMESPACE,
+  PeerbenchError,
+  RateLimiter,
+  ScoringMethod,
   bufferToString,
-  buildPrompt,
-  buildResponse,
-  buildScore,
-  calculateCID,
-  calculateSHA256,
-  checkValidationError,
-  extractJSONFromResponse,
-  formatMs,
-  formatString,
-  paragraphMerge,
-  parseJSONL,
+  captureStackTrace,
+  defineRunner,
+  idGeneratorUUIDv7,
   parseResponseAsJSON,
-  parseValidationError,
-  preparePrompt,
-  readFile,
-  removeDIDPrefix,
   sleep,
-  stableStringify,
-  stringToBuffer,
-  tryParseJson
+  stringToBuffer
 };
 //# sourceMappingURL=index.js.map