flexeval 0.17.3__tar.gz → 0.18.1__tar.gz

{flexeval-0.17.3 → flexeval-0.18.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flexeval
-Version: 0.17.3
+Version: 0.18.1
 Summary: 
 License-File: LICENSE
 Author: ryokan-ri
@@ -13,7 +13,7 @@ Classifier: Programming Language :: Python :: 3.12
 Provides-Extra: vllm
 Provides-Extra: wandb
 Requires-Dist: datasets (>=2.14.6,<3.0.0)
-Requires-Dist: evaluate (>=0.4.1,<0.5.0)
+Requires-Dist: evaluate (>=0.4.6,<0.5.0)
 Requires-Dist: fuzzywuzzy (>=0.18.0,<0.19.0)
 Requires-Dist: google-api-python-client (>=2.131.0,<3.0.0)
 Requires-Dist: jinja2 (>=3.1.2,<4.0.0)
@@ -22,8 +22,9 @@ Requires-Dist: jsonargparse[jsonnet] (>=4.26.1,<5.0.0)
 Requires-Dist: litellm (>=1.52.9,<2.0.0)
 Requires-Dist: loguru (>=0.7.2,<0.8.0)
 Requires-Dist: math-verify[antlr4-13-2] (>=0.7.0,<0.8.0)
-Requires-Dist: openai (>=1.52.2,<2.0.0)
-Requires-Dist: peft (>=0.10.0,<0.11.0)
+Requires-Dist: openai (>=2.0.0,<3.0.0)
+Requires-Dist: peft (>=0.19.1,<0.20.0)
+Requires-Dist: pillow (>=11.0.0,<12.0.0)
 Requires-Dist: pyarrow (==16.1.0)
 Requires-Dist: python-levenshtein (>=0.23.0,<0.24.0)
 Requires-Dist: rouge (>=1.0.1,<2.0.0)
@@ -33,8 +34,9 @@ Requires-Dist: scipy (>=1.13.0,<2.0.0)
 Requires-Dist: smart-open (>=7.1.0,<8.0.0)
 Requires-Dist: sudachipy (>=0.6.10)
 Requires-Dist: tiktoken (>=0.9.0,<0.10.0)
-Requires-Dist: transformers[ja,sentencepiece,torch] (>=4.34.1,<5.0.0)
-Requires-Dist: vllm (==0.17.1) ; extra == "vllm"
+Requires-Dist: transformers[ja,sentencepiece,torch] (>=5.5.3,<6)
+Requires-Dist: uv (>=0.12.14,<0.13.0) ; extra == "vllm"
+Requires-Dist: vllm (==0.24.0) ; extra == "vllm"
 Requires-Dist: wandb (>=0.17.2,<0.18.0) ; extra == "wandb"
 Description-Content-Type: text/markdown
 

{flexeval-0.17.3 → flexeval-0.18.1}/flexeval/core/chat_dataset/__init__.py

@@ -1,5 +1,6 @@
 from .base import ChatDataset, ChatInstance
 from .chatbot_bench import ChatbotBench
+from .image_preprocessors import ConvertImageListToBase64, ConvertImageToBase64, EnsureMinSize
 from .openai_messages import OpenAIMessagesDataset
 from .sacrebleu_dataset import SacreBleuChatDataset
 from .template_based import HFChatDataset, JsonlChatDataset, Preprocessor, TemplateChatDataset, load_jinja2_template

flexeval-0.18.1/flexeval/core/chat_dataset/image_preprocessors.py

@@ -0,0 +1,141 @@
+from __future__ import annotations
+
+import base64
+import dataclasses
+from io import BytesIO
+from typing import Any, Literal
+
+from loguru import logger
+from PIL import Image
+
+from .template_based import Preprocessor
+
+Data = dict[str, Any]
+
+
+def image_to_base64(image: Image.Image, image_format: str, max_length: int | None = None) -> str:
+    """Encode a PIL image as a `data:image/...;base64,...` URL string.
+
+    If `max_length` is given and the encoded string exceeds it, the image is
+    iteratively downscaled until the encoded string fits.
+    """
+
+    def _to_base64(img: Image.Image) -> str:
+        buffer = BytesIO()
+        img.convert("RGB").save(buffer, format=image_format)
+        return base64.b64encode(buffer.getvalue()).decode("utf-8")
+
+    base64_image = _to_base64(image)
+    if max_length is not None and len(base64_image) > max_length:
+        factor = 1.0
+        new_size = image.size
+        while len(base64_image) > max_length:
+            factor *= 0.9
+            new_size = (int(image.width * factor), int(image.height * factor))
+            resized_image = image.resize(new_size, resample=Image.Resampling.BILINEAR)
+            base64_image = _to_base64(resized_image)
+        logger.debug(f"Image size reduced to {new_size} to fit max_length {max_length}.")
+    return f"data:image/{image_format.lower()};base64,{base64_image}"
+
+
+def load_image(image: Image.Image | dict) -> Image.Image:
+    """Normalize an image value into a PIL Image.
+
+    Accepts PIL Image objects or dicts of the form `{"bytes": ..., "path": ...}`
+    (the raw form stored in parquet files that is obtained when images are
+    loaded without the HF `Image` feature metadata, e.g. via the `parquet`
+    dataset loader).
+    """
+    if isinstance(image, Image.Image):
+        return image
+    if isinstance(image, dict):
+        if image.get("bytes") is not None:
+            return Image.open(BytesIO(image["bytes"]))
+        if image.get("path") is not None:
+            return Image.open(image["path"])
+    msg = f"Unsupported image value: {type(image)}"
+    raise TypeError(msg)
+
+
+@dataclasses.dataclass
+class ConvertImageToBase64(Preprocessor):
+    """Convert an image stored under `key` to a base64 data-URL string stored
+    under `{key}_base64`, ready to be embedded as an OpenAI-compatible
+    `image_url` content part.
+    """
+
+    key: str
+    format: Literal["PNG", "JPEG"] = "PNG"
+    max_length: int | None = None
+
+    def __call__(self, item: Data) -> Data:
+        image = item[self.key]
+        if image is None:
+            base64_image = None
+        else:
+            base64_image = image_to_base64(
+                load_image(image),
+                image_format=self.format,
+                max_length=self.max_length,
+            )
+
+        item[f"{self.key}_base64"] = base64_image
+        return item
+
+
+@dataclasses.dataclass
+class ConvertImageListToBase64(Preprocessor):
+    """Convert a list of images (e.g., for multi-image VQA) stored under `key`
+    to a list of base64 data-URL strings stored under `{key}_base64`.
+    """
+
+    key: str
+    format: Literal["PNG", "JPEG"] = "PNG"
+    max_length: int | None = None
+
+    def __call__(self, item: Data) -> Data:
+        images = item[self.key]
+        if images is None:
+            item[f"{self.key}_base64"] = None
+        elif isinstance(images, (list, tuple)):
+            item[f"{self.key}_base64"] = [
+                image_to_base64(load_image(image), image_format=self.format, max_length=self.max_length)
+                for image in images
+            ]
+        else:
+            msg = f"Unsupported image list type: {type(images)}"
+            raise TypeError(msg)
+
+        return item
+
+
+@dataclasses.dataclass
+class EnsureMinSize(Preprocessor):
+    """Ensure the image under `key` is at least `min_size` pixels on its
+    shorter side, upscaling in place (preserving aspect ratio) if necessary.
+    """
+
+    key: str
+    min_size: int
+    interpolation: Image.Resampling = Image.Resampling.BILINEAR
+
+    def __call__(self, item: Data) -> Data:
+        image = item[self.key]
+        if image is None:
+            return item
+
+        if not isinstance(image, Image.Image):
+            msg = f"Unsupported image type: {type(image)}"
+            raise TypeError(msg)
+
+        width, height = image.size
+        if width >= self.min_size and height >= self.min_size:
+            return item
+        if width <= height:
+            new_width = self.min_size
+            new_height = int(self.min_size * height / width)
+        else:
+            new_height = self.min_size
+            new_width = int(self.min_size * width / height)
+        item[self.key] = image.resize((new_width, new_height), resample=self.interpolation)
+        return item

{flexeval-0.17.3 → flexeval-0.18.1}/flexeval/core/chat_dataset/template_based.py

@@ -170,7 +170,9 @@ class HFChatDataset(TemplateChatDataset):
         path: The path to the Hugging Face dataset.
         split: The split of the dataset.
         input_template: A Jinja2 template for the user input.
-        subset: The subset of the dataset.
+        subset: The subset of the dataset. If a list is given, all subsets are
+            loaded and concatenated, and each item gains a "subset" key with
+            its subset name (unless the column already exists).
         dataset_kwargs: The keyword arguments to pass to the Hugging Face dataset.
     """
 
@@ -179,7 +181,7 @@ class HFChatDataset(TemplateChatDataset):
         path: str,
         split: str,
         input_template: str | PathLike[str],
-        subset: str | None = None,
+        subset: str | list[str] | None = None,
         dataset_kwargs: dict[str, Any] | None = None,
         reference_template: str | PathLike[str] | None = None,
         reference_list_template: str | PathLike[str] | None = None,
@@ -193,8 +195,17 @@ class HFChatDataset(TemplateChatDataset):
         preprocessors: list[Preprocessor] | None = None,
     ) -> None:
         dataset_kwargs = dataset_kwargs or {}
-        dataset = datasets.load_dataset(path, name=subset, split=split, **dataset_kwargs)
-        items = [dict(item) for item in dataset]
+        if isinstance(subset, list):
+            items = []
+            for subset_name in subset:
+                dataset = datasets.load_dataset(path, name=subset_name, split=split, **dataset_kwargs)
+                for item in dataset:
+                    item_dict = dict(item)
+                    item_dict.setdefault("subset", subset_name)
+                    items.append(item_dict)
+        else:
+            dataset = datasets.load_dataset(path, name=subset, split=split, **dataset_kwargs)
+            items = [dict(item) for item in dataset]
 
         super().__init__(
             items=items,

{flexeval-0.17.3 → flexeval-0.18.1}/flexeval/core/language_model/vllm_serve_lm.py

@@ -39,15 +39,25 @@ class VLLMServerManager:
         model_kwargs: Additional keyword arguments to pass as command-line options to `vllm serve`.
             Each key-value pair is converted to a corresponding CLI argument.
         timeout: Maximum time in seconds to wait for the server to become available.
+        vllm_version: If specified, `vllm serve` is launched via `uvx --from vllm==<vllm_version>` instead of
+            the `vllm` command on the current environment. This allows running any vllm version regardless of
+            what (if anything) is installed in the current Python environment.
     """
 
-    def __init__(self, model: str, model_kwargs: dict[str, Any] | None = None, timeout: int = 3600) -> None:
+    def __init__(
+        self,
+        model: str,
+        model_kwargs: dict[str, Any] | None = None,
+        timeout: int = 3600,
+        vllm_version: str | None = None,
+    ) -> None:
         self.model = model
         self.model_kwargs = model_kwargs or {}
         self.host = "localhost"
         self.port = find_free_port()
         self.base_url = f"http://{self.host}:{self.port}/v1"
         self.timeout = timeout
+        self.vllm_version = vllm_version
         self.process: subprocess.Popen | None = None
         self._log_threads: list[threading.Thread] = []
 
@@ -59,8 +69,8 @@ class VLLMServerManager:
             logger.warning("vLLM server is already running. Skipping start.")
             return self.host, self.port
 
-        cmd = [
-            "vllm",
+        cmd = ["uvx", "--from", f"vllm=={self.vllm_version}", "vllm"] if self.vllm_version is not None else ["vllm"]
+        cmd += [
             "serve",
             self.model,
             "--host",
@@ -106,6 +116,17 @@ class VLLMServerManager:
 
         interval_second = 1
         for _ in range(self.timeout // interval_second):
+            # Fail fast if the vLLM process disappears or exits while the server is starting.
+            if self.process is None:
+                msg = "vLLM server process is not initialized."
+                raise RuntimeError(msg)
+
+            returncode = self.process.poll()
+            if returncode is not None:
+                self.stop()
+                msg = f"vLLM server process terminated unexpectedly. {returncode=}"
+                raise RuntimeError(msg)
+
             if self.is_ready():
                 logger.info(f"vLLM server is ready at {self.base_url}")
                 return self.host, self.port
@@ -164,6 +185,10 @@ class VLLMServeLM(OpenAIChatAPI):
             Do not include the prefix "--".
             See also: https://docs.vllm.ai/en/latest/cli/index.html#options
         booting_timeout: Maximum time in seconds to wait for the server to become available.
+        vllm_version: If specified, `vllm serve` is launched via `uvx --from vllm==<vllm_version>` instead of
+            the `vllm` command on the current environment, so that any vllm version can be used regardless
+            of the version (if any) installed in the current Python environment. Requires `uv`/`uvx` to be
+            available on PATH.
         default_gen_kwargs: Default generation kwargs to use when calling the API.
         developer_message: Instructions to the model that are prioritized ahead of user messages.
             Previously called the system prompt.
@@ -180,6 +205,7 @@ class VLLMServeLM(OpenAIChatAPI):
         api_headers: dict[str, str] | None = None,
         model_kwargs: dict[str, Any] | None = None,
         booting_timeout: int = 3600,
+        vllm_version: str | None = None,
         default_gen_kwargs: dict[str, Any] | None = None,
         developer_message: str | None = None,
         string_processors: StringProcessor | list[StringProcessor] | None = None,
@@ -195,7 +221,9 @@ class VLLMServeLM(OpenAIChatAPI):
         model_kwargs = model_kwargs or {}
         if "tensor_parallel_size" not in model_kwargs and "tensor-parallel-size" not in model_kwargs:
             model_kwargs["tensor_parallel_size"] = torch.cuda.device_count()
-        self.manager = VLLMServerManager(model=model, model_kwargs=model_kwargs, timeout=booting_timeout)
+        self.manager = VLLMServerManager(
+            model=model, model_kwargs=model_kwargs, timeout=booting_timeout, vllm_version=vllm_version
+        )
         if api_headers is None:
             api_headers = {}
         api_headers["base_url"] = self.manager.base_url

{flexeval-0.17.3 → flexeval-0.18.1}/flexeval/core/metric/llm_label.py

@@ -104,6 +104,8 @@ class LLMLabel(Metric):
         category_key: A key to create category-wise mean score.
             The category key is expected to be in extra_info.
         metric_prefix: A prefix to be added to the metric keys in the summary and instance details.
+        output_reasoning_text: If True, store the evaluator's reasoning content
+            as `llm_label_reasoning_text` in the instance details.
 
     Examples:
         >>> from flexeval import OpenAIChatAPI, Jinja2PromptTemplate, LLMLabel
@@ -146,6 +148,7 @@ class LLMLabel(Metric):
         valid_score_range: tuple[int, int] | None = None,
         category_key: str | None = None,
         metric_prefix: str | None = None,
+        output_reasoning_text: bool = False,
     ) -> None:
         self.language_model = language_model
         self.prompt_template = prompt_template
@@ -166,6 +169,7 @@ class LLMLabel(Metric):
         self.valid_score_range = valid_score_range
         self.category_key = category_key
         self.metric_prefix = f"{metric_prefix}-" if metric_prefix else ""
+        self.output_reasoning_text = output_reasoning_text
 
     def evaluate(
         self,
@@ -218,6 +222,11 @@ class LLMLabel(Metric):
                     f"{self.metric_prefix}llm_label_input": eval_in,
                     f"{self.metric_prefix}llm_label_output": eval_out.text,
                 }
+                | (
+                    {f"{self.metric_prefix}llm_label_reasoning_text": eval_out.reasoning_text}
+                    if self.output_reasoning_text
+                    else {}
+                )
                 for eval_label, eval_score, eval_in, eval_out in zip(
                     evaluator_label_list,
                     evaluator_score_list,
@@ -251,6 +260,8 @@ class ChatLLMLabel(Metric):
         category_key: A key to create category-wise mean score.
             The category key is expected to be in extra_info.
         metric_prefix: A prefix to be added to the metric keys in the summary and instance details.
+        output_reasoning_text: If True, store the evaluator's reasoning content
+            as `llm_label_reasoning_text` in the instance details.
 
     Examples:
         >>> from flexeval import ChatLLMScore, OpenAIChatAPI, Jinja2PromptTemplate
@@ -294,6 +305,7 @@ class ChatLLMLabel(Metric):
         disable_tqdm: bool = False,
         category_key: str | None = None,
         metric_prefix: str | None = None,
+        output_reasoning_text: bool = False,
     ) -> None:
         self.language_model = language_model
         self.prompt_template = prompt_template
@@ -314,6 +326,7 @@ class ChatLLMLabel(Metric):
         self.disable_tqdm = disable_tqdm
         self.category_key = category_key
         self.metric_prefix = f"{metric_prefix}-" if metric_prefix else ""
+        self.output_reasoning_text = output_reasoning_text
 
     def evaluate(
         self,
@@ -367,6 +380,11 @@ class ChatLLMLabel(Metric):
                     f"{self.metric_prefix}llm_label_input": eval_in,
                     f"{self.metric_prefix}llm_label_output": eval_out.text,
                 }
+                | (
+                    {f"{self.metric_prefix}llm_label_reasoning_text": eval_out.reasoning_text}
+                    if self.output_reasoning_text
+                    else {}
+                )
                 for eval_label, eval_score, eval_in, eval_out in zip(
                     evaluator_label_list,
                     evaluator_score_list,

{flexeval-0.17.3 → flexeval-0.18.1}/flexeval/core/metric/llm_score.py

@@ -161,7 +161,7 @@ def generate_evaluations(
         disable=disable_tqdm,
         desc=desc_for_tqdm,
     ) as pbar:
-        evaluator_output_list: list[str] = []
+        evaluator_output_list: list[LMOutput] = []
         for batch_inputs in batch_iter(
             evaluator_input_list,
             batch_size=batch_size,
@@ -196,6 +196,8 @@ class LLMScore(Metric):
             The category keys are expected to be in extra_info.
         metric_prefix: A prefix to be added to the metric keys in the summary and instance details.
         regex_to_parse_score: A regular expression to parse score.
+        output_reasoning_text: If True, store the evaluator's reasoning content
+            as `llm_score_reasoning_text` in the instance details.
 
     Examples:
         >>> from flexeval import LLMScore, OpenAIChatAPI, Jinja2PromptTemplate
@@ -231,6 +233,7 @@ class LLMScore(Metric):
         category_key: str | list[str] | None = None,
         metric_prefix: str | None = None,
         regex_to_parse_score: str = r"(\d+)",
+        output_reasoning_text: bool = False,
     ) -> None:
         self.language_model = language_model
         self.prompt_template = prompt_template
@@ -240,6 +243,7 @@ class LLMScore(Metric):
         self.category_key = category_key
         self.metric_prefix = f"{metric_prefix}-" if metric_prefix else ""
         self.regex_to_parse_score = regex_to_parse_score
+        self.output_reasoning_text = output_reasoning_text
 
     def evaluate(
         self,
@@ -287,6 +291,11 @@ class LLMScore(Metric):
                     f"{self.metric_prefix}llm_score_input": eval_in,
                     f"{self.metric_prefix}llm_score_output": eval_out.text,
                 }
+                | (
+                    {f"{self.metric_prefix}llm_score_reasoning_text": eval_out.reasoning_text}
+                    if self.output_reasoning_text
+                    else {}
+                )
                 for eval_score, eval_in, eval_out in zip(
                     evaluator_score_list,
                     evaluator_input_list,
@@ -320,6 +329,8 @@ class ChatLLMScore(Metric):
             The category keys are expected to be in extra_info.
         metric_prefix: A prefix to be added to the metric keys in the summary and instance details.
         regex_to_parse_score: A regular expression to parse score.
+        output_reasoning_text: If True, store the evaluator's reasoning content
+            as `llm_score_reasoning_text` in the instance details.
 
     Examples:
         >>> from flexeval import ChatLLMScore, OpenAIChatAPI, Jinja2PromptTemplate
@@ -357,6 +368,7 @@ class ChatLLMScore(Metric):
         category_key: str | list[str] | None = None,
         metric_prefix: str | None = None,
         regex_to_parse_score: str = r"(\d+)",
+        output_reasoning_text: bool = False,
     ) -> None:
         self.language_model = language_model
         self.prompt_template = prompt_template
@@ -367,6 +379,7 @@ class ChatLLMScore(Metric):
         self.category_key = category_key
         self.metric_prefix = f"{metric_prefix}-" if metric_prefix else ""
         self.regex_to_parse_score = regex_to_parse_score
+        self.output_reasoning_text = output_reasoning_text
 
     def evaluate(
         self,
@@ -412,6 +425,11 @@ class ChatLLMScore(Metric):
                     f"{self.metric_prefix}llm_score_input": eval_in,
                     f"{self.metric_prefix}llm_score_output": eval_out.text,
                 }
+                | (
+                    {f"{self.metric_prefix}llm_score_reasoning_text": eval_out.reasoning_text}
+                    if self.output_reasoning_text
+                    else {}
+                )
                 for eval_score, eval_in, eval_out in zip(
                     evaluator_score_list,
                     evaluator_input_list,

{flexeval-0.17.3 → flexeval-0.18.1}/flexeval/core/reward_model/pairwise_judge_reward_model.py

@@ -4,7 +4,7 @@ from dataclasses import asdict, dataclass
 from enum import Enum
 from typing import Any
 
-from flexeval.core.language_model.base import LanguageModel
+from flexeval.core.language_model.base import LanguageModel, LMOutput
 from flexeval.core.prompt_template.base import PromptTemplate
 from flexeval.core.reward_bench_dataset import RewardBenchInstance
 from flexeval.core.reward_model.base import RewardModel
@@ -40,38 +40,48 @@ def evaluate_model_output(model_output: str, gold_label: PairwiseChoice) -> bool
 
 def aggregate_judge_results(
     outputs: list[dict],
-    judge_outputs: list,
+    judge_outputs: list[LMOutput],
     chosen_is_better_list: list[bool],
+    output_reasoning_text: bool = False,
 ) -> tuple[list[bool], list[dict]]:
     """
     Aggregates the doubled-length `judge_outputs` and `chosen_is_better_list`
     into final results for each individual instance.
 
+    Args:
+        outputs: Per-instance details to merge into the aggregated outputs.
+        judge_outputs: Judge outputs, twice the length of `outputs`.
+        chosen_is_better_list: Whether the chosen answer won, twice the length of `outputs`.
+        output_reasoning_text: If True, add `llm_reasoning_texts` to the aggregated outputs.
+
     Returns:
         final_results: list[bool]
             A list indicating whether each instance is ultimately judged as correct.
+        aggregated_outputs: list[dict]
+            A list of per-instance details.
     """
     aggregated_results: list[bool] = []
     aggregated_outputs: list[dict] = []
 
     for i, output in enumerate(outputs):
-        ab_output_text = judge_outputs[i * 2].text
-        ba_output_text = judge_outputs[i * 2 + 1].text
+        ab_output = judge_outputs[i * 2]
+        ba_output = judge_outputs[i * 2 + 1]
         ab_eval = chosen_is_better_list[i * 2]
         ba_eval = chosen_is_better_list[i * 2 + 1]
 
         consistent = ab_eval == ba_eval
         is_correct = consistent and ab_eval
 
-        aggregated_outputs.append(
-            {
-                "llm_outputs": [ab_output_text, ba_output_text],
-                "evaluation_results": [ab_eval, ba_eval],
-                "consistent": consistent,
-                "is_correct": is_correct,
-                **output,
-            }
-        )
+        aggregated_output = {
+            "llm_outputs": [ab_output.text, ba_output.text],
+            "evaluation_results": [ab_eval, ba_eval],
+            "consistent": consistent,
+            "is_correct": is_correct,
+            **output,
+        }
+        if output_reasoning_text:
+            aggregated_output["llm_reasoning_texts"] = [ab_output.reasoning_text, ba_output.reasoning_text]
+        aggregated_outputs.append(aggregated_output)
         aggregated_results.append(is_correct)
 
     return aggregated_results, aggregated_outputs
@@ -102,6 +112,8 @@ class PairwiseJudgeRewardModel(RewardModel):
                          Be sure to include {{prompt}}, {{answer_a}}, and {{answer_b}}.
         system_message: The system message to prepend to the chat messages.
         gen_kwargs: Generation kwargs for the language model.
+        output_reasoning_text: If True, store the judge's reasoning content
+            as `llm_reasoning_texts` in the outputs.
     """
 
     def __init__(
@@ -110,6 +122,7 @@ class PairwiseJudgeRewardModel(RewardModel):
         prompt_template: PromptTemplate,
         system_message: str | PromptTemplate | None = None,
         gen_kwargs: dict[str, Any] | None = None,
+        output_reasoning_text: bool = False,
     ) -> None:
         if gen_kwargs is None:
             gen_kwargs = {}
@@ -117,6 +130,7 @@ class PairwiseJudgeRewardModel(RewardModel):
         self.prompt_template = prompt_template
         self.system_message = system_message
         self.gen_kwargs = gen_kwargs
+        self.output_reasoning_text = output_reasoning_text
 
     def _create_input_chat_messages_list(self, pairwise_instance: PairwiseInstance) -> list[dict[str, str]]:
         pairwise_instance_asdict = asdict(pairwise_instance)
@@ -182,6 +196,8 @@ class PairwiseJudgeRewardModel(RewardModel):
             msg = "The number of outputs should be twice the number of inputs."
             raise ValueError(msg)
 
-        aggregated_results, aggregated_outputs = aggregate_judge_results(outputs, judge_outputs, chosen_is_better_list)
+        aggregated_results, aggregated_outputs = aggregate_judge_results(
+            outputs, judge_outputs, chosen_is_better_list, self.output_reasoning_text
+        )
 
         return aggregated_results, aggregated_outputs

{flexeval-0.17.3 → flexeval-0.18.1}/flexeval/core/string_processor/__init__.py

@@ -1,5 +1,6 @@
 from .aio import AIONormalizer
 from .base import StringProcessor
+from .last_choice import LastChoiceExtractor
 from .last_line import LastLineExtractor
 from .lower import StringLower
 from .mgsm import RemoveCommaProcessor, SimpleEvalMGSMProcessor

flexeval-0.18.1/flexeval/core/string_processor/last_choice.py

@@ -0,0 +1,81 @@
+import re
+import string
+from typing import Literal
+
+from .base import StringProcessor
+
+
+class LastChoiceExtractor(StringProcessor):
+    """
+    StringProcessor that extracts the last mentioned multiple-choice option symbol.
+    Useful to extract an answer after a step-by-step derivation.
+
+    MCQ benchmark scores (e.g. MMMU/JMMMU `mcq_exact_match`) depend on this
+    extraction logic; changing it silently shifts reported scores, so keep it
+    stable across releases.
+
+    Args:
+        lang: Language of the response, "en" or "ja".
+            With "ja", option symbols surrounded by Japanese characters
+            (e.g. "正解はAです。") are also detected.
+        option_type: The style of option symbols:
+            "number" (1, 2, ...), "alphabet" (a, b, ...), or "Alphabet" (A, B, ...).
+        max_num_options: The maximum number of options.
+
+    Examples:
+        >>> from flexeval import LastChoiceExtractor
+        >>> processor = LastChoiceExtractor(lang="en")
+        >>> text = "The answer is A."
+        >>> print(processor(text))
+        A
+    """
+
+    def __init__(
+        self,
+        lang: Literal["ja", "en"] = "en",
+        option_type: Literal["number", "alphabet", "Alphabet"] = "Alphabet",
+        max_num_options: int = 4,
+    ) -> None:
+        self.lang = lang
+        if option_type == "number":
+            self.choices = [str(i) for i in range(1, max_num_options + 1)]
+        elif option_type == "alphabet":
+            self.choices = list(string.ascii_lowercase[:max_num_options])
+        elif option_type == "Alphabet":
+            self.choices = list(string.ascii_uppercase[:max_num_options])
+        else:
+            msg = f"Unsupported option_type: {option_type}"
+            raise ValueError(msg)
+
+    def _remove_punctuations(self, response: str) -> str:
+        punctuation = string.punctuation
+        if self.lang == "ja":
+            # add full-width punctuations
+            punctuation += "！”＃＄％＆’（）*+，−．／：；＜＝＞？＠［＼］＾＿｀｛｜｝〜、。￥・"
+        return response.strip().strip(punctuation)
+
+    def _find_choice_with_brackets(self, response: str) -> list[str]:
+        # e.g. (A), (B), ...
+        return [c for c in self.choices if f"({c})" in response]
+
+    def _find_choice_without_brackets(self, response: str) -> list[str]:
+        # e.g. A, B, ...
+        return [c for c in self.choices if re.search(rf"\b{c}\b", response)]
+
+    def _find_choice_between_japanese(self, response: str) -> list[str]:
+        # e.g. 正解はAです。
+        japanese_char_pattern = r"[\u3040-\u30FF\u4E00-\u9FFF]"
+        return [c for c in self.choices if re.search(rf"{japanese_char_pattern}{c}{japanese_char_pattern}", response)]
+
+    def __call__(self, text: str) -> str:
+        candidates = self._find_choice_with_brackets(text)
+        if not candidates:
+            text = self._remove_punctuations(text)
+            candidates = self._find_choice_without_brackets(text)
+        if not candidates and self.lang == "ja":
+            candidates = self._find_choice_between_japanese(text)
+
+        if not candidates:
+            return ""
+        # choose the last mentioned option if multiple options are extracted
+        return max(candidates, key=text.rfind)