flexeval 0.17.1__tar.gz → 0.17.2__tar.gz

{flexeval-0.17.1 → flexeval-0.17.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flexeval
-Version: 0.17.1
+Version: 0.17.2
 Summary: 
 License-File: LICENSE
 Author: ryokan-ri

{flexeval-0.17.1 → flexeval-0.17.2}/flexeval/__init__.py

@@ -13,6 +13,7 @@ from .core.metric import *
 from .core.multiple_choice_dataset import *
 from .core.pairwise_comparison import *
 from .core.prompt_template import *
+from .core.reasoning_parser import *
 from .core.result_recorder import *
 from .core.reward_bench_dataset import *
 from .core.reward_model import *

{flexeval-0.17.1 → flexeval-0.17.2}/flexeval/core/evaluate_chat_response.py

@@ -122,6 +122,7 @@ def execute_conversation_flow(
                         "finish_reason": lm_output.finish_reason,
                     }
                     | ({"raw_content": lm_output.raw_text} if lm_output.raw_text else {})
+                    | ({"reasoning_content": lm_output.reasoning_text} if lm_output.reasoning_text else {})
                     | ({"tool_calls": lm_output.tool_calls} if lm_output.tool_calls else {})
                     | (
                         {"tool_call_validation_result": lm_output.tool_call_validation_result}
@@ -215,22 +216,25 @@ def evaluate_chat_response(  # noqa: C901, PLR0912
     # legacy: `{"lm_output": "...", "finish_reason": "...", "task_inputs": {...}, "references": [...], **metrics}`
     restructured_outputs: list[dict[str, Any]] = []
     for output in outputs:
+        lm_output: LMOutput = output["lm_output"]
         extra_info = output["chat_instance"].extra_info | {"messages": output["messages"]}
-        if output["lm_output"].tool_calls:
-            extra_info["tool_calls"] = output["lm_output"].tool_calls
         if output["chat_instance"].tools:
             extra_info["tools"] = output["chat_instance"].tools
         restructured_output = {
-            "lm_output": output["lm_output"].text,
+            "lm_output": lm_output.text,
             "finish_reason": output["lm_output"].finish_reason,
             "extra_info": extra_info,
             "references": output["chat_instance"].references,
             **output["metrics"],
         }
-        if output["lm_output"].raw_text:
-            restructured_output["raw_lm_output"] = output["lm_output"].raw_text
-        if output["lm_output"].reasoning_text:
-            restructured_output["reasoning_text"] = output["lm_output"].reasoning_text
+        if lm_output.raw_text:
+            restructured_output["raw_lm_output"] = lm_output.raw_text
+        if lm_output.reasoning_text:
+            restructured_output["reasoning_text"] = lm_output.reasoning_text
+        if lm_output.tool_calls:
+            restructured_output["tool_calls"] = lm_output.tool_calls
+        if lm_output.tool_call_validation_result:
+            restructured_output["tool_call_validation_result"] = lm_output.tool_call_validation_result
         restructured_outputs.append(restructured_output)
 
     return metrics_summary_dict, restructured_outputs

{flexeval-0.17.1 → flexeval-0.17.2}/flexeval/core/evaluate_from_data.py

@@ -5,18 +5,20 @@ from typing import Any
 
 from loguru import logger
 
+from flexeval.core.language_model.base import LMOutput
+
 from .chat_dataset import ChatDataset
 from .generation_dataset import GenerationDataset
 from .metric import Metric
 
 
-def evaluate_from_data(
+def evaluate_from_data(  # noqa: C901
     eval_data: Iterable[dict[str, Any]],
     metrics: list[Metric],
     eval_dataset: GenerationDataset | ChatDataset | None = None,
 ) -> tuple[dict[str, float], list[dict[str, Any]]]:
     extra_info_list: list[dict[str, Any]] = []
-    lm_output_list: list[str] = []
+    lm_output_list: list[str | LMOutput] = []
     references_list: list[list[str]] = []
     for item in eval_data:
         # ignore extra info if empty for backward compatibility
@@ -24,7 +26,23 @@ def evaluate_from_data(
         if "extra_info" not in item:
             item["extra_info"] = {}
         extra_info_list.append(item["extra_info"])
-        lm_output_list.append(item["lm_output"])
+        # We adapt the current output format of evaluate_chat_response(). This will be changed in the future.
+        if isinstance(item["lm_output"], str):
+            lm_output = LMOutput(
+                text=item["lm_output"],
+                raw_text=item.get("raw_lm_output"),
+                reasoning_text=item.get("reasoning_text"),
+                finish_reason=item.get("finish_reason"),
+                tool_calls=item.get("tool_calls"),
+                tool_call_validation_result=item.get("tool_call_validation_result"),
+            )
+        elif isinstance(item["lm_output"], LMOutput):
+            lm_output = item["lm_output"]
+        else:
+            msg = f"Invalid type for lm_output: {type(item['lm_output'])}"
+            raise TypeError(msg)
+
+        lm_output_list.append(lm_output)
         references_list.append(item["references"])
 
     if eval_dataset is not None:

{flexeval-0.17.1 → flexeval-0.17.2}/flexeval/core/language_model/base.py

@@ -46,6 +46,17 @@ class LMOutput:
                 msg = "Both `text` and `tool_calls` are empty."
                 logger.warning(msg)
 
+    def __str__(self) -> str:
+        """Return the `text` attribute as a string for backward compatibility.
+
+        Before `LMOutput` was introduced, outputs were saved as plain strings.
+        Therefore, several existing configs using LLM-judge metrics such as `ChatLLMScore` assume
+        that `lm_output` is a plain string. By implementing `__str__`, an `LMOutput`
+        instance embedded in a Jinja2 template renders as its `text` attribute,
+        preserving that contract.
+        """
+        return self.text if self.text is not None else ""
+
 
 class LanguageModel:
     """LanguageModel is what you want to evaluate with this library.
@@ -226,7 +237,10 @@ class LanguageModel:
         # Post-process the generated text
         if self.string_processors:
             for lm_output in lm_outputs:
-                lm_output.raw_text = lm_output.text
+                # If a subclass preprocesses the output (e.g., via ToolParser or ReasoningParser),
+                # lm_output.raw_text should already hold the unprocessed text.
+                if lm_output.raw_text is None:
+                    lm_output.raw_text = lm_output.text
                 for string_processor in self.string_processors:
                     lm_output.text = string_processor(lm_output.text)
 

{flexeval-0.17.1 → flexeval-0.17.2}/flexeval/core/language_model/hf_lm.py

@@ -13,6 +13,7 @@ import transformers
 from loguru import logger
 from transformers import AutoModelForCausalLM, AutoTokenizer, BatchEncoding, PreTrainedModel, PreTrainedTokenizer
 
+from flexeval.core.reasoning_parser.base import ReasoningParser
 from flexeval.core.string_processor import StringProcessor
 from flexeval.core.tool_parser.base import ToolParser
 from flexeval.utils.hf_utils import get_default_model_kwargs
@@ -186,8 +187,13 @@ class HuggingFaceLM(LanguageModel):
             If this value is set to less than or equal to the model's capacity and the input exceeds it,
             an empty string is returned instead of raising an error.
             If set to “default”, the value will be automatically determined when possible.
+        reasoning_parser: A ReasoningParser object to extract the reasoning_text from the model's output.
         tool_parser: A ToolParser object to extract the tool_calls from the model's output.
         tools: Default tools to use in chat responses when no tools are explicitly provided.
+        prefix_str_for_chat: A string to prepend to the assistant's response in chat generation.
+            This string is appended to the prompt (after `apply_chat_template`) so that the model
+            continues generation from it. The same string is also prepended to the final output
+            so that the returned text includes the forced prefix.
     """
 
     def __init__(
@@ -206,8 +212,10 @@ class HuggingFaceLM(LanguageModel):
         default_gen_kwargs: dict[str, Any] | None = None,
         string_processors: StringProcessor | list[StringProcessor] | None = None,
         model_limit_tokens: int | None | Literal["default"] = "default",
+        reasoning_parser: ReasoningParser | None = None,
         tool_parser: ToolParser | None = None,
         tools: list[dict[str, Any]] | None = None,
+        prefix_str_for_chat: str = "",
     ) -> None:
         super().__init__(string_processors=string_processors, tools=tools)
         self._model_name_or_path = model
@@ -225,7 +233,9 @@ class HuggingFaceLM(LanguageModel):
         self.load_peft = load_peft
         self.amp_dtype = amp_dtype
         self.model_limit_tokens = model_limit_tokens
+        self.reasoning_parser = reasoning_parser
         self.tool_parser = tool_parser
+        self.prefix_str_for_chat = prefix_str_for_chat
         logger.info(f"amp_dtype: {amp_dtype}")
         logger.info(f"random seed: {random_seed}")
         transformers.set_seed(random_seed)
@@ -411,18 +421,30 @@ class HuggingFaceLM(LanguageModel):
                 chat_template=self.custom_chat_template,
                 **self.chat_template_kwargs,
             )
+            + self.prefix_str_for_chat
             for chat_messages, tools in zip(chat_messages_list, tools_list)
         ]
         lm_outputs = self._batch_complete_text(chat_messages_as_string, **kwargs)
-        if self.tool_parser:
-            for lm_output, tools in zip(lm_outputs, tools_list):
-                if tools is None:
-                    continue
-                lm_output: LMOutput
+
+        for lm_output, tools in zip(lm_outputs, tools_list):
+            if lm_output.text is None:
+                continue
+
+            lm_output.text = self.prefix_str_for_chat + lm_output.text
+
+            if self.reasoning_parser:
+                if lm_output.raw_text is None:
+                    lm_output.raw_text = lm_output.text
+                reasoning = self.reasoning_parser(lm_output.text)
+                lm_output.text = reasoning.text
+                lm_output.reasoning_text = reasoning.reasoning_text
+
+            if self.tool_parser and tools is not None:
                 parsed_tool_calling_message = self.tool_parser(lm_output.text)
                 lm_output.tool_calls = parsed_tool_calling_message.tool_call_dicts
-                lm_output.raw_text = parsed_tool_calling_message.raw_text
                 lm_output.text = parsed_tool_calling_message.text
+                if lm_output.raw_text is None:
+                    lm_output.raw_text = parsed_tool_calling_message.raw_text
                 lm_output.tool_call_validation_result = parsed_tool_calling_message.validation_result
 
         return lm_outputs

{flexeval-0.17.1 → flexeval-0.17.2}/flexeval/core/language_model/vllm_model.py

@@ -8,6 +8,7 @@ import torch
 from loguru import logger
 from transformers import AutoTokenizer, PreTrainedTokenizer
 
+from flexeval.core.reasoning_parser.base import ReasoningParser
 from flexeval.core.string_processor import StringProcessor
 from flexeval.core.tool_parser.base import ToolParser
 
@@ -94,8 +95,13 @@ class VLLM(LanguageModel):
             If this value is set to less than or equal to the model's capacity and the input exceeds it,
             an empty string is returned instead of raising an error.
             If set to “default”, the value will be automatically determined when possible.
+        reasoning_parser: A ReasoningParser object to extract the reasoning and content from the model's output.
         tool_parser: A ToolParser object to extract the tool_calls from the model's output.
         tools: Default tools to use in chat responses when no tools are explicitly provided.
+        prefix_str_for_chat: A string to prepend to the assistant's response in chat generation.
+            This string is appended to the prompt (after `apply_chat_template`) so that the model
+            continues generation from it. The same string is also prepended to the final output
+            so that the returned text includes the forced prefix.
     """
 
     def __init__(
@@ -111,8 +117,10 @@ class VLLM(LanguageModel):
         default_gen_kwargs: dict[str, Any] | None = None,
         string_processors: StringProcessor | list[StringProcessor] | None = None,
         model_limit_tokens: int | None | Literal["default"] = "default",
+        reasoning_parser: ReasoningParser | None = None,
         tool_parser: ToolParser | None = None,
         tools: list[dict[str, Any]] | None = None,
+        prefix_str_for_chat: str = "",
     ) -> None:
         super().__init__(string_processors=string_processors, tools=tools)
         self.model_name = model
@@ -141,6 +149,8 @@ class VLLM(LanguageModel):
         self.llm: LLM | None = None
         self.model_limit_tokens = model_limit_tokens
         self.tool_parser = tool_parser
+        self.reasoning_parser = reasoning_parser
+        self.prefix_str_for_chat = prefix_str_for_chat
 
     @staticmethod
     def load_model(method: Callable) -> Callable:
@@ -259,17 +269,30 @@ class VLLM(LanguageModel):
                 chat_template=self.custom_chat_template,
                 **self.chat_template_kwargs,
             )
+            + self.prefix_str_for_chat
             for chat_messages, tools in zip(chat_messages_list, tools_list)
         ]
         lm_outputs = self._batch_complete_text(chat_messages_as_string, **kwargs)
-        if self.tool_parser:
-            for lm_output, tools in zip(lm_outputs, tools_list):
-                if tools is None:
-                    continue
+
+        for lm_output, tools in zip(lm_outputs, tools_list):
+            if lm_output.text is None:
+                continue
+
+            lm_output.text = self.prefix_str_for_chat + lm_output.text
+
+            if self.reasoning_parser:
+                if lm_output.raw_text is None:
+                    lm_output.raw_text = lm_output.text
+                reasoning = self.reasoning_parser(lm_output.text)
+                lm_output.text = reasoning.text
+                lm_output.reasoning_text = reasoning.reasoning_text
+
+            if self.tool_parser and tools is not None:
                 parsed_tool_calling_message = self.tool_parser(lm_output.text)
                 lm_output.tool_calls = parsed_tool_calling_message.tool_call_dicts
-                lm_output.raw_text = parsed_tool_calling_message.raw_text
                 lm_output.text = parsed_tool_calling_message.text
+                if lm_output.raw_text is None:
+                    lm_output.raw_text = parsed_tool_calling_message.raw_text
                 lm_output.tool_call_validation_result = parsed_tool_calling_message.validation_result
 
         return lm_outputs

{flexeval-0.17.1 → flexeval-0.17.2}/flexeval/core/metric/llm_geval_score.py

@@ -15,7 +15,7 @@ from flexeval.core.utils.data_util import batch_iter
 
 from .base import Metric, MetricResult
 from .llm_score import prepare_chat_input_for_evaluator, prepare_text_input_for_evaluator
-from .utils import extract_text_from_outputs, validate_inputs
+from .utils import validate_inputs
 
 
 def calculate_weighted_average(
@@ -238,8 +238,6 @@ class LLMGEvalScore(Metric):
 
         validate_inputs(lm_outputs, references_list, extra_info_list)
 
-        lm_outputs = extract_text_from_outputs(lm_outputs)
-
         # Compute metrics
         evaluator_input_list: list[str] = prepare_text_input_for_evaluator(
             lm_outputs, references_list, extra_info_list, self.prompt_template
@@ -411,8 +409,6 @@ class ChatLLMGEvalScore(Metric):
         if references_list is None:
             references_list = [[] for _ in lm_outputs]
 
-        lm_outputs = extract_text_from_outputs(lm_outputs)
-
         # Compute metrics
         evaluator_input_list = prepare_chat_input_for_evaluator(
             lm_outputs, references_list, extra_info_list, self.prompt_template, self.system_message

{flexeval-0.17.1 → flexeval-0.17.2}/flexeval/core/metric/llm_label.py

@@ -14,7 +14,7 @@ from flexeval.core.metric.llm_score import (
 from flexeval.core.prompt_template import PromptTemplate
 
 from .base import Metric, MetricResult
-from .utils import extract_text_from_outputs, validate_inputs
+from .utils import validate_inputs
 
 
 def parse_label_from_evaluator_output(evaluator_output: str, label_names: list[str]) -> str | None:
@@ -180,9 +180,6 @@ class LLMLabel(Metric):
 
         validate_inputs(lm_outputs, references_list, extra_info_list)
 
-        # Extract text from LMOutput objects
-        lm_outputs = extract_text_from_outputs(lm_outputs)
-
         # Compute metrics
         evaluator_input_list: list[str] = prepare_text_input_for_evaluator(
             lm_outputs, references_list, extra_info_list, self.prompt_template
@@ -331,9 +328,6 @@ class ChatLLMLabel(Metric):
 
         validate_inputs(lm_outputs, references_list, extra_info_list)
 
-        # Extract text from LMOutput objects
-        lm_outputs = extract_text_from_outputs(lm_outputs)
-
         # Compute metrics
         evaluator_input_list = prepare_chat_input_for_evaluator(
             lm_outputs, references_list, extra_info_list, self.prompt_template, self.system_message

{flexeval-0.17.1 → flexeval-0.17.2}/flexeval/core/metric/llm_score.py

@@ -11,7 +11,7 @@ from flexeval.core.prompt_template import PromptTemplate
 from flexeval.core.utils.data_util import batch_iter
 
 from .base import Metric, MetricResult
-from .utils import extract_text_from_outputs, validate_inputs
+from .utils import validate_inputs
 
 
 def parse_score_from_evaluator_output(
@@ -71,7 +71,7 @@ def summarize_evaluator_scores(
 
 
 def prepare_text_input_for_evaluator(
-    lm_outputs: list[str],
+    lm_outputs: list[LMOutput | str],
     references_list: list[list[str]],
     extra_info_list: list[dict[str, str]],
     prompt_template: PromptTemplate,
@@ -80,6 +80,10 @@ def prepare_text_input_for_evaluator(
     by integrating the extra_info, the model outputs, and the prompt template for evaluator.
     """
 
+    lm_outputs: list[LMOutput] = [
+        LMOutput(text=lm_output) if isinstance(lm_output, str) else lm_output for lm_output in lm_outputs
+    ]
+
     evaluator_input_list: list[str] = []
     for lm_output, extra_info, references in zip(
         lm_outputs,
@@ -97,7 +101,7 @@ def prepare_text_input_for_evaluator(
 
 
 def prepare_chat_input_for_evaluator(
-    lm_outputs: list[str],
+    lm_outputs: list[LMOutput | str],
     references_list: list[list[str]],
     extra_info_list: list[dict[str, str]],
     prompt_template: PromptTemplate,
@@ -107,6 +111,10 @@ def prepare_chat_input_for_evaluator(
     by integrating the extra_info, the model outputs, and the prompt template for evaluator.
     """
 
+    lm_outputs: list[LMOutput] = [
+        LMOutput(text=lm_output) if isinstance(lm_output, str) else lm_output for lm_output in lm_outputs
+    ]
+
     evaluator_input_list: list[list[dict[str, str]]] = []
     for lm_output, extra_info, references in zip(
         lm_outputs,
@@ -246,9 +254,6 @@ class LLMScore(Metric):
 
         validate_inputs(lm_outputs, references_list, extra_info_list)
 
-        # Extract text from LMOutput objects
-        lm_outputs = extract_text_from_outputs(lm_outputs)
-
         # Compute metrics
         evaluator_input_list: list[str] = prepare_text_input_for_evaluator(
             lm_outputs, references_list, extra_info_list, self.prompt_template
@@ -374,9 +379,6 @@ class ChatLLMScore(Metric):
         if references_list is None:
             references_list = [[] for _ in lm_outputs]
 
-        # Extract text from LMOutput objects
-        lm_outputs = extract_text_from_outputs(lm_outputs)
-
         # Compute metrics
         evaluator_input_list = prepare_chat_input_for_evaluator(
             lm_outputs, references_list, extra_info_list, self.prompt_template, self.system_message

flexeval-0.17.2/flexeval/core/reasoning_parser/__init__.py

@@ -0,0 +1,2 @@
+from .base import Reasoning, ReasoningParser
+from .regex_reasoning_parser import SeparatedRegexReasoningParser, UnifiedRegexReasoningParser

flexeval-0.17.2/flexeval/core/reasoning_parser/base.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+
+
+@dataclass
+class Reasoning:
+    text: str | None
+    reasoning_text: str | None
+
+
+class ReasoningParser(ABC):
+    """Base class for parsing raw LLM output text into reasoning and content parts.
+
+    Subclasses must implement `__call__`, which receives the raw text and returns
+    an instance of `Reasoning` class.
+    """
+
+    @abstractmethod
+    def __call__(self, raw_text: str) -> Reasoning:
+        """Parse raw LLM output and return the reasoning and content parts.
+
+        Args:
+            raw_text: Raw output string produced by the language model.
+
+        Returns:
+            An instance of `Reasoning` class.
+        """
+        raise NotImplementedError

flexeval-0.17.2/flexeval/core/reasoning_parser/regex_reasoning_parser.py

@@ -0,0 +1,56 @@
+import re
+
+from flexeval.core.reasoning_parser.base import Reasoning, ReasoningParser
+
+
+class UnifiedRegexReasoningParser(ReasoningParser):
+    """Extracts reasoning and content using a single regex with named groups.
+
+    The pattern must contain named groups ``(?P<reasoning_content>...)`` and/or
+    ``(?P<content>...)`` to populate the corresponding fields of :class:`Reasoning`.
+    """
+
+    def __init__(self, pattern: str) -> None:
+        super().__init__()
+        self.pattern = pattern
+
+    def __call__(self, raw_text: str) -> Reasoning:
+        m = re.search(self.pattern, raw_text, re.DOTALL)
+        if m is None:
+            return Reasoning(text=None, reasoning_text=None)
+        gd = m.groupdict()
+        return Reasoning(
+            text=gd.get("content"),
+            reasoning_text=gd.get("reasoning_content"),
+        )
+
+
+class SeparatedRegexReasoningParser(ReasoningParser):
+    """Extracts reasoning and content using two independent regex patterns.
+
+    Each pattern is applied separately.  If a pattern has a capture group,
+    the last group is used; otherwise the full match is used.
+    """
+
+    def __init__(
+        self,
+        pattern_for_content: str,
+        pattern_for_reasoning_content: str,
+    ) -> None:
+        super().__init__()
+        self.pattern_for_content = re.compile(pattern_for_content, re.DOTALL)
+        self.pattern_for_reasoning_content = re.compile(pattern_for_reasoning_content, re.DOTALL)
+
+    def __call__(self, raw_text: str) -> Reasoning:
+        content = None
+        reasoning_text = None
+
+        found_content = self.pattern_for_content.findall(raw_text)
+        if found_content:
+            content = found_content[-1]
+
+        found_reasoning = self.pattern_for_reasoning_content.findall(raw_text)
+        if found_reasoning:
+            reasoning_text = found_reasoning[-1]
+
+        return Reasoning(text=content, reasoning_text=reasoning_text)

{flexeval-0.17.1 → flexeval-0.17.2}/flexeval/scripts/flexeval_file.py

@@ -26,7 +26,7 @@ class EvalDataLoader(ABC):
     A class to load evaluation data.
     The evaluation data should be a list of dictionaries with the following keys:
     - extra_info (dict[str, Any]): A dictionary containing the input data for the task and any other informations.
-    - lm_output (str): The output of the language model.
+    - lm_output (str|LMOutput): The output of the language model.
     - references (list[str]): A list of reference outputs.
     - extra_info (dict[str, Any]): alias for "extra_info". Older versions used this key.
     """

{flexeval-0.17.1 → flexeval-0.17.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "flexeval"
-version = "0.17.1" # This will be automatically set from git tag by poetry-dynamic-versioning
+version = "0.17.2" # This will be automatically set from git tag by poetry-dynamic-versioning
 description = ""
 authors = ["ryokan-ri <ryokan.ri@sbintuitions.co.jp>"]
 readme = "README.md"