flexeval 0.3.2__tar.gz → 0.4.0__tar.gz

{flexeval-0.3.2 → flexeval-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flexeval
-Version: 0.3.2
+Version: 0.4.0
 Summary: 
 Author: ryokan-ri
 Author-email: ryokan.ri@sbintuitions.co.jp
@@ -18,6 +18,7 @@ Requires-Dist: google-api-python-client (>=2.131.0,<3.0.0)
 Requires-Dist: jinja2 (>=3.1.2,<4.0.0)
 Requires-Dist: jiwer (>=3.0.4,<4.0.0)
 Requires-Dist: jsonargparse[jsonnet] (>=4.26.1,<5.0.0)
+Requires-Dist: loguru (>=0.7.2,<0.8.0)
 Requires-Dist: openai (>=1.16.1,<2.0.0)
 Requires-Dist: peft (>=0.10.0,<0.11.0)
 Requires-Dist: python-levenshtein (>=0.23.0,<0.24.0)

{flexeval-0.3.2 → flexeval-0.4.0}/flexeval/core/chat_dataset/__init__.py

@@ -1,3 +1,4 @@
 from .base import ChatDataset, ChatInstance
 from .chatbot_bench import ChatbotBench
 from .hf_dataset import HfChatDataset
+from .sacrebleu_dataset import SacreBleuChatDataset

{flexeval-0.3.2 → flexeval-0.4.0}/flexeval/core/chat_dataset/base.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
-from typing import Any
+from typing import Any, Sequence
 
 
 @dataclass
@@ -43,8 +43,17 @@ class ChatInstance:
             msg = "extra_info cannot contain a key named 'messages'. It will conflict with the 'messages' attribute."
             raise ValueError(msg)
 
+    @property
+    def inputs(self) -> list[dict[str, str]]:
+        """
+        Alias for `messages`.
+        This is used in `FewShotGenerator` so that it can access the inputs with the same attribute name as
+        `GenerationInstance` and `MultipleChoiceInstance`.
+        """
+        return self.messages
+
 
-class ChatDataset(ABC):
+class ChatDataset(Sequence[ChatInstance], ABC):
     """A dataset holding `ChatInstance`."""
 
     @abstractmethod

flexeval-0.4.0/flexeval/core/chat_dataset/sacrebleu_dataset.py

@@ -0,0 +1,32 @@
+import sacrebleu
+
+from .base import ChatDataset, ChatInstance
+
+
+class SacreBleuChatDataset(ChatDataset):
+    """Load datasets from the [sacrebleu](https://github.com/mjpost/sacrebleu) library.
+    The available datasets are defined in sacrebleu.DATASETS.
+    """
+
+    def __init__(self, dataset_name: str, langpair: str) -> None:
+        self._source_list: list[str] = list(sacrebleu.DATASETS[dataset_name].source(langpair))
+        self._references_list: list[list[str]] = [
+            [r.strip() for r in refs] for refs in sacrebleu.DATASETS[dataset_name].references(langpair)
+        ]
+
+        if len(self._source_list) != len(self._references_list):
+            msg = "The number of source and reference pairs should be the same."
+            raise ValueError(msg)
+
+    def require_incremental_response(self) -> bool:
+        return False
+
+    def __len__(self) -> int:
+        return len(self._source_list)
+
+    def __getitem__(self, i: int) -> ChatInstance:
+        return ChatInstance(
+            messages=[{"role": "user", "content": self._source_list[i]}],
+            references=self._references_list[i],
+            extra_info={},
+        )

{flexeval-0.3.2 → flexeval-0.4.0}/flexeval/core/evaluate_chat_response.py

@@ -1,35 +1,53 @@
 from __future__ import annotations
 
-import logging
-from typing import Any
+from typing import Any, Sequence
 
+from loguru import logger
 from tqdm import tqdm
 
 from .chat_dataset import ChatDataset, ChatInstance
+from .few_shot_generator import FewShotGenerator
 from .language_model import LanguageModel
 from .metric import Metric
 from .utils.data_util import batch_iter
 
-logger = logging.getLogger(__name__)
 
-
-def evaluate_chat_response(
+def evaluate_chat_response(  # noqa: C901,PLR0912
     language_model: LanguageModel,
     gen_kwargs: dict[str, Any],
     eval_dataset: ChatDataset,
     metrics: list[Metric],
     batch_size: int,
+    max_instances: int | None = None,
+    few_shot_generator: FewShotGenerator | None = None,
 ) -> tuple[dict[str, float], list[dict[str, Any]]]:
     logger.info(f"Evaluate the model with gen_kwargs: {gen_kwargs}")
 
+    eval_instances: Sequence[ChatInstance] = eval_dataset
+    if max_instances is not None:
+        eval_instances = [eval_dataset[i] for i in range(min(max_instances, len(eval_dataset)))]
+
     all_messages_list: list[list[dict[str, str]]] = []
     references_list: list[list[str]] = []
     extra_info_list: list[dict[str, Any]] = []
-    with tqdm(total=len(eval_dataset)) as pbar:
-        for i, batch in enumerate(batch_iter(eval_dataset, batch_size)):
-            batch: list[ChatInstance]
-
+    with tqdm(total=len(eval_instances)) as pbar:
+        for batch_id, batch in enumerate(batch_iter(eval_instances, batch_size)):
             input_messages_list = [chat_instance.messages for chat_instance in batch]
+
+            if few_shot_generator is not None:
+                for input_id in range(len(input_messages_list)):
+                    few_shot_instances = few_shot_generator(eval_inputs=input_messages_list[input_id])
+                    few_shot_messages: list[dict[str, str]] = []
+                    for few_shot_instance in few_shot_instances:
+                        if not isinstance(few_shot_instance, ChatInstance):
+                            msg = f"Invalid instance type: {type(few_shot_instance)}"
+                            raise TypeError(msg)
+                        few_shot_messages += few_shot_instance.messages
+                        if few_shot_instance.references:
+                            # use the first reference as the assistant message
+                            few_shot_messages += [{"role": "assistant", "content": few_shot_instance.references[0]}]
+                    input_messages_list[input_id] = [*few_shot_messages, *input_messages_list[input_id]]
+
             if not eval_dataset.require_incremental_response():
                 lm_outputs = language_model.batch_generate_chat_response(
                     input_messages_list,
@@ -65,7 +83,7 @@ def evaluate_chat_response(
             references_list += [chat_instance.references for chat_instance in batch]
             extra_info_list += [chat_instance.extra_info for chat_instance in batch]
 
-            if i == 0:
+            if batch_id == 0:
                 logger.info("Example of the conversation")
                 logger.info(f"{all_messages_list[0]}")
 

{flexeval-0.3.2 → flexeval-0.4.0}/flexeval/core/evaluate_from_file.py

@@ -1,16 +1,15 @@
 from __future__ import annotations
 
 import json
-import logging
 from os import PathLike
 from typing import Any
 
+from loguru import logger
+
 from .chat_dataset import ChatDataset
 from .generation_dataset import GenerationDataset
 from .metric import Metric
 
-logger = logging.getLogger(__name__)
-
 
 def evaluate_from_file(
     eval_file: str | PathLike[str],

{flexeval-0.3.2 → flexeval-0.4.0}/flexeval/core/evaluate_generation.py

@@ -1,8 +1,8 @@
 from __future__ import annotations
 
-import logging
-from typing import Any
+from typing import Any, Sequence
 
+from loguru import logger
 from tqdm import tqdm
 
 from .few_shot_generator import FewShotGenerator
@@ -12,25 +12,28 @@ from .metric import Metric
 from .prompt_template import PromptTemplate
 from .utils.data_util import batch_iter
 
-logger = logging.getLogger(__name__)
 
-
-def evaluate_generation(
+def evaluate_generation(  # noqa: C901
     language_model: LanguageModel,
     gen_kwargs: dict[str, Any],
     eval_dataset: GenerationDataset,
     prompt_template: PromptTemplate,
     metrics: list[Metric],
     batch_size: int,
+    max_instances: int | None = None,
     few_shot_generator: FewShotGenerator | None = None,
 ) -> tuple[dict[str, float], list[dict[str, Any]]]:
     logger.info(f"Evaluate the model with gen_kwargs: {gen_kwargs}")
     logger.info(f"Prompt template: {prompt_template}")
-    eval_instance_list: list[GenerationInstance] = []
+
+    eval_instances: Sequence[GenerationInstance] = eval_dataset
+    if max_instances is not None:
+        eval_instances = [eval_dataset[i] for i in range(min(max_instances, len(eval_dataset)))]
+
     lm_prompt_list: list[str] = []
     lm_output_list: list[str] = []
-    with tqdm(total=len(eval_dataset)) as pbar:
-        for i, batch in enumerate(batch_iter(eval_dataset, batch_size)):
+    with tqdm(total=len(eval_instances)) as pbar:
+        for i, batch in enumerate(batch_iter(eval_instances, batch_size)):
             lm_prompts: list[str] = []
             for eval_instance in batch:
                 template_inputs = eval_instance.inputs
@@ -59,17 +62,16 @@ def evaluate_generation(
                 logger.info(f"lm_outputs: {lm_outputs[0]}")
 
             lm_prompt_list += lm_prompts
-            eval_instance_list += batch
             lm_output_list += lm_outputs
 
             pbar.update(len(batch))
     metrics_summary_dict: dict[str, float] = {}
-    instance_metrics_list: list[dict[str, Any]] = [{} for _ in range(len(eval_instance_list))]
+    instance_metrics_list: list[dict[str, Any]] = [{} for _ in range(len(eval_instances))]
     for metric in metrics:
         metric_result = metric.evaluate(
             lm_outputs=lm_output_list,
-            references_list=[i.references for i in eval_instance_list],
-            task_inputs_list=[i.inputs for i in eval_instance_list],
+            references_list=[i.references for i in eval_instances],
+            task_inputs_list=[i.inputs for i in eval_instances],
         )
 
         metrics_summary_dict.update(metric_result.summary)
@@ -93,7 +95,7 @@ def evaluate_generation(
         for lm_prompt, lm_output, eval_instance, instance_metrics in zip(
             lm_prompt_list,
             lm_output_list,
-            eval_instance_list,
+            eval_instances,
             instance_metrics_list,
         )
     ]

{flexeval-0.3.2 → flexeval-0.4.0}/flexeval/core/evaluate_multiple_choice.py

@@ -1,8 +1,8 @@
 from __future__ import annotations
 
-import logging
-from typing import Any
+from typing import Any, Sequence
 
+from loguru import logger
 from tqdm import tqdm
 
 from .few_shot_generator import FewShotGenerator
@@ -11,19 +11,22 @@ from .multiple_choice_dataset import MultipleChoiceDataset, MultipleChoiceInstan
 from .prompt_template import PromptTemplate
 from .utils.data_util import batch_iter
 
-logger = logging.getLogger(__name__)
-
 
 def evaluate_multiple_choice(
     language_model: LanguageModel,
     eval_dataset: MultipleChoiceDataset,
     prompt_template: PromptTemplate,
     batch_size: int,
+    max_instances: int | None = None,
     few_shot_generator: FewShotGenerator | None = None,
 ) -> tuple[dict[str, float], list[dict[str, Any]]]:
+    eval_instances: Sequence[MultipleChoiceInstance] = eval_dataset
+    if max_instances is not None:
+        eval_instances = [eval_dataset[i] for i in range(min(max_instances, len(eval_dataset)))]
+
     results: list[dict[str, Any]] = []
-    with tqdm(total=len(eval_dataset)) as pbar:
-        for batch_id, batch in enumerate(batch_iter(eval_dataset, batch_size)):
+    with tqdm(total=len(eval_instances)) as pbar:
+        for batch_id, batch in enumerate(batch_iter(eval_instances, batch_size)):
             batch: list[MultipleChoiceInstance]
 
             batch_prefixes: list[str] = []

{flexeval-0.3.2 → flexeval-0.4.0}/flexeval/core/evaluate_pairwise.py

@@ -1,9 +1,10 @@
 from __future__ import annotations
 
-import logging
 from dataclasses import asdict
 from typing import Any
 
+from loguru import logger
+
 from .pairwise_comparison import (
     AllCombinations,
     BradleyTerryScorer,
@@ -16,8 +17,6 @@ from .pairwise_comparison import (
 )
 from .utils.data_util import batch_iter
 
-logger = logging.getLogger(__name__)
-
 
 def evaluate_pairwise(
     model_items: dict[str, list[dict[str, Any]]],

{flexeval-0.3.2 → flexeval-0.4.0}/flexeval/core/evaluate_perplexity.py

@@ -1,9 +1,10 @@
 from __future__ import annotations
 
-import logging
 import math
 from collections import defaultdict
+from typing import Sequence
 
+from loguru import logger
 from tqdm import tqdm
 
 from .language_model import LanguageModel
@@ -11,20 +12,23 @@ from .metric.tokenizer import Tokenizer
 from .text_dataset import TextDataset
 from .utils.data_util import batch_iter
 
-logger = logging.getLogger(__name__)
-
 
 def evaluate_perplexity(
     language_model: LanguageModel,
     eval_dataset: TextDataset,
     batch_size: int,
+    max_instances: int | None = None,
     tokenizer: Tokenizer | None = None,
 ) -> dict[str, float]:
     total_log_prob = 0.0
 
+    eval_instances: Sequence[str] = eval_dataset
+    if max_instances is not None:
+        eval_instances = [eval_dataset[i] for i in range(min(max_instances, len(eval_dataset)))]
+
     token_counts: dict[str, int] = defaultdict(int)
-    with tqdm() as pbar:
-        for batch in batch_iter(eval_dataset, batch_size):
+    with tqdm(total=len(eval_instances)) as pbar:
+        for batch in batch_iter(eval_instances, batch_size):
             log_probs = language_model.batch_compute_log_probs(batch)
             total_log_prob += sum(log_probs)
 

{flexeval-0.3.2 → flexeval-0.4.0}/flexeval/core/few_shot_generator/balanced.py

@@ -38,7 +38,10 @@ class BalancedFewShotGenerator(FewShotGenerator):
             label_to_ids[instance.references[0]].append(i)
         self._label_to_ids = label_to_ids
 
-    def _sample_instances(self, eval_inputs: dict[str, Any] | None = None) -> list[GenerationInstance]:
+    def _sample_instances(
+        self,
+        eval_inputs: list[dict[str, Any]] | dict[str, Any] | None = None,
+    ) -> list[GenerationInstance]:
         # Shuffle labels
         labels = list(self._label_to_ids.keys())
         self._rnd.shuffle(labels)

{flexeval-0.3.2 → flexeval-0.4.0}/flexeval/core/few_shot_generator/base.py

@@ -3,11 +3,12 @@ from __future__ import annotations
 from abc import ABC, abstractmethod
 from typing import Any, Union
 
+from flexeval.core.chat_dataset import ChatDataset, ChatInstance
 from flexeval.core.generation_dataset import GenerationDataset, GenerationInstance
 from flexeval.core.multiple_choice_dataset import MultipleChoiceDataset, MultipleChoiceInstance
 
-Dataset = Union[GenerationDataset, MultipleChoiceDataset]
-Instance = Union[GenerationInstance, MultipleChoiceInstance]
+Dataset = Union[GenerationDataset, MultipleChoiceDataset, ChatDataset]
+Instance = Union[GenerationInstance, MultipleChoiceInstance, ChatInstance]
 
 
 class FewShotGenerator(ABC):
@@ -15,10 +16,27 @@ class FewShotGenerator(ABC):
         self._num_trials_to_avoid_leak = num_trials_to_avoid_leak
 
     @abstractmethod
-    def _sample_instances(self, eval_inputs: dict[str, Any] | None = None) -> list[Instance]:
+    def _sample_instances(self, eval_inputs: list[dict[str, Any]] | dict[str, Any] | None = None) -> list[Instance]:
+        """
+        Sample instances for few-shot learning.
+        This method should be implemented in the derived class.
+        """
         raise NotImplementedError
 
-    def __call__(self, eval_inputs: dict[str, Any] | None = None) -> list[Instance]:
+    def __call__(self, eval_inputs: list[dict[str, Any]] | dict[str, Any] | None = None) -> list[Instance]:
+        """
+        Sample instances for few-shot learning.
+        This method calls `_sample_instances` and
+        checks if the sampled instances have the same inputs as the evaluation instance.
+
+        Args:
+            eval_inputs: The inputs of the evaluation instance.
+                This is used to avoid data leakage
+                by checking if the sampled instances have the same inputs as the evaluation instance.
+
+        Returns:
+            A list of instances for few-shot learning.
+        """
         sampled_instances = self._sample_instances(eval_inputs=eval_inputs)
 
         # check if the sampled instances are the same as the eval_instance

{flexeval-0.3.2 → flexeval-0.4.0}/flexeval/core/few_shot_generator/rand.py

@@ -27,6 +27,6 @@ class RandomFewShotGenerator(FewShotGenerator):
         self._num_shots = num_shots
         self._rnd = random.Random(seed)
 
-    def _sample_instances(self, eval_inputs: dict[str, Any] | None = None) -> list[Instance]:
+    def _sample_instances(self, eval_inputs: list[dict[str, Any]] | dict[str, Any] | None = None) -> list[Instance]:
         sampled_indices = self._rnd.sample(range(len(self._dataset)), self._num_shots)
         return [self._dataset[i] for i in sampled_indices]

{flexeval-0.3.2 → flexeval-0.4.0}/flexeval/core/generation_dataset/base.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
+from typing import Sequence
 
 
 @dataclass
@@ -22,7 +23,7 @@ class GenerationInstance:
     """
 
 
-class GenerationDataset(ABC):
+class GenerationDataset(Sequence[GenerationInstance], ABC):
     """A dataset holding `GenerationInstance`."""
 
     @abstractmethod

{flexeval-0.3.2 → flexeval-0.4.0}/flexeval/core/language_model/__init__.py

@@ -1,4 +1,4 @@
 from .base import LanguageModel
 from .hf_lm import HuggingFaceLM
-from .openai_chatgpt import OpenAIChatGPT
+from .openai_chatgpt import OpenAIChatAPI
 from .vllm_model import VllmModel

{flexeval-0.3.2 → flexeval-0.4.0}/flexeval/core/language_model/hf_lm.py

@@ -1,18 +1,16 @@
 from __future__ import annotations
 
 import contextlib
-import logging
 from typing import Any, Literal, TypeVar
 
 import torch
 import torch.nn.functional as F  # noqa: N812
 import transformers
+from loguru import logger
 from transformers import AutoModelForCausalLM, AutoTokenizer, BatchEncoding, PreTrainedModel, PreTrainedTokenizer
 
 from .base import LanguageModel
 
-logger = logging.getLogger(__name__)
-
 T = TypeVar("T")
 
 
@@ -177,13 +175,17 @@ class HuggingFaceLM(LanguageModel):
         model_kwargs = {**model_kwargs}  # copy kwargs to avoid modifying the original dict
         if "device_map" not in model_kwargs:
             model_kwargs["device_map"] = "auto"
-        if "torch_dtype" not in model_kwargs or model_kwargs["torch_dtype"] == "auto":
+        if "torch_dtype" not in model_kwargs:
             # You need to set torch_dtype to use the optimal dtype for the model.
             # https://huggingface.co/docs/transformers/main/main_classes/model#model-instantiation-dtype
             model_kwargs["torch_dtype"] = "auto"
-        else:
+        elif model_kwargs["torch_dtype"] != "auto":
             # Convert string to torch.dtype
-            model_kwargs["torch_dtype"] = getattr(torch, model_kwargs["torch_dtype"])
+            # We allow either "bfloat16" or "torch.bfloat16"
+            torch_dtype_str = model_kwargs["torch_dtype"]
+            if torch_dtype_str.startswith("torch."):
+                torch_dtype_str = torch_dtype_str[len("torch.") :]
+            model_kwargs["torch_dtype"] = getattr(torch, torch_dtype_str)
             if not isinstance(model_kwargs["torch_dtype"], torch.dtype):
                 msg = f"Invalid torch_dtype: {model_kwargs['torch_dtype']}"
                 raise ValueError(msg)

{flexeval-0.3.2 → flexeval-0.4.0}/flexeval/core/language_model/openai_chatgpt.py

@@ -1,16 +1,14 @@
 from __future__ import annotations
 
 import asyncio
-import logging
 from typing import Awaitable, Callable, TypeVar
 
 import openai
+from loguru import logger
 from openai import AsyncOpenAI
 
 from .base import LanguageModel
 
-logger = logging.getLogger(__name__)
-
 T = TypeVar("T")
 
 
@@ -21,20 +19,18 @@ async def _retry_on_error(
 ) -> Awaitable[T] | None:
     for i in range(max_num_trials):
         try:
-            # 関数を実行する
             return await openai_call()
         except openai.APIError as e:  # noqa: PERF203
-            # 試行回数が上限に達したらエラーを送出
             if i == max_num_trials - 1:
                 raise
-            logger.info(f"エラーを受け取りました：{e}")
+            logger.info(f"We got an error：{e}")
             wait_time_seconds = first_wait_time * (2**i)
-            logger.info(f"{wait_time_seconds}秒待機します")
+            logger.info(f"Wait for {wait_time_seconds} seconds...")
             await asyncio.sleep(wait_time_seconds)
     return None
 
 
-class OpenAIChatGPT(LanguageModel):
+class OpenAIChatAPI(LanguageModel):
     """
     LanguageModel implementation using OpenAI's ChatGPT API.
 

{flexeval-0.3.2 → flexeval-0.4.0}/flexeval/core/metric/code_eval.py

@@ -8,6 +8,7 @@ import evaluate
 from flexeval.core.utils.jinja2_env import JINJA2_ENV
 
 from .base import Metric, MetricResult
+from .normalizer import Normalizer
 
 # by default, the program is not allowed to execute code and we need to set this environment variable
 os.environ["HF_ALLOW_CODE_EVAL"] = "1"
@@ -21,15 +22,17 @@ class CodeEval(Metric):
         code_prompt_template: A Jinja2 template string that will prepend the generated code.
             The template should contain variables that will be replaced with the values in `task_inputs_list`.
             If `None`, the code prompt will be the generated code itself.
+        normalizer: A normalizer applied to model outputs before evaluation.
     """
 
-    def __init__(self, code_prompt_template: str | None = None) -> None:
+    def __init__(self, code_prompt_template: str | None = None, normalizer: Normalizer | None = None) -> None:
         self._code_prompt_template = None
         if code_prompt_template is not None:
             self._code_prompt_template = JINJA2_ENV.from_string(
                 code_prompt_template,
             )
         self._code_eval = evaluate.load("code_eval")
+        self._normalizer = normalizer
 
     def evaluate(
         self,
@@ -48,6 +51,9 @@ class CodeEval(Metric):
             task_inputs_list,
             references_list,
         ):
+            if self._normalizer is not None:
+                lm_output = self._normalizer.normalize(lm_output)  # noqa: PLW2901
+
             generated_function = lm_output
             if self._code_prompt_template is not None:
                 generated_function = self._code_prompt_template.render(**task_inputs) + lm_output

{flexeval-0.3.2 → flexeval-0.4.0}/flexeval/core/metric/normalizer/regex.py

@@ -12,7 +12,7 @@ class RegexNormalizer(Normalizer):
     """
 
     def __init__(self, pattern: str) -> None:
-        self._pattern = re.compile(pattern)
+        self._pattern = re.compile(pattern, flags=re.DOTALL)
 
     def normalize(self, text: str) -> str:
         found = self._pattern.findall(text)

{flexeval-0.3.2 → flexeval-0.4.0}/flexeval/core/metric/perspective_api.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import logging
 import os
 import time
 from typing import Any, Callable
@@ -8,11 +7,10 @@ from typing import Any, Callable
 import numpy as np
 from googleapiclient import discovery
 from googleapiclient.errors import HttpError
+from loguru import logger
 
 from .base import Metric, MetricResult
 
-logger = logging.getLogger(__name__)
-
 PERSPECTIVE_API_KEY = os.getenv("PERSPECTIVE_API_KEY")
 
 

{flexeval-0.3.2 → flexeval-0.4.0}/flexeval/core/multiple_choice_dataset/base.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
+from typing import Sequence
 
 
 @dataclass
@@ -26,7 +27,7 @@ class MultipleChoiceInstance:
     """
 
 
-class MultipleChoiceDataset(ABC):
+class MultipleChoiceDataset(Sequence[MultipleChoiceInstance], ABC):
     @abstractmethod
     def __len__(self) -> int:
         """

{flexeval-0.3.2 → flexeval-0.4.0}/flexeval/core/pairwise_comparison/judge/base.py

@@ -25,23 +25,6 @@ class PairwiseJudge(ABC):
     The output is a tuple of the winner and the rationale.
     """
 
-    @abstractmethod
-    def judge(
-        self,
-        model1_item: dict[str, Any],
-        model2_item: dict[str, Any],
-    ) -> tuple[Winner, str]:
-        """
-        Judge which model is better given two items.
-
-        Args:
-            model1_item: The first model item, containing the model output and other information needed for judging.
-            model2_item: The second model item, containing the model output and other information needed for judging.
-
-        Returns:
-            A tuple of the winner and the rationale.
-        """
-
     @abstractmethod
     def batch_judge(
         self,

{flexeval-0.3.2 → flexeval-0.4.0}/flexeval/core/pairwise_comparison/judge/llm_judge.py

@@ -1,16 +1,15 @@
 from __future__ import annotations
 
-import logging
 import re
 from typing import Any
 
+from loguru import logger
+
 from flexeval.core.language_model.base import LanguageModel
 from flexeval.core.prompt_template.base import PromptTemplate
 
 from .base import PairwiseJudge, Winner
 
-logger = logging.getLogger(__name__)
-
 
 class ChatLLMPairwiseJudge(PairwiseJudge):
     """
@@ -60,9 +59,6 @@ class ChatLLMPairwiseJudge(PairwiseJudge):
         else:
             return winner, rationale
 
-    def judge(self, model1_item: dict[str, Any], model2_item: dict[str, Any]) -> tuple[Winner, str]:
-        return self.batch_judge([(model1_item, model2_item)])[0]
-
     def batch_judge(self, batch_model_items: list[tuple[dict[str, Any], dict[str, Any]]]) -> list[tuple[Winner, str]]:
         input_chat_messages_list: list[list[dict[str, str]]] = []
         for model1_item, model2_item in batch_model_items: