flexeval 0.7.2__tar.gz → 0.7.3__tar.gz

{flexeval-0.7.2 → flexeval-0.7.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flexeval
-Version: 0.7.2
+Version: 0.7.3
 Summary: 
 Author: ryokan-ri
 Author-email: ryokan.ri@sbintuitions.co.jp
@@ -10,6 +10,7 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Provides-Extra: vllm
 Provides-Extra: wandb
 Requires-Dist: datasets (>=2.14.6,<3.0.0)

{flexeval-0.7.2 → flexeval-0.7.3}/flexeval/core/language_model/openai_api.py

@@ -6,28 +6,56 @@ from typing import Any, Awaitable, Callable, TypeVar
 import openai
 from loguru import logger
 from openai import AsyncOpenAI
+from openai.types.chat import ChatCompletion, ChatCompletionMessage
+from openai.types.chat.chat_completion import Choice
 
 from .base import LanguageModel, normalize_stop_sequences
 
 T = TypeVar("T")
 
 
+# NOTE: current implementation uses only choices[0].message.content field.
+EMPTY_RESPONSE = ChatCompletion(
+    id="dummy",
+    choices=[
+        Choice(
+            finish_reason="stop",
+            index=0,
+            message=ChatCompletionMessage(
+                content="", refusal=None, role="assistant", function_call=None, tool_calls=None
+            ),
+        )
+    ],
+    created=946652400,  # dummy integer
+    model="dummy_model",
+    object="chat.completion",
+    service_tier=None,
+    system_fingerprint=None,
+    usage=None,
+)
+
+
 async def _retry_on_error(
     openai_call: Callable[[], Awaitable[T]],
     max_num_trials: int = 5,
     first_wait_time: int = 10,
-) -> Awaitable[T] | None:
+) -> Awaitable[T]:
     for i in range(max_num_trials):
         try:
             return await openai_call()
         except openai.APIError as e:  # noqa: PERF203
             if i == max_num_trials - 1:
-                raise
+                # Since reaching maximum number of trials, exit for-loop and return
+                # empty response.
+                break
             logger.warning(f"We got an error: {e}")
             wait_time_seconds = first_wait_time * (2**i)
             logger.warning(f"Wait for {wait_time_seconds} seconds...")
             await asyncio.sleep(wait_time_seconds)
-    return None
+
+    logger.warning(f"We reached maximum number of trials ({max_num_trials} trials.).")
+    logger.warning("Response including empty string is returned.")
+    return EMPTY_RESPONSE
 
 
 class OpenAIChatAPI(LanguageModel):
@@ -108,7 +136,10 @@ class OpenAIChatAPI(LanguageModel):
                 **kwargs,
             ),
         )
-        return [res.choices[0].message.content for res in api_responses]
+        completions = [res.choices[0].message.content for res in api_responses]
+        if all(completion == "" for completion in completions):
+            logger.warning("All generated texts are empty strings. Something may be wrong.")
+        return completions
 
     def batch_generate_chat_response(
         self,
@@ -118,7 +149,10 @@ class OpenAIChatAPI(LanguageModel):
         api_responses = asyncio.run(
             self._async_batch_run_chatgpt(chat_messages_list, **kwargs),
         )
-        return [res.choices[0].message.content for res in api_responses]
+        completions = [res.choices[0].message.content for res in api_responses]
+        if all(completion == "" for completion in completions):
+            logger.warning("All generated texts are empty string. Something may go wrong.")
+        return completions
 
     def __repr__(self) -> str:
         return f"{self.__class__.__name__}(model={self.model})"

{flexeval-0.7.2 → flexeval-0.7.3}/flexeval/core/metric/llm_score.py

@@ -49,7 +49,7 @@ def summarize_evaluator_scores(
         if score is None or category_key is None:
             continue
         if category_key in task_inputs:
-            category2valid_scores[task_inputs["category"]].append(score)
+            category2valid_scores[task_inputs[category_key]].append(score)
 
     category2mean_score: dict[str, float] = {}
     for category, valid_scores in category2valid_scores.items():

flexeval-0.7.3/flexeval/core/prompt_template/jinja2.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from typing import Any
+
+from flexeval.core.utils.jinja2_utils import JINJA2_ENV
+
+from .base import PromptTemplate
+
+
+class Jinja2PromptTemplate(PromptTemplate):
+    """
+    Embed task inputs using Jinja2 template engine.
+
+    Args:
+        template: The Jinja2 template to use.
+        template_path: The path to a file with the Jinja2 template to use.
+    """
+
+    def __init__(self, template: str | None = None, template_path: str | None = None) -> None:
+        if template is None and template_path is None:
+            msg = "Either template or template_path must be provided"
+            raise ValueError(msg)
+        if template is not None and template_path is not None:
+            msg = "Only one of template or template_path can be provided"
+            raise ValueError(msg)
+
+        if template_path is not None:
+            with open(template_path) as f:
+                self.template = f.read()
+        else:
+            self.template = template
+
+    def embed_inputs(self, input_dict: dict[str, Any]) -> str:
+        return JINJA2_ENV.from_string(self.template).render(input_dict)
+
+    def __repr__(self) -> str:
+        return f"Jinja2PromptTemplate(template={self.template!r})"

{flexeval-0.7.2 → flexeval-0.7.3}/flexeval/core/reward_model/pairwise_judge_reward_model.py

@@ -120,6 +120,6 @@ class PairwiseJudgeRewardModel(RewardModel):
 
         for i in range(len(outputs)):
             outputs[i]["llm_outputs"] = [judge_outputs[i * 2], judge_outputs[i * 2 + 1]]
-            outputs[i]["is_corrects"] = [chosen_is_betters[i * 2], chosen_is_betters[i * 2 + 1]]
+            outputs[i]["evaluation_results"] = [chosen_is_betters[i * 2], chosen_is_betters[i * 2 + 1]]
 
         return chosen_is_betters, outputs

{flexeval-0.7.2 → flexeval-0.7.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "flexeval"
-version = "0.7.2" # This will be automatically set from git tag by poetry-dynamic-versioning
+version = "0.7.3" # This will be automatically set from git tag by poetry-dynamic-versioning
 description = ""
 authors = ["ryokan-ri <ryokan.ri@sbintuitions.co.jp>"]
 readme = "README.md"

flexeval-0.7.2/flexeval/core/prompt_template/jinja2.py

@@ -1,25 +0,0 @@
-from __future__ import annotations
-
-from typing import Any
-
-from flexeval.core.utils.jinja2_utils import JINJA2_ENV
-
-from .base import PromptTemplate
-
-
-class Jinja2PromptTemplate(PromptTemplate):
-    """
-    Embed task inputs using Jinja2 template engine.
-
-    Args:
-        template: The Jinja2 template to use.
-    """
-
-    def __init__(self, template: str) -> None:
-        self.template = template
-
-    def embed_inputs(self, input_dict: dict[str, Any]) -> str:
-        return JINJA2_ENV.from_string(self.template).render(input_dict)
-
-    def __repr__(self) -> str:
-        return f"Jinja2PromptTemplate(template={self.template!r})"