flexeval 0.7.2__tar.gz → 0.7.4__tar.gz

{flexeval-0.7.2 → flexeval-0.7.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flexeval
-Version: 0.7.2
+Version: 0.7.4
 Summary: 
 Author: ryokan-ri
 Author-email: ryokan.ri@sbintuitions.co.jp
@@ -10,6 +10,7 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Provides-Extra: vllm
 Provides-Extra: wandb
 Requires-Dist: datasets (>=2.14.6,<3.0.0)

{flexeval-0.7.2 → flexeval-0.7.4}/flexeval/core/language_model/openai_api.py

@@ -6,28 +6,56 @@ from typing import Any, Awaitable, Callable, TypeVar
 import openai
 from loguru import logger
 from openai import AsyncOpenAI
+from openai.types.chat import ChatCompletion, ChatCompletionMessage
+from openai.types.chat.chat_completion import Choice
 
 from .base import LanguageModel, normalize_stop_sequences
 
 T = TypeVar("T")
 
 
+# NOTE: current implementation uses only choices[0].message.content field.
+EMPTY_RESPONSE = ChatCompletion(
+    id="dummy",
+    choices=[
+        Choice(
+            finish_reason="stop",
+            index=0,
+            message=ChatCompletionMessage(
+                content="", refusal=None, role="assistant", function_call=None, tool_calls=None
+            ),
+        )
+    ],
+    created=946652400,  # dummy integer
+    model="dummy_model",
+    object="chat.completion",
+    service_tier=None,
+    system_fingerprint=None,
+    usage=None,
+)
+
+
 async def _retry_on_error(
     openai_call: Callable[[], Awaitable[T]],
     max_num_trials: int = 5,
     first_wait_time: int = 10,
-) -> Awaitable[T] | None:
+) -> Awaitable[T]:
     for i in range(max_num_trials):
         try:
             return await openai_call()
         except openai.APIError as e:  # noqa: PERF203
             if i == max_num_trials - 1:
-                raise
+                # Since reaching maximum number of trials, exit for-loop and return
+                # empty response.
+                break
             logger.warning(f"We got an error: {e}")
             wait_time_seconds = first_wait_time * (2**i)
             logger.warning(f"Wait for {wait_time_seconds} seconds...")
             await asyncio.sleep(wait_time_seconds)
-    return None
+
+    logger.warning(f"We reached maximum number of trials ({max_num_trials} trials.).")
+    logger.warning("Response including empty string is returned.")
+    return EMPTY_RESPONSE
 
 
 class OpenAIChatAPI(LanguageModel):
@@ -108,7 +136,10 @@ class OpenAIChatAPI(LanguageModel):
                 **kwargs,
             ),
         )
-        return [res.choices[0].message.content for res in api_responses]
+        completions = [res.choices[0].message.content for res in api_responses]
+        if all(completion == "" for completion in completions):
+            logger.warning("All generated texts are empty strings. Something may be wrong.")
+        return completions
 
     def batch_generate_chat_response(
         self,
@@ -118,7 +149,10 @@ class OpenAIChatAPI(LanguageModel):
         api_responses = asyncio.run(
             self._async_batch_run_chatgpt(chat_messages_list, **kwargs),
         )
-        return [res.choices[0].message.content for res in api_responses]
+        completions = [res.choices[0].message.content for res in api_responses]
+        if all(completion == "" for completion in completions):
+            logger.warning("All generated texts are empty string. Something may go wrong.")
+        return completions
 
     def __repr__(self) -> str:
         return f"{self.__class__.__name__}(model={self.model})"

{flexeval-0.7.2 → flexeval-0.7.4}/flexeval/core/language_model/openai_batch_api.py

@@ -6,6 +6,7 @@ import os
 import tempfile
 import uuid
 from enum import Enum
+from pprint import pformat
 from typing import Any
 
 from loguru import logger
@@ -128,6 +129,10 @@ class OpenAIChatBatchAPI(LanguageModel):
             logger.info(f"Current status: {status.value}")
         return status, batch_response
 
+    def _retrieve_file_content(self, file_id: str) -> list[dict[any, any]]:
+        file_response = asyncio.run(self._client.files.content(file_id))
+        return [json.loads(line) for line in file_response.text.strip().split("\n")]
+
     def _execute_batch_requests(
         self,
         messages_list: list[list[dict[str, str]]],
@@ -136,13 +141,15 @@ class OpenAIChatBatchAPI(LanguageModel):
         custom_id_2_message: dict[str, list[dict[str, str]]] = {
             str(uuid.uuid4()): messages for messages in messages_list
         }
-        custom_id_2_response: dict[str, str | None] = {custom_id: None for custom_id in custom_id_2_message}
-        exec_cnt = 0
+        # The response will be an empty string if the API produces an error.
+        custom_id_2_response: dict[str, str] = {custom_id: "" for custom_id in custom_id_2_message}
+        exec_cnt = 1
 
         while len(custom_id_2_message) > 0:
             if exec_cnt > MAX_NUM_TRIALS:
                 break
             logger.info(f"Trial {exec_cnt}")
+            exec_cnt += 1
             batch_id = asyncio.run(self._post_batch_requests(custom_id_2_message, **kwargs))
 
             status, batch_response = asyncio.run(
@@ -152,13 +159,25 @@ class OpenAIChatBatchAPI(LanguageModel):
                 error_message = f"Failed: {batch_response}"
                 raise ValueError(error_message)
 
-            file_response = asyncio.run(self._client.files.content(batch_response.output_file_id))
-
-            data = []
-            for line in file_response.text.strip().split("\n"):
-                json_data = json.loads(line)
-                data.append(json_data)
-
+            # Check error_file_id exists and if exists, log error details.
+            error_file_id = batch_response.error_file_id
+            # If any request fails, error_file_id is set.
+            if error_file_id is not None:
+                logger.warning("Request on some messages failed following reason.")
+                data: list[dict[str, Any]] = self._retrieve_file_content(error_file_id)
+                # [Error](https://github.com/openai/openai-openapi/blob/master/openapi.yaml#L8857])
+                # instance is embedded in response.
+                for data_i in data:
+                    error = data_i["response"]
+                    logger.warning(f"Failed: {error}")
+
+            output_file_id = batch_response.output_file_id
+            # If completion on all input fails, output_file_id is None.
+            if output_file_id is None:
+                logger.warning("All request failed. Continue...")
+                continue
+
+            data: list[dict[str, Any]] = self._retrieve_file_content(output_file_id)
             for data_i in data:
                 if data_i["error"] is not None:
                     continue
@@ -167,11 +186,10 @@ class OpenAIChatBatchAPI(LanguageModel):
                 custom_id_2_message.pop(custom_id)
                 custom_id_2_response[custom_id] = data_i["response"]["body"]["choices"][0]["message"]["content"]
 
-            exec_cnt += 1
-
-        if sum([response is not None for response in custom_id_2_response.values()]) < len(messages_list):
-            error_message = "Exec failed"
-            raise ValueError(error_message)
+        # The remaining elements are all those that failed to complete request.
+        if custom_id_2_message:
+            logger.warning("The following messages failed to complete request.")
+            logger.warning(pformat(list(custom_id_2_message.values())))
 
         return list(custom_id_2_response.values())
 

{flexeval-0.7.2 → flexeval-0.7.4}/flexeval/core/metric/llm_score.py

@@ -49,7 +49,7 @@ def summarize_evaluator_scores(
         if score is None or category_key is None:
             continue
         if category_key in task_inputs:
-            category2valid_scores[task_inputs["category"]].append(score)
+            category2valid_scores[task_inputs[category_key]].append(score)
 
     category2mean_score: dict[str, float] = {}
     for category, valid_scores in category2valid_scores.items():

flexeval-0.7.4/flexeval/core/prompt_template/jinja2.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from typing import Any
+
+from flexeval.core.utils.jinja2_utils import JINJA2_ENV
+
+from .base import PromptTemplate
+
+
+class Jinja2PromptTemplate(PromptTemplate):
+    """
+    Embed task inputs using Jinja2 template engine.
+
+    Args:
+        template: The Jinja2 template to use.
+        template_path: The path to a file with the Jinja2 template to use.
+    """
+
+    def __init__(self, template: str | None = None, template_path: str | None = None) -> None:
+        if template is None and template_path is None:
+            msg = "Either template or template_path must be provided"
+            raise ValueError(msg)
+        if template is not None and template_path is not None:
+            msg = "Only one of template or template_path can be provided"
+            raise ValueError(msg)
+
+        if template_path is not None:
+            with open(template_path) as f:
+                self.template = f.read()
+        else:
+            self.template = template
+
+    def embed_inputs(self, input_dict: dict[str, Any]) -> str:
+        return JINJA2_ENV.from_string(self.template).render(input_dict)
+
+    def __repr__(self) -> str:
+        return f"Jinja2PromptTemplate(template={self.template!r})"

{flexeval-0.7.2 → flexeval-0.7.4}/flexeval/core/reward_model/pairwise_judge_reward_model.py

@@ -120,6 +120,6 @@ class PairwiseJudgeRewardModel(RewardModel):
 
         for i in range(len(outputs)):
             outputs[i]["llm_outputs"] = [judge_outputs[i * 2], judge_outputs[i * 2 + 1]]
-            outputs[i]["is_corrects"] = [chosen_is_betters[i * 2], chosen_is_betters[i * 2 + 1]]
+            outputs[i]["evaluation_results"] = [chosen_is_betters[i * 2], chosen_is_betters[i * 2 + 1]]
 
         return chosen_is_betters, outputs

{flexeval-0.7.2 → flexeval-0.7.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "flexeval"
-version = "0.7.2" # This will be automatically set from git tag by poetry-dynamic-versioning
+version = "0.7.4" # This will be automatically set from git tag by poetry-dynamic-versioning
 description = ""
 authors = ["ryokan-ri <ryokan.ri@sbintuitions.co.jp>"]
 readme = "README.md"

flexeval-0.7.2/flexeval/core/prompt_template/jinja2.py

@@ -1,25 +0,0 @@
-from __future__ import annotations
-
-from typing import Any
-
-from flexeval.core.utils.jinja2_utils import JINJA2_ENV
-
-from .base import PromptTemplate
-
-
-class Jinja2PromptTemplate(PromptTemplate):
-    """
-    Embed task inputs using Jinja2 template engine.
-
-    Args:
-        template: The Jinja2 template to use.
-    """
-
-    def __init__(self, template: str) -> None:
-        self.template = template
-
-    def embed_inputs(self, input_dict: dict[str, Any]) -> str:
-        return JINJA2_ENV.from_string(self.template).render(input_dict)
-
-    def __repr__(self) -> str:
-        return f"Jinja2PromptTemplate(template={self.template!r})"