eval-framework 0.3.6__tar.gz → 0.3.7__tar.gz

{eval_framework-0.3.6 → eval_framework-0.3.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.3.6
+Version: 0.3.7
 Summary: Evalulation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License

{eval_framework-0.3.6 → eval_framework-0.3.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.3.6"
+version = "0.3.7"
 description = "Evalulation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }

{eval_framework-0.3.6 → eval_framework-0.3.7}/src/eval_framework/llm/openai.py

@@ -8,18 +8,38 @@ from functools import partial
 
 import tiktoken
 from openai import OpenAI
-from openai.types.chat import ChatCompletionAssistantMessageParam, ChatCompletionUserMessageParam
+from openai.types.chat import (
+    ChatCompletionAssistantMessageParam,
+    ChatCompletionMessageParam,
+    ChatCompletionSystemMessageParam,
+    ChatCompletionUserMessageParam,
+)
 from tokenizers import Tokenizer
 from transformers import AutoTokenizer
 
 from eval_framework.llm.base import BaseLLM
 from eval_framework.shared.types import ConcatCompression, Error, RawCompletion, RawLoglikelihood
 from eval_framework.tasks.base import Sample
-from template_formatting.formatter import BaseFormatter, ConcatFormatter, HFFormatter, Message
+from template_formatting.formatter import BaseFormatter, ConcatFormatter, HFFormatter, Message, Role
 
 logger = logging.getLogger(__name__)
 
 
+def _to_chat_completion_message(message: Message) -> ChatCompletionMessageParam:
+    match message.role:
+        case Role.SYSTEM:
+            return ChatCompletionSystemMessageParam(role="system", content=message.content)
+        case Role.USER:
+            return ChatCompletionUserMessageParam(role="user", content=message.content)
+        case Role.ASSISTANT:
+            return ChatCompletionAssistantMessageParam(role="assistant", content=message.content)
+        case None:
+            raise ValueError(
+                "Cannot send a Message without a role through the chat completion API; "
+                "the legacy roleless format is only supported for fine-tuning."
+            )
+
+
 class OpenAIModel(BaseLLM):
     """
     LLM wrapper for OpenAI API providing text/chat completions and log-probability evaluation output.
@@ -158,14 +178,7 @@ class OpenAIModel(BaseLLM):
 
             else:
                 # Use chat completion API
-                chat_messages = [
-                    (
-                        ChatCompletionUserMessageParam(role="user", content=m.content)
-                        if m.role is not None and m.role.value.lower() == "user"
-                        else ChatCompletionAssistantMessageParam(role="assistant", content=m.content)
-                    )
-                    for m in single_messages
-                ]
+                chat_messages = [_to_chat_completion_message(m) for m in single_messages]
                 assert self._model_name is not None
                 chat_response = self._client.chat.completions.create(
                     model=self._model_name,

{eval_framework-0.3.6 → eval_framework-0.3.7}/src/eval_framework/tasks/benchmarks/ifeval.py

@@ -1,6 +1,7 @@
 from typing import Any
 
 from eval_framework.metrics.completion.ifeval import IFEvalMetric, IFEvalMetricContext
+from eval_framework.metrics.completion.language_checker import LanguageRawConsistencyChecker
 from eval_framework.tasks.base import NO_SUBJECT, BaseTask, Language, ResponseType
 
 
@@ -76,3 +77,4 @@ class IFEvalDe(IFEval):
     DATASET_PATH = "jzhang86/de_ifeval"
     SUBJECTS = [NO_SUBJECT]
     LANGUAGE = {NO_SUBJECT: Language.DEU}
+    METRICS = [IFEvalMetric, LanguageRawConsistencyChecker]