EuroEval 15.9.1__py3-none-any.whl → 15.9.2__py3-none-any.whl

euroeval/benchmark_modules/vllm.py

@@ -26,11 +26,13 @@ from transformers.trainer import Trainer
 from urllib3.exceptions import RequestError
 
 from ..constants import (
+    CUSTOM_STOP_TOKENS,
     GENERATIVE_PIPELINE_TAGS,
     MAX_CONTEXT_LENGTH,
     MAX_LOGPROBS,
     MERGE_TAGS,
     REASONING_MAX_TOKENS,
+    REASONING_TOKENS,
     TASKS_USING_JSON,
     VLLM_BF16_MIN_CUDA_COMPUTE_CAPABILITY,
 )
@@ -135,9 +137,15 @@ class VLLMModel(HuggingFaceEncoderModel):
         )
         self._model: LLM = model
         self._tokenizer: PreTrainedTokenizer = tokenizer
-        self.end_of_reasoning_token_id = get_end_of_reasoning_token_id(
+        self.end_of_reasoning_token = get_end_of_reasoning_token(
             model=self._model, tokenizer=self._tokenizer, model_id=model_config.model_id
         )
+        self.custom_stop_tokens = get_custom_stop_tokens(
+            model=self._model,
+            tokenizer=self._tokenizer,
+            model_id=model_config.model_id,
+            is_reasoning_model=self.end_of_reasoning_token is not None,
+        )
 
         # We specify `HuggingFaceEncoderModel` here instead of `VLLMModel`, as we want
         # to call the `__init__` method of the `BenchmarkModule` class.
@@ -183,7 +191,7 @@ class VLLMModel(HuggingFaceEncoderModel):
         """
         if not hasattr(self, "_tokenizer"):
             return None
-        elif self.end_of_reasoning_token_id is not None:
+        elif self.end_of_reasoning_token is not None:
             return GenerativeType.REASONING
         elif self._tokenizer.chat_template is not None:
             return GenerativeType.INSTRUCTION_TUNED
@@ -299,7 +307,7 @@ class VLLMModel(HuggingFaceEncoderModel):
         # token, end-of-sentence token, and a double newline if the model isn't
         # instruction tuned (since these separate the few-shot examples in the input in
         # this case)
-        stop_tokens: list[str] = list()
+        stop_tokens: list[str] = self.custom_stop_tokens.copy()
         if self.buffer["instruction_model"] is False:
             stop_tokens.append("\n\n")
         if self._tokenizer.pad_token_id is not None:
@@ -497,30 +505,26 @@ class VLLMModel(HuggingFaceEncoderModel):
         completion_ids: list[list[int]] = [
             output.outputs[0].token_ids for output in raw_outputs
         ]
-        if self.end_of_reasoning_token_id in completion_ids[0]:
-            # Find the latest index of the end of reasoning token and slice
-            # the token IDs to only include the tokens after it
-            completion_ids = [
-                token_ids[
-                    max(
-                        [
-                            i
-                            for i, x in enumerate(token_ids)
-                            if x == self.end_of_reasoning_token_id
-                        ]
-                    )
-                    + 1 :
-                ]
-                if self.end_of_reasoning_token_id in token_ids
-                else token_ids
-                for token_ids in completion_ids
-            ]
         completions = self._tokenizer.batch_decode(
             sequences=[
                 torch.LongTensor(completion_id) for completion_id in completion_ids
-            ],
-            skip_special_tokens=True,
+            ]
         )
+        if self.end_of_reasoning_token is not None:
+            completions = [
+                completion.split(self.end_of_reasoning_token)[-1]
+                for completion in completions
+            ]
+        if self.custom_stop_tokens:
+            stop_token_pattern = re.compile(
+                "|".join(
+                    re.escape(stop_token) for stop_token in self.custom_stop_tokens
+                )
+            )
+            completions = [
+                re.split(pattern=stop_token_pattern, string=completion)[0]
+                for completion in completions
+            ]
         completions = [completion.strip() for completion in completions]
 
         # Sanity check
@@ -541,17 +545,6 @@ class VLLMModel(HuggingFaceEncoderModel):
                 ]
                 for raw_output in raw_outputs
             ]
-            scores = [
-                score_list[
-                    raw_output.outputs[0].token_ids.index(
-                        self.end_of_reasoning_token_id
-                    )
-                    + 2 :
-                ]
-                if self.end_of_reasoning_token_id in raw_output.outputs[0].token_ids
-                else score_list
-                for raw_output, score_list in zip(raw_outputs, scores)
-            ]
             output = GenerativeModelOutput(sequences=completions, scores=scores)
         else:
             output = GenerativeModelOutput(sequences=completions)
@@ -1016,14 +1009,10 @@ def clear_vllm() -> None:
     clear_memory()
 
 
-def get_end_of_reasoning_token_id(
+def get_end_of_reasoning_token(
     model: "LLM", tokenizer: "PreTrainedTokenizer", model_id: str
-) -> int | None:
-    """Get the end of reasoning token ID for a generative model.
-
-    This assumes that the reasoning token is of the form <X> and that the end of
-    reasoning token is </X> (for X being any string without spaces). We disallow the
-    reasoning token to be the same as the beginning-of-sentence token.
+) -> str | None:
+    """Get the end-of-reasoning token for a generative model.
 
     Args:
         model:
@@ -1034,86 +1023,146 @@ def get_end_of_reasoning_token_id(
             The model ID.
 
     Returns:
-        The end of reasoning token ID, or None if it could not be found.
+        The end of reasoning token, or None if it could not be found.
     """
-    if tokenizer.chat_template is None:
-        prompt = "What is your name?"
-    else:
+    # Create a prompt to check if the model uses the reasoning tokens
+    prompt = "What is your name?"
+    if tokenizer.chat_template is not None:
         templated_prompt = tokenizer.apply_chat_template(
-            conversation=[dict(role="user", content="What is your name?")],
+            conversation=[dict(role="user", content=prompt)],
             add_generation_prompt=True,
             tokenize=False,
         )
         assert isinstance(templated_prompt, str)
         prompt = templated_prompt
 
-    # Generate a completion and remove the BOS token from it, to not confuse it with the
-    # potential reasoning token
-    model_output = model.generate(
-        prompts=[prompt],
-        sampling_params=SamplingParams(max_tokens=3, temperature=0.0),
-        use_tqdm=False,
+    # Check that the beginning-of-reasoning token is actually used by the model
+    completion = (
+        model.generate(
+            prompts=[prompt],
+            sampling_params=SamplingParams(max_tokens=10),
+            use_tqdm=False,
+        )[0]
+        .outputs[0]
+        .text
     )
-    completion = model_output[0].outputs[0].text
-
-    if tokenizer.bos_token is not None:
-        if isinstance(tokenizer.bos_token, str):
-            prompt = prompt.replace(tokenizer.bos_token, "").strip()
-            completion = completion.replace(tokenizer.bos_token, "").strip()
-        elif isinstance(tokenizer.bos_token, list):
-            for bos_token in tokenizer.bos_token:
-                prompt = prompt.replace(bos_token, "").strip()
-                completion = completion.replace(bos_token, "").strip()
-
-    # If it doesn't contain a reasoning token, we can't find the end of reasoning token
-    prompt_match = re.search(pattern=r"<\w+>", string=prompt)
-    completion_match = re.search(pattern=r"<\w+>", string=completion)
-    if completion_match is None and prompt_match is None:
+    bor_reasoning_matches = [
+        (bor_token, eor_token)
+        for bor_token, eor_token in REASONING_TOKENS
+        if bor_token in prompt or bor_token in completion
+    ]
+    if not bor_reasoning_matches:
         log_once(
-            f"Could not find a reasoning token for model {model_id!r}, so assuming "
-            "the model is not a reasoning model.",
-            level=logging.DEBUG,
+            f"The model {model_id!r} did not generate any beginning-of-reasoning "
+            "tokens in the prompt or the completion. Assuming the model is not "
+            "a reasoning model.",
+            level=logging.INFO,
         )
         return None
 
-    # Check that the found reasoning token and its associated end-of-reasoning tokens
-    # are both special tokens
-    elif completion_match is not None:
-        reasoning_token = completion_match.group()
-    else:
-        assert prompt_match is not None
-        reasoning_token = prompt_match.group()
-    end_of_reasoning_token = f"</{reasoning_token[1:-1]}>"
-    special_tokens = [
-        decoder_token.content
-        for decoder_token in tokenizer.added_tokens_decoder.values()
-    ]
-    special_tokens.extend(
-        [encoder_token for encoder_token in tokenizer.added_tokens_encoder.keys()]
+    # Check that the beginning-of-reasoning token is actually used by the model
+    completion = (
+        model.generate(
+            prompts=[prompt],
+            sampling_params=SamplingParams(max_tokens=REASONING_MAX_TOKENS),
+            use_tqdm=False,
+        )[0]
+        .outputs[0]
+        .text
     )
-    special_tokens.extend(tokenizer.all_special_tokens)
-    if (
-        reasoning_token not in special_tokens
-        or end_of_reasoning_token not in special_tokens
-    ):
+    eor_reasoning_matches = [
+        (bor_token, eor_token)
+        for bor_token, eor_token in bor_reasoning_matches
+        if eor_token in completion
+    ]
+    if not eor_reasoning_matches:
         log_once(
-            f"Detected reasoning token {reasoning_token!r} and end-of-reasoning "
-            f"token {end_of_reasoning_token!r} for model {model_id!r}, but one of "
-            "them is not registered as a special token, so assuming it is not a "
-            "real reasoning token.",
-            level=logging.DEBUG,
+            f"The model {model_id!r} did not generate any end-of-reasoning "
+            "tokens in the prompt or the completion, even though it generated "
+            "the beginning-of-reasoning tokens "
+            f"{[bor_token for bor_token, _ in bor_reasoning_matches]!r}. "
+            "This is probably not correct, so please report this issue.",
+            level=logging.INFO,
         )
         return None
 
+    if len(eor_reasoning_matches) > 1:
+        log_once(
+            f"Found multiple reasoning tokens {eor_reasoning_matches} for "
+            f"model {model_id!r}. Using {eor_reasoning_matches[0]!r} as "
+            "the reasoning token. If this is not the correct reasoning token, "
+            "please report this issue.",
+            level=logging.INFO,
+        )
+
+    bor_token, eor_token = eor_reasoning_matches[0]
     log_once(
-        f"Detected reasoning token {reasoning_token!r} and end-of-reasoning "
-        f"token {end_of_reasoning_token!r} for model {model_id!r}.",
-        level=logging.DEBUG,
+        f"Detected beginning-of-reasoning token {bor_token!r} and end-of-reasoning "
+        f"token {eor_token!r} for model {model_id!r}.",
+        level=logging.INFO,
     )
 
-    # Encode the end of reasoning token and return its ID
-    end_of_reasoning_token_id = tokenizer.encode(
-        text=end_of_reasoning_token, add_special_tokens=False
-    )[0]
+    return eor_token
+
+
+def get_custom_stop_tokens(
+    model: "LLM",
+    tokenizer: "PreTrainedTokenizer",
+    model_id: str,
+    is_reasoning_model: bool,
+) -> list[str]:
+    """Get the stop tokens for a generative model.
+
+    Args:
+        model:
+            The vLLM model.
+        tokenizer:
+            The tokenizer.
+        model_id:
+            The model ID.
+        is_reasoning_model:
+            Whether the model is a reasoning model. This is used to determine the number
+            of generated tokens to allow before stopping the generation.
+
+    Returns:
+        A list of stop tokens.
+    """
+    candidate_stop_tokens = CUSTOM_STOP_TOKENS
+
+    # Create a prompt to check if the model uses the reasoning tokens
+    prompt = "Hello"
+    if tokenizer.chat_template is not None:
+        templated_prompt = tokenizer.apply_chat_template(
+            conversation=[dict(role="user", content=prompt)],
+            add_generation_prompt=True,
+            tokenize=False,
+        )
+        assert isinstance(templated_prompt, str)
+        prompt = templated_prompt
+
+    # Check that the beginning-of-reasoning token is actually used by the model
+    max_tokens = REASONING_MAX_TOKENS if is_reasoning_model else 10
+    completion = (
+        model.generate(
+            prompts=[prompt],
+            sampling_params=SamplingParams(max_tokens=max_tokens, temperature=0.0),
+            use_tqdm=False,
+        )[0]
+        .outputs[0]
+        .text
+    )
+
+    stop_tokens = [
+        stop_token
+        for stop_token in candidate_stop_tokens
+        if stop_token in prompt or stop_token in completion
+    ]
+    if stop_tokens:
+        logger.debug(
+            f"Found the following custom stop tokens for model {model_id!r}: "
+            f"{stop_tokens}."
+        )
+    else:
+        logger.debug(f"Found no custom stop tokens for model {model_id!r}.")
 
-    return end_of_reasoning_token_id
+    return stop_tokens

euroeval/constants.py

@@ -64,3 +64,16 @@ MERGE_TAGS = ["merge", "mergekit"]
 
 # The minimum required CUDA compute capability for using bfloat16 in vLLM
 VLLM_BF16_MIN_CUDA_COMPUTE_CAPABILITY = 8.0
+
+# Used to detect whether a model is a reasoning model
+REASONING_TOKENS = [
+    ("<think>", "</think>"),
+    ("<reason>", "</reason>"),
+    ("<reasoning>", "</reasoning>"),
+]
+
+# These tokens are sometimes used by models to indicate the end of a generated
+# response, but they do not use them as a proper EOS token, so we have to deal with them
+# manually. We only use them as stop tokens if they actually appear in the model's
+# output
+CUSTOM_STOP_TOKENS = ["<sep>"]

euroeval/task_group_utils/sequence_classification.py

@@ -135,7 +135,7 @@ def extract_labels_from_generation(
         if first_label_token_mapping is False:
             raise InvalidBenchmark(
                 "The model outputted logprobs, but the first label token mapping is "
-                "not provided. This means that the model should not output logprobs."
+                "not provided, which is not supported."
             )
         labels = get_closest_logprobs_labels(
             generation_logprobs=model_output.scores,

euroeval/tokenization_utils.py

@@ -8,7 +8,6 @@ import torch
 
 from .constants import TASK_GROUPS_USING_LOGPROBS
 from .enums import GenerativeType
-from .exceptions import InvalidModel
 from .utils import log_once
 
 if t.TYPE_CHECKING:
@@ -153,7 +152,9 @@ def should_prefix_space_be_added_to_labels(
     return add_prefix_space
 
 
-def get_bos_token(tokenizer: "PreTrainedTokenizer") -> tuple[str, int]:
+def get_bos_token(
+    tokenizer: "PreTrainedTokenizer",
+) -> tuple[str, int] | tuple[None, None]:
     """Get the beginning-of-sequence token from a tokenizer.
 
     Args:
@@ -162,7 +163,7 @@ def get_bos_token(tokenizer: "PreTrainedTokenizer") -> tuple[str, int]:
 
     Returns:
         A pair (token, token_id) representing the beginning-of-sequence token and its
-        token ID.
+        token ID, or (None, None) if no BOS token is found.
     """
     if isinstance(tokenizer.bos_token, str) and isinstance(tokenizer.bos_token_id, int):
         return tokenizer.bos_token, tokenizer.bos_token_id
@@ -176,15 +177,20 @@ def get_bos_token(tokenizer: "PreTrainedTokenizer") -> tuple[str, int]:
             bos_token_id = vocab[bos_token]
             break
     else:
-        raise InvalidModel(
+        log_once(
             "The model does not have a beginning-of-sequence token. Please ensure that "
-            "this has been set in the tokenizer's configuration."
+            "this has been set in the tokenizer's configuration. Using no BOS token."
+            " This may lead to unexpected behavior in the model.",
+            level=logging.INFO,
         )
+        return None, None
 
     return bos_token, bos_token_id
 
 
-def get_eos_token(tokenizer: "PreTrainedTokenizer") -> tuple[str, int]:
+def get_eos_token(
+    tokenizer: "PreTrainedTokenizer",
+) -> tuple[str, int] | tuple[None, None]:
     """Get the end-of-sequence token from a tokenizer.
 
     Args:
@@ -193,7 +199,7 @@ def get_eos_token(tokenizer: "PreTrainedTokenizer") -> tuple[str, int]:
 
     Returns:
         A pair (token, token_id) representing the end-of-sequence token and its token
-        ID.
+        ID, or (None, None) if no EOS token is found.
     """
     if isinstance(tokenizer.eos_token, str) and isinstance(tokenizer.eos_token_id, int):
         return tokenizer.eos_token, tokenizer.eos_token_id
@@ -207,10 +213,13 @@ def get_eos_token(tokenizer: "PreTrainedTokenizer") -> tuple[str, int]:
             eos_token_id = vocab[eos_token]
             break
     else:
-        raise InvalidModel(
+        log_once(
             "The model does not have an end-of-sequence token. Please ensure that this "
-            "has been set in the tokenizer's configuration."
+            "has been set in the tokenizer's configuration. Using no EOS token. This "
+            "may lead to unexpected behavior in the model.",
+            level=logging.INFO,
         )
+        return None, None
 
     return eos_token, eos_token_id
 

{euroeval-15.9.1.dist-info → euroeval-15.9.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: EuroEval
-Version: 15.9.1
+Version: 15.9.2
 Summary: The robust European language model benchmark.
 Project-URL: Repository, https://github.com/EuroEval/EuroEval
 Project-URL: Issues, https://github.com/EuroEval/EuroEval/issues

{euroeval-15.9.1.dist-info → euroeval-15.9.2.dist-info}/RECORD

@@ -3,7 +3,7 @@ euroeval/benchmark_config_factory.py,sha256=icTeT5C-bNCJmvSWFlxKdEpRboZN8OjwaHGu
 euroeval/benchmarker.py,sha256=wmgrYVS31PMhhrVienjaVHHyfnZAy51kUvC6OjooiOw,48047
 euroeval/callbacks.py,sha256=F1AJCLB8FJpxqYprwLi_PsH4Bc0x4lyR8UiTG-GlFLY,2452
 euroeval/cli.py,sha256=d8JztMi_RbpUlEBXidd6DQ-xeC-xhozf_qU6Vkzye20,8161
-euroeval/constants.py,sha256=p6kp_R6-Tq5LBvyXyT6Sa6N3SkjEElGS2LSZRBoQaYs,1985
+euroeval/constants.py,sha256=0KHrH74zGM8vNF4uZG_a5qFJRZH5YgyQULYZtCKlo68,2452
 euroeval/data_loading.py,sha256=L_REtxefte5Ke4xE_Cz01zkfCyKlOYhSqT5ZXXulHPc,3992
 euroeval/data_models.py,sha256=7nAGDpN58Y35Lt9JZE_y0y5iOYesw2htcwHc68MkBZU,22953
 euroeval/enums.py,sha256=L9LcNeruuhHvze9vKRogXY9vonRzoBqDzWSP6hxKQ7A,3195
@@ -19,7 +19,7 @@ euroeval/model_loading.py,sha256=B6dyjYO0Dg7NOcUXls8Sjwe6W0c2UqJ1OGw-RkzoSSQ,223
 euroeval/scores.py,sha256=TovjCZD8wmGrIjA4v5oAQp18P5KVcHvakkByDh0Hstk,3059
 euroeval/speed_benchmark.py,sha256=J7VKWMf7GU_l0lRR8f0QeUr_vAaBQqTbgQ_yToHhp_0,3980
 euroeval/tasks.py,sha256=87gbe__K5KNIb1aBSuwGnMPmZgamJFecNNYmNgMxaVo,7069
-euroeval/tokenization_utils.py,sha256=kghOIZMM3H0P9YDv0VBSNI7drzgJXlkRtMwt3Cgeev8,13907
+euroeval/tokenization_utils.py,sha256=_B4KN3ZcuvVr8y3LedtfxBJfmPKjfVMjpbtl8bbQAuc,14278
 euroeval/types.py,sha256=E0JhLfg-ek5pdFcYJbnGRUSodHxkuR3o8XGuIrBcuRM,2485
 euroeval/utils.py,sha256=e83OnWc0GJn0Tn_vP3tbqh1DAbLy2ky-LnIlTEOKzKU,11410
 euroeval/benchmark_modules/__init__.py,sha256=TNO-sNDwlXE-LMFXfwwqjQqUy55gywSmwRBcoPUFuaU,236
@@ -27,7 +27,7 @@ euroeval/benchmark_modules/base.py,sha256=LcG46I2O5wcvu_3T_irBY6VkUhWVPKifBhcP-l
 euroeval/benchmark_modules/fresh.py,sha256=_LWmpqiNGGTA-NoVC0v3-fS1sraDS9n-pgKUzz89jVk,9919
 euroeval/benchmark_modules/hf.py,sha256=CoiaNakjhg6gm_5IbUUeevXQZebg2VrRLuhzEi2Hhrk,44617
 euroeval/benchmark_modules/litellm.py,sha256=SxSr_0C6b_jVavR3y9QyhfkCOP5-va4zijGfghFTArY,48362
-euroeval/benchmark_modules/vllm.py,sha256=rz_Xau5TGiFeb2VkdVpW_fYOfRCCvYrH0q9BGzCwZlo,42156
+euroeval/benchmark_modules/vllm.py,sha256=SbQ_EYSwUFBVLsp9io1Q75A9S_H-iw6AzLOn3rlEhK0,43034
 euroeval/dataset_configs/__init__.py,sha256=kWKtlSAOY-olOQL3UtFqL6I3Tki3G3waMZSd2YChjCg,1895
 euroeval/dataset_configs/danish.py,sha256=MTt9EcriSer0QaFQ7_6evYxh-g9OPjroWegYdFpiKag,3395
 euroeval/dataset_configs/dutch.py,sha256=r21nxEvMmBkKqPXVW082batPsxJ9d0RB4DzngOTMJSk,3185
@@ -51,11 +51,11 @@ euroeval/prompt_templates/summarization.py,sha256=mcWeKNhGWmp7IG_iY64T-VOSabQg5w
 euroeval/task_group_utils/__init__.py,sha256=CorGVkixkoEDOQuDsrOGlTmF1zmM0wnGHs8psWTfD28,72
 euroeval/task_group_utils/multiple_choice_classification.py,sha256=nB78TzOgd0HBvTclmjOYJid9ZVAgu8IHZsqB_n1SAZU,6178
 euroeval/task_group_utils/question_answering.py,sha256=kZBABJ_WYNTH4Xgo2jIvfx7iYvfoGt0EUObSaXRCGmk,27700
-euroeval/task_group_utils/sequence_classification.py,sha256=Yqx0pUhuHYmSkv1ZUfOndSLTvpr0lWCk19oYITfSjV4,13555
+euroeval/task_group_utils/sequence_classification.py,sha256=zwRUgVHqLlREILwyg-yuDPkrIQOfqGVPsFBai-2D9a8,13525
 euroeval/task_group_utils/text_to_text.py,sha256=Nu1_qRPLbboCd9Q5rxqY4fQFJ_aGXu80aWQqoTG1cYc,5047
 euroeval/task_group_utils/token_classification.py,sha256=3idWB81Fcx9UhTuk-gxMfXENrCBmiWBDUWdULXoIhpw,17863
-euroeval-15.9.1.dist-info/METADATA,sha256=UkGmFcnarstFwD1J1eS6h3gbyxnucnaAVLnB5QhkdSo,13555
-euroeval-15.9.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-euroeval-15.9.1.dist-info/entry_points.txt,sha256=tKQRxN0HX2mGtbZbZQdCRFUDZIecA_z4mZduueor3Ug,135
-euroeval-15.9.1.dist-info/licenses/LICENSE,sha256=oZp5fpOSQ7w-vFui8QNwrBIosrO7cnpArItdbvn52Ao,1082
-euroeval-15.9.1.dist-info/RECORD,,
+euroeval-15.9.2.dist-info/METADATA,sha256=LwHTlJ51OGVwcRTUPulH-gh8IFxu82CUFYHZ1uOUyT0,13555
+euroeval-15.9.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+euroeval-15.9.2.dist-info/entry_points.txt,sha256=tKQRxN0HX2mGtbZbZQdCRFUDZIecA_z4mZduueor3Ug,135
+euroeval-15.9.2.dist-info/licenses/LICENSE,sha256=oZp5fpOSQ7w-vFui8QNwrBIosrO7cnpArItdbvn52Ao,1082
+euroeval-15.9.2.dist-info/RECORD,,