flexeval 0.17.2__tar.gz → 0.18.0__tar.gz

{flexeval-0.17.2 → flexeval-0.18.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flexeval
-Version: 0.17.2
+Version: 0.18.0
 Summary: 
 License-File: LICENSE
 Author: ryokan-ri
@@ -13,7 +13,7 @@ Classifier: Programming Language :: Python :: 3.12
 Provides-Extra: vllm
 Provides-Extra: wandb
 Requires-Dist: datasets (>=2.14.6,<3.0.0)
-Requires-Dist: evaluate (>=0.4.1,<0.5.0)
+Requires-Dist: evaluate (>=0.4.6,<0.5.0)
 Requires-Dist: fuzzywuzzy (>=0.18.0,<0.19.0)
 Requires-Dist: google-api-python-client (>=2.131.0,<3.0.0)
 Requires-Dist: jinja2 (>=3.1.2,<4.0.0)
@@ -22,8 +22,8 @@ Requires-Dist: jsonargparse[jsonnet] (>=4.26.1,<5.0.0)
 Requires-Dist: litellm (>=1.52.9,<2.0.0)
 Requires-Dist: loguru (>=0.7.2,<0.8.0)
 Requires-Dist: math-verify[antlr4-13-2] (>=0.7.0,<0.8.0)
-Requires-Dist: openai (>=1.52.2,<2.0.0)
-Requires-Dist: peft (>=0.10.0,<0.11.0)
+Requires-Dist: openai (>=2.0.0,<3.0.0)
+Requires-Dist: peft (>=0.19.1,<0.20.0)
 Requires-Dist: pyarrow (==16.1.0)
 Requires-Dist: python-levenshtein (>=0.23.0,<0.24.0)
 Requires-Dist: rouge (>=1.0.1,<2.0.0)
@@ -33,8 +33,8 @@ Requires-Dist: scipy (>=1.13.0,<2.0.0)
 Requires-Dist: smart-open (>=7.1.0,<8.0.0)
 Requires-Dist: sudachipy (>=0.6.10)
 Requires-Dist: tiktoken (>=0.9.0,<0.10.0)
-Requires-Dist: transformers[ja,sentencepiece,torch] (>=4.34.1,<5.0.0)
-Requires-Dist: vllm (==0.17.1) ; extra == "vllm"
+Requires-Dist: transformers[ja,sentencepiece,torch] (>=5.5.3,<6)
+Requires-Dist: vllm (==0.24.0) ; extra == "vllm"
 Requires-Dist: wandb (>=0.17.2,<0.18.0) ; extra == "wandb"
 Description-Content-Type: text/markdown
 

{flexeval-0.17.2 → flexeval-0.18.0}/flexeval/core/language_model/hf_lm.py

@@ -383,15 +383,36 @@ class HuggingFaceLM(LanguageModel):
 
         # We strip the input text and stop sequences from the output text.
         lm_outputs: list[LMOutput] = []
-        for generated_tensor in generated_tokens:
+        for attention_mask, generated_tensor in zip(model_inputs["attention_mask"], generated_tokens):
             input_tensor = generated_tensor[:input_token_length]
             output_tensor = generated_tensor[input_token_length:]
 
-            input_tokens = [t for t in input_tensor.tolist() if t != self.tokenizer.pad_token_id]
-            output_tokens = [t for t in output_tensor.tolist() if t != self.tokenizer.pad_token_id]
+            # Use the attention mask (rather than filtering by `pad_token_id`) to strip the
+            # left-padding from the input, since `pad_token_id` may equal `eos_token_id` (the
+            # tokenizer has no native pad token), in which case a value-based filter would also
+            # strip genuine EOS tokens that are part of the actual prompt content.
+            input_tokens = [t for t, m in zip(input_tensor.tolist(), attention_mask.tolist()) if m != 0]
+
+            # Find where generation actually stopped: the first occurrence of a stop token id in
+            # the output. Everything from that point on (the stop token itself and the padding
+            # that follows it) is discarded. Note that `pad_token_id` may equal one of the
+            # `stop_token_ids` (when the tokenizer has no native pad token), so a value-based pad
+            # filter would incorrectly strip the genuine stop token as well, making it impossible
+            # to tell "stop" from "length" below.
+            stop_token_pos = next(
+                (i for i, t in enumerate(output_tensor.tolist()) if t in stop_token_ids),
+                None,
+            )
+            if stop_token_pos is not None:
+                finish_reason = "stop"
+                output_tokens = output_tensor[:stop_token_pos].tolist()
+            else:
+                finish_reason = "length"
+                # Fall back to filtering by `pad_token_id`, as a safety net for models with a
+                # dedicated pad token (distinct from eos) that may still pad the output.
+                output_tokens = [t for t in output_tensor.tolist() if t != self.tokenizer.pad_token_id]
             decoded_text = decode_for_lm_continuation(output_tokens, input_tokens, self.tokenizer)
 
-            finish_reason = "length"
             for stop_seq in stop_sequences:
                 idx = decoded_text.find(stop_seq)
                 if idx != -1:
@@ -436,13 +457,17 @@ class HuggingFaceLM(LanguageModel):
                 if lm_output.raw_text is None:
                     lm_output.raw_text = lm_output.text
                 reasoning = self.reasoning_parser(lm_output.text)
-                lm_output.text = reasoning.text
+                # `reasoning.text` is None when the reasoning pattern fails to match.
+                # LMOutput.text must never be None (see LMOutput.__post_init__), so fall back to "".
+                lm_output.text = reasoning.text if reasoning.text is not None else ""
                 lm_output.reasoning_text = reasoning.reasoning_text
 
             if self.tool_parser and tools is not None:
                 parsed_tool_calling_message = self.tool_parser(lm_output.text)
                 lm_output.tool_calls = parsed_tool_calling_message.tool_call_dicts
-                lm_output.text = parsed_tool_calling_message.text
+                lm_output.text = (
+                    parsed_tool_calling_message.text if parsed_tool_calling_message.text is not None else ""
+                )
                 if lm_output.raw_text is None:
                     lm_output.raw_text = parsed_tool_calling_message.raw_text
                 lm_output.tool_call_validation_result = parsed_tool_calling_message.validation_result

{flexeval-0.17.2 → flexeval-0.18.0}/flexeval/core/language_model/openai_api.py

@@ -226,7 +226,7 @@ class OpenAIChatAPI(LanguageModel):
             LMOutput(
                 text=res.choices[0].message.content,
                 reasoning_text=get_reasoning_text(res.choices[0].message),
-                finish_reason=res.choices[0].finish_reason,
+                finish_reason="empty" if res is self.empty_response else res.choices[0].finish_reason,
             )
             for res in api_responses
         ]
@@ -246,7 +246,7 @@ class OpenAIChatAPI(LanguageModel):
             LMOutput(
                 text=res.choices[0].message.content,
                 reasoning_text=get_reasoning_text(res.choices[0].message),
-                finish_reason=res.choices[0].finish_reason,
+                finish_reason="empty" if res is self.empty_response else res.choices[0].finish_reason,
                 tool_calls=[tool_call.to_dict() for tool_call in res.choices[0].message.tool_calls]
                 if res.choices[0].message.tool_calls
                 else None,
@@ -295,17 +295,21 @@ class OpenAIChatAPI(LanguageModel):
         )
 
         log_probs = []
-        top_logprobs_list = [res.choices[0].logprobs.content[0].top_logprobs for res in api_responses]
+        top_logprobs_list = [
+            None if res is self.empty_response else res.choices[0].logprobs.content[0].top_logprobs
+            for res in api_responses
+        ]
         for index, prompt in enumerate(prompt_list):
             target_token = response_contents[index]
             index_in_unique = unique_prompt_list.index(prompt)
 
-            log_prob = None  # if target token not in top_logprobs, return None for log_prob of the token
+            log_prob = None  # if target token not in top_logprobs, or the request errored, return None
             top_logprobs = top_logprobs_list[index_in_unique]
-            for token_logprob in top_logprobs:
-                if token_logprob.token == target_token:
-                    log_prob = token_logprob.logprob
-                    break
+            if top_logprobs is not None:
+                for token_logprob in top_logprobs:
+                    if token_logprob.token == target_token:
+                        log_prob = token_logprob.logprob
+                        break
             log_probs.append(log_prob)
 
         return log_probs
@@ -450,7 +454,12 @@ class OpenAICompletionAPI(LanguageModel):
             **kwargs,
         )
 
-        return [LMOutput(text=res.choices[0].text, finish_reason=res.choices[0].finish_reason) for res in api_responses]
+        return [
+            LMOutput(text="", finish_reason="empty")
+            if res is self.empty_response
+            else LMOutput(text=res.choices[0].text, finish_reason=res.choices[0].finish_reason)
+            for res in api_responses
+        ]
 
     def __repr__(self) -> str:
         return f"{self.__class__.__name__}(model={self.model})"

{flexeval-0.17.2 → flexeval-0.18.0}/flexeval/core/language_model/openai_batch_api.py

@@ -257,7 +257,12 @@ class OpenAIChatBatchAPI(LanguageModel):
             **kwargs,
         )
         return [
-            LMOutput(text=res["choices"][0]["message"]["content"], finish_reason=res["choices"][0]["finish_reason"])
+            LMOutput(text="", finish_reason="empty")
+            if isinstance(res, str)
+            else LMOutput(
+                text=res["choices"][0]["message"]["content"],
+                finish_reason=res["choices"][0]["finish_reason"],
+            )
             for res in api_responses
         ]
 
@@ -273,7 +278,9 @@ class OpenAIChatBatchAPI(LanguageModel):
             **kwargs,
         )
         return [
-            LMOutput(
+            LMOutput(text="", finish_reason="empty")
+            if isinstance(res, str)
+            else LMOutput(
                 text=res["choices"][0]["message"]["content"],
                 finish_reason=res["choices"][0]["finish_reason"],
                 tool_calls=res["choices"][0]["message"].get("tool_calls", None),
@@ -319,17 +326,21 @@ class OpenAIChatBatchAPI(LanguageModel):
         )
 
         log_probs = []
-        top_logprobs_list = [res["choices"][0]["logprobs"]["content"][0]["top_logprobs"] for res in api_responses]
+        top_logprobs_list = [
+            None if isinstance(res, str) else res["choices"][0]["logprobs"]["content"][0]["top_logprobs"]
+            for res in api_responses
+        ]
         for index, prompt in enumerate(prompt_list):
             target_token = response_contents[index]
             index_in_unique = unique_prompt_list.index(prompt)
 
-            log_prob = None  # if target token not in top_logprobs, return None for log_prob of the token
+            log_prob = None  # if target token not in top_logprobs, or the request errored, return None
             top_logprobs = top_logprobs_list[index_in_unique]
-            for token_logprob in top_logprobs:
-                if token_logprob["token"] == target_token:
-                    log_prob = token_logprob["logprob"]
-                    break
+            if top_logprobs is not None:
+                for token_logprob in top_logprobs:
+                    if token_logprob["token"] == target_token:
+                        log_prob = token_logprob["logprob"]
+                        break
             log_probs.append(log_prob)
 
         return log_probs

{flexeval-0.17.2 → flexeval-0.18.0}/flexeval/core/language_model/vllm_model.py

@@ -284,13 +284,17 @@ class VLLM(LanguageModel):
                 if lm_output.raw_text is None:
                     lm_output.raw_text = lm_output.text
                 reasoning = self.reasoning_parser(lm_output.text)
-                lm_output.text = reasoning.text
+                # `reasoning.text` is None when the reasoning pattern fails to match.
+                # LMOutput.text must never be None (see LMOutput.__post_init__), so fall back to "".
+                lm_output.text = reasoning.text if reasoning.text is not None else ""
                 lm_output.reasoning_text = reasoning.reasoning_text
 
             if self.tool_parser and tools is not None:
                 parsed_tool_calling_message = self.tool_parser(lm_output.text)
                 lm_output.tool_calls = parsed_tool_calling_message.tool_call_dicts
-                lm_output.text = parsed_tool_calling_message.text
+                lm_output.text = (
+                    parsed_tool_calling_message.text if parsed_tool_calling_message.text is not None else ""
+                )
                 if lm_output.raw_text is None:
                     lm_output.raw_text = parsed_tool_calling_message.raw_text
                 lm_output.tool_call_validation_result = parsed_tool_calling_message.validation_result

{flexeval-0.17.2 → flexeval-0.18.0}/flexeval/core/language_model/vllm_serve_lm.py

@@ -106,6 +106,17 @@ class VLLMServerManager:
 
         interval_second = 1
         for _ in range(self.timeout // interval_second):
+            # Fail fast if the vLLM process disappears or exits while the server is starting.
+            if self.process is None:
+                msg = "vLLM server process is not initialized."
+                raise RuntimeError(msg)
+
+            returncode = self.process.poll()
+            if returncode is not None:
+                self.stop()
+                msg = f"vLLM server process terminated unexpectedly. {returncode=}"
+                raise RuntimeError(msg)
+
             if self.is_ready():
                 logger.info(f"vLLM server is ready at {self.base_url}")
                 return self.host, self.port
@@ -193,7 +204,7 @@ class VLLMServeLM(OpenAIChatAPI):
         logging.getLogger("httpx").setLevel(logging.WARNING)
         logging.getLogger("httpcore").setLevel(logging.WARNING)
         model_kwargs = model_kwargs or {}
-        if "tensor_parallel_size" not in model_kwargs:
+        if "tensor_parallel_size" not in model_kwargs and "tensor-parallel-size" not in model_kwargs:
             model_kwargs["tensor_parallel_size"] = torch.cuda.device_count()
         self.manager = VLLMServerManager(model=model, model_kwargs=model_kwargs, timeout=booting_timeout)
         if api_headers is None:

{flexeval-0.17.2 → flexeval-0.18.0}/flexeval/utils/hf_utils.py

@@ -29,7 +29,7 @@ def get_default_model_kwargs(model_kwargs: None | dict[str, Any] = None) -> dict
     """
     This provides a default set of model kwargs for initializing a model in flexeval.
     - It uses the "auto" device_map to use any available device (GPU, MPS, etc.).
-    - It uses the "auto" torch_dtype to use the optimal dtype for the model.
+    - It uses the "auto" dtype to use the optimal dtype for the model.
     """
     model_kwargs = model_kwargs or {}
     model_kwargs = {**model_kwargs}  # copy kwargs to avoid modifying the original dict
@@ -39,17 +39,21 @@ def get_default_model_kwargs(model_kwargs: None | dict[str, Any] = None) -> dict
             # If the macOS version is older than 14.0, we need to use "cpu" instead of "auto".
             # This is to avoid MPS-related issues on older macOS versions.
             model_kwargs["device_map"] = "cpu"
-    if "torch_dtype" not in model_kwargs:
-        # You need to set torch_dtype to use the optimal dtype for the model.
+    # `torch_dtype` was renamed to `dtype` in transformers v5. Keep accepting the old
+    # name so existing configs don't break.
+    if "torch_dtype" in model_kwargs:
+        model_kwargs["dtype"] = model_kwargs.pop("torch_dtype")
+    if "dtype" not in model_kwargs:
+        # You need to set dtype to use the optimal dtype for the model.
         # https://huggingface.co/docs/transformers/main/main_classes/model#model-instantiation-dtype
-        model_kwargs["torch_dtype"] = "auto"
-    elif model_kwargs["torch_dtype"] != "auto":
+        model_kwargs["dtype"] = "auto"
+    elif model_kwargs["dtype"] != "auto":
         # Convert string to torch.dtype
         # We allow either "bfloat16" or "torch.bfloat16"
-        torch_dtype_str = model_kwargs["torch_dtype"]
-        torch_dtype_str = torch_dtype_str.removeprefix("torch.")
-        model_kwargs["torch_dtype"] = getattr(torch, torch_dtype_str)
-        if not isinstance(model_kwargs["torch_dtype"], torch.dtype):
-            msg = f"Invalid torch_dtype: {model_kwargs['torch_dtype']}"
+        dtype_str = model_kwargs["dtype"]
+        dtype_str = dtype_str.removeprefix("torch.")
+        model_kwargs["dtype"] = getattr(torch, dtype_str)
+        if not isinstance(model_kwargs["dtype"], torch.dtype):
+            msg = f"Invalid dtype: {model_kwargs['dtype']}"
             raise ValueError(msg)
     return model_kwargs

{flexeval-0.17.2 → flexeval-0.18.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "flexeval"
-version = "0.17.2" # This will be automatically set from git tag by poetry-dynamic-versioning
+version = "0.18.0" # This will be automatically set from git tag by poetry-dynamic-versioning
 description = ""
 authors = ["ryokan-ri <ryokan.ri@sbintuitions.co.jp>"]
 readme = "README.md"
@@ -16,10 +16,10 @@ flexeval_presets = "flexeval.scripts.flexeval_presets:main"
 
 [tool.poetry.dependencies]
 python = ">=3.10,<3.13"  # set version bound because we get "Unable to find installation candidates" with torch 2.5.1
-transformers = {extras = ["ja", "sentencepiece", "torch"], version = "^4.34.1"}
+transformers = {extras = ["ja", "sentencepiece", "torch"], version = ">=5.5.3,<6"}
 datasets = "^2.14.6"
-evaluate = "^0.4.1"
-peft = "^0.10.0"
+evaluate = "^0.4.6"  # 0.4.1-0.4.5 call huggingface_hub's removed `HfFolder` API and break with huggingface_hub>=1.0
+peft = "^0.19.1"
 jsonargparse = {extras = ["jsonnet"], version = "^4.26.1"}
 jinja2 = "^3.1.2"
 fuzzywuzzy = "^0.18.0"
@@ -27,9 +27,9 @@ python-levenshtein = "^0.23.0"
 rouge = "^1.0.1"
 sacrebleu = {extras = ["ja"], version = "^2.4.1"}
 jiwer = "^3.0.4"
-openai = "^1.52.2"
+openai = ">=2.0.0,<3.0.0"
 google-api-python-client = "^2.131.0"
-vllm = {version = "0.17.1", optional = true }
+vllm = {version = "0.24.0", optional = true }
 loguru = "^0.7.2"
 wandb = {version = "^0.17.2", optional = true}
 pyarrow = "16.1.0"  # set the version because we get "Unable to find installation candidates" with 17.0.0