flexeval 0.17.2__tar.gz → 0.17.3__tar.gz

{flexeval-0.17.2 → flexeval-0.17.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flexeval
-Version: 0.17.2
+Version: 0.17.3
 Summary: 
 License-File: LICENSE
 Author: ryokan-ri

{flexeval-0.17.2 → flexeval-0.17.3}/flexeval/core/language_model/hf_lm.py

@@ -383,15 +383,36 @@ class HuggingFaceLM(LanguageModel):
 
         # We strip the input text and stop sequences from the output text.
         lm_outputs: list[LMOutput] = []
-        for generated_tensor in generated_tokens:
+        for attention_mask, generated_tensor in zip(model_inputs["attention_mask"], generated_tokens):
             input_tensor = generated_tensor[:input_token_length]
             output_tensor = generated_tensor[input_token_length:]
 
-            input_tokens = [t for t in input_tensor.tolist() if t != self.tokenizer.pad_token_id]
-            output_tokens = [t for t in output_tensor.tolist() if t != self.tokenizer.pad_token_id]
+            # Use the attention mask (rather than filtering by `pad_token_id`) to strip the
+            # left-padding from the input, since `pad_token_id` may equal `eos_token_id` (the
+            # tokenizer has no native pad token), in which case a value-based filter would also
+            # strip genuine EOS tokens that are part of the actual prompt content.
+            input_tokens = [t for t, m in zip(input_tensor.tolist(), attention_mask.tolist()) if m != 0]
+
+            # Find where generation actually stopped: the first occurrence of a stop token id in
+            # the output. Everything from that point on (the stop token itself and the padding
+            # that follows it) is discarded. Note that `pad_token_id` may equal one of the
+            # `stop_token_ids` (when the tokenizer has no native pad token), so a value-based pad
+            # filter would incorrectly strip the genuine stop token as well, making it impossible
+            # to tell "stop" from "length" below.
+            stop_token_pos = next(
+                (i for i, t in enumerate(output_tensor.tolist()) if t in stop_token_ids),
+                None,
+            )
+            if stop_token_pos is not None:
+                finish_reason = "stop"
+                output_tokens = output_tensor[:stop_token_pos].tolist()
+            else:
+                finish_reason = "length"
+                # Fall back to filtering by `pad_token_id`, as a safety net for models with a
+                # dedicated pad token (distinct from eos) that may still pad the output.
+                output_tokens = [t for t in output_tensor.tolist() if t != self.tokenizer.pad_token_id]
             decoded_text = decode_for_lm_continuation(output_tokens, input_tokens, self.tokenizer)
 
-            finish_reason = "length"
             for stop_seq in stop_sequences:
                 idx = decoded_text.find(stop_seq)
                 if idx != -1:
@@ -436,13 +457,17 @@ class HuggingFaceLM(LanguageModel):
                 if lm_output.raw_text is None:
                     lm_output.raw_text = lm_output.text
                 reasoning = self.reasoning_parser(lm_output.text)
-                lm_output.text = reasoning.text
+                # `reasoning.text` is None when the reasoning pattern fails to match.
+                # LMOutput.text must never be None (see LMOutput.__post_init__), so fall back to "".
+                lm_output.text = reasoning.text if reasoning.text is not None else ""
                 lm_output.reasoning_text = reasoning.reasoning_text
 
             if self.tool_parser and tools is not None:
                 parsed_tool_calling_message = self.tool_parser(lm_output.text)
                 lm_output.tool_calls = parsed_tool_calling_message.tool_call_dicts
-                lm_output.text = parsed_tool_calling_message.text
+                lm_output.text = (
+                    parsed_tool_calling_message.text if parsed_tool_calling_message.text is not None else ""
+                )
                 if lm_output.raw_text is None:
                     lm_output.raw_text = parsed_tool_calling_message.raw_text
                 lm_output.tool_call_validation_result = parsed_tool_calling_message.validation_result

{flexeval-0.17.2 → flexeval-0.17.3}/flexeval/core/language_model/openai_api.py

@@ -226,7 +226,7 @@ class OpenAIChatAPI(LanguageModel):
             LMOutput(
                 text=res.choices[0].message.content,
                 reasoning_text=get_reasoning_text(res.choices[0].message),
-                finish_reason=res.choices[0].finish_reason,
+                finish_reason="empty" if res is self.empty_response else res.choices[0].finish_reason,
             )
             for res in api_responses
         ]
@@ -246,7 +246,7 @@ class OpenAIChatAPI(LanguageModel):
             LMOutput(
                 text=res.choices[0].message.content,
                 reasoning_text=get_reasoning_text(res.choices[0].message),
-                finish_reason=res.choices[0].finish_reason,
+                finish_reason="empty" if res is self.empty_response else res.choices[0].finish_reason,
                 tool_calls=[tool_call.to_dict() for tool_call in res.choices[0].message.tool_calls]
                 if res.choices[0].message.tool_calls
                 else None,
@@ -295,17 +295,21 @@ class OpenAIChatAPI(LanguageModel):
         )
 
         log_probs = []
-        top_logprobs_list = [res.choices[0].logprobs.content[0].top_logprobs for res in api_responses]
+        top_logprobs_list = [
+            None if res is self.empty_response else res.choices[0].logprobs.content[0].top_logprobs
+            for res in api_responses
+        ]
         for index, prompt in enumerate(prompt_list):
             target_token = response_contents[index]
             index_in_unique = unique_prompt_list.index(prompt)
 
-            log_prob = None  # if target token not in top_logprobs, return None for log_prob of the token
+            log_prob = None  # if target token not in top_logprobs, or the request errored, return None
             top_logprobs = top_logprobs_list[index_in_unique]
-            for token_logprob in top_logprobs:
-                if token_logprob.token == target_token:
-                    log_prob = token_logprob.logprob
-                    break
+            if top_logprobs is not None:
+                for token_logprob in top_logprobs:
+                    if token_logprob.token == target_token:
+                        log_prob = token_logprob.logprob
+                        break
             log_probs.append(log_prob)
 
         return log_probs
@@ -450,7 +454,12 @@ class OpenAICompletionAPI(LanguageModel):
             **kwargs,
         )
 
-        return [LMOutput(text=res.choices[0].text, finish_reason=res.choices[0].finish_reason) for res in api_responses]
+        return [
+            LMOutput(text="", finish_reason="empty")
+            if res is self.empty_response
+            else LMOutput(text=res.choices[0].text, finish_reason=res.choices[0].finish_reason)
+            for res in api_responses
+        ]
 
     def __repr__(self) -> str:
         return f"{self.__class__.__name__}(model={self.model})"

{flexeval-0.17.2 → flexeval-0.17.3}/flexeval/core/language_model/openai_batch_api.py

@@ -257,7 +257,12 @@ class OpenAIChatBatchAPI(LanguageModel):
             **kwargs,
         )
         return [
-            LMOutput(text=res["choices"][0]["message"]["content"], finish_reason=res["choices"][0]["finish_reason"])
+            LMOutput(text="", finish_reason="empty")
+            if isinstance(res, str)
+            else LMOutput(
+                text=res["choices"][0]["message"]["content"],
+                finish_reason=res["choices"][0]["finish_reason"],
+            )
             for res in api_responses
         ]
 
@@ -273,7 +278,9 @@ class OpenAIChatBatchAPI(LanguageModel):
             **kwargs,
         )
         return [
-            LMOutput(
+            LMOutput(text="", finish_reason="empty")
+            if isinstance(res, str)
+            else LMOutput(
                 text=res["choices"][0]["message"]["content"],
                 finish_reason=res["choices"][0]["finish_reason"],
                 tool_calls=res["choices"][0]["message"].get("tool_calls", None),
@@ -319,17 +326,21 @@ class OpenAIChatBatchAPI(LanguageModel):
         )
 
         log_probs = []
-        top_logprobs_list = [res["choices"][0]["logprobs"]["content"][0]["top_logprobs"] for res in api_responses]
+        top_logprobs_list = [
+            None if isinstance(res, str) else res["choices"][0]["logprobs"]["content"][0]["top_logprobs"]
+            for res in api_responses
+        ]
         for index, prompt in enumerate(prompt_list):
             target_token = response_contents[index]
             index_in_unique = unique_prompt_list.index(prompt)
 
-            log_prob = None  # if target token not in top_logprobs, return None for log_prob of the token
+            log_prob = None  # if target token not in top_logprobs, or the request errored, return None
             top_logprobs = top_logprobs_list[index_in_unique]
-            for token_logprob in top_logprobs:
-                if token_logprob["token"] == target_token:
-                    log_prob = token_logprob["logprob"]
-                    break
+            if top_logprobs is not None:
+                for token_logprob in top_logprobs:
+                    if token_logprob["token"] == target_token:
+                        log_prob = token_logprob["logprob"]
+                        break
             log_probs.append(log_prob)
 
         return log_probs

{flexeval-0.17.2 → flexeval-0.17.3}/flexeval/core/language_model/vllm_model.py

@@ -284,13 +284,17 @@ class VLLM(LanguageModel):
                 if lm_output.raw_text is None:
                     lm_output.raw_text = lm_output.text
                 reasoning = self.reasoning_parser(lm_output.text)
-                lm_output.text = reasoning.text
+                # `reasoning.text` is None when the reasoning pattern fails to match.
+                # LMOutput.text must never be None (see LMOutput.__post_init__), so fall back to "".
+                lm_output.text = reasoning.text if reasoning.text is not None else ""
                 lm_output.reasoning_text = reasoning.reasoning_text
 
             if self.tool_parser and tools is not None:
                 parsed_tool_calling_message = self.tool_parser(lm_output.text)
                 lm_output.tool_calls = parsed_tool_calling_message.tool_call_dicts
-                lm_output.text = parsed_tool_calling_message.text
+                lm_output.text = (
+                    parsed_tool_calling_message.text if parsed_tool_calling_message.text is not None else ""
+                )
                 if lm_output.raw_text is None:
                     lm_output.raw_text = parsed_tool_calling_message.raw_text
                 lm_output.tool_call_validation_result = parsed_tool_calling_message.validation_result

{flexeval-0.17.2 → flexeval-0.17.3}/flexeval/core/language_model/vllm_serve_lm.py

@@ -193,7 +193,7 @@ class VLLMServeLM(OpenAIChatAPI):
         logging.getLogger("httpx").setLevel(logging.WARNING)
         logging.getLogger("httpcore").setLevel(logging.WARNING)
         model_kwargs = model_kwargs or {}
-        if "tensor_parallel_size" not in model_kwargs:
+        if "tensor_parallel_size" not in model_kwargs and "tensor-parallel-size" not in model_kwargs:
             model_kwargs["tensor_parallel_size"] = torch.cuda.device_count()
         self.manager = VLLMServerManager(model=model, model_kwargs=model_kwargs, timeout=booting_timeout)
         if api_headers is None:

{flexeval-0.17.2 → flexeval-0.17.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "flexeval"
-version = "0.17.2" # This will be automatically set from git tag by poetry-dynamic-versioning
+version = "0.17.3" # This will be automatically set from git tag by poetry-dynamic-versioning
 description = ""
 authors = ["ryokan-ri <ryokan.ri@sbintuitions.co.jp>"]
 readme = "README.md"