flexeval 0.17.3__tar.gz → 0.18.0__tar.gz

{flexeval-0.17.3 → flexeval-0.18.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flexeval
-Version: 0.17.3
+Version: 0.18.0
 Summary: 
 License-File: LICENSE
 Author: ryokan-ri
@@ -13,7 +13,7 @@ Classifier: Programming Language :: Python :: 3.12
 Provides-Extra: vllm
 Provides-Extra: wandb
 Requires-Dist: datasets (>=2.14.6,<3.0.0)
-Requires-Dist: evaluate (>=0.4.1,<0.5.0)
+Requires-Dist: evaluate (>=0.4.6,<0.5.0)
 Requires-Dist: fuzzywuzzy (>=0.18.0,<0.19.0)
 Requires-Dist: google-api-python-client (>=2.131.0,<3.0.0)
 Requires-Dist: jinja2 (>=3.1.2,<4.0.0)
@@ -22,8 +22,8 @@ Requires-Dist: jsonargparse[jsonnet] (>=4.26.1,<5.0.0)
 Requires-Dist: litellm (>=1.52.9,<2.0.0)
 Requires-Dist: loguru (>=0.7.2,<0.8.0)
 Requires-Dist: math-verify[antlr4-13-2] (>=0.7.0,<0.8.0)
-Requires-Dist: openai (>=1.52.2,<2.0.0)
-Requires-Dist: peft (>=0.10.0,<0.11.0)
+Requires-Dist: openai (>=2.0.0,<3.0.0)
+Requires-Dist: peft (>=0.19.1,<0.20.0)
 Requires-Dist: pyarrow (==16.1.0)
 Requires-Dist: python-levenshtein (>=0.23.0,<0.24.0)
 Requires-Dist: rouge (>=1.0.1,<2.0.0)
@@ -33,8 +33,8 @@ Requires-Dist: scipy (>=1.13.0,<2.0.0)
 Requires-Dist: smart-open (>=7.1.0,<8.0.0)
 Requires-Dist: sudachipy (>=0.6.10)
 Requires-Dist: tiktoken (>=0.9.0,<0.10.0)
-Requires-Dist: transformers[ja,sentencepiece,torch] (>=4.34.1,<5.0.0)
-Requires-Dist: vllm (==0.17.1) ; extra == "vllm"
+Requires-Dist: transformers[ja,sentencepiece,torch] (>=5.5.3,<6)
+Requires-Dist: vllm (==0.24.0) ; extra == "vllm"
 Requires-Dist: wandb (>=0.17.2,<0.18.0) ; extra == "wandb"
 Description-Content-Type: text/markdown
 

{flexeval-0.17.3 → flexeval-0.18.0}/flexeval/core/language_model/vllm_serve_lm.py

@@ -106,6 +106,17 @@ class VLLMServerManager:
 
         interval_second = 1
         for _ in range(self.timeout // interval_second):
+            # Fail fast if the vLLM process disappears or exits while the server is starting.
+            if self.process is None:
+                msg = "vLLM server process is not initialized."
+                raise RuntimeError(msg)
+
+            returncode = self.process.poll()
+            if returncode is not None:
+                self.stop()
+                msg = f"vLLM server process terminated unexpectedly. {returncode=}"
+                raise RuntimeError(msg)
+
             if self.is_ready():
                 logger.info(f"vLLM server is ready at {self.base_url}")
                 return self.host, self.port

{flexeval-0.17.3 → flexeval-0.18.0}/flexeval/utils/hf_utils.py

@@ -29,7 +29,7 @@ def get_default_model_kwargs(model_kwargs: None | dict[str, Any] = None) -> dict
     """
     This provides a default set of model kwargs for initializing a model in flexeval.
     - It uses the "auto" device_map to use any available device (GPU, MPS, etc.).
-    - It uses the "auto" torch_dtype to use the optimal dtype for the model.
+    - It uses the "auto" dtype to use the optimal dtype for the model.
     """
     model_kwargs = model_kwargs or {}
     model_kwargs = {**model_kwargs}  # copy kwargs to avoid modifying the original dict
@@ -39,17 +39,21 @@ def get_default_model_kwargs(model_kwargs: None | dict[str, Any] = None) -> dict
             # If the macOS version is older than 14.0, we need to use "cpu" instead of "auto".
             # This is to avoid MPS-related issues on older macOS versions.
             model_kwargs["device_map"] = "cpu"
-    if "torch_dtype" not in model_kwargs:
-        # You need to set torch_dtype to use the optimal dtype for the model.
+    # `torch_dtype` was renamed to `dtype` in transformers v5. Keep accepting the old
+    # name so existing configs don't break.
+    if "torch_dtype" in model_kwargs:
+        model_kwargs["dtype"] = model_kwargs.pop("torch_dtype")
+    if "dtype" not in model_kwargs:
+        # You need to set dtype to use the optimal dtype for the model.
         # https://huggingface.co/docs/transformers/main/main_classes/model#model-instantiation-dtype
-        model_kwargs["torch_dtype"] = "auto"
-    elif model_kwargs["torch_dtype"] != "auto":
+        model_kwargs["dtype"] = "auto"
+    elif model_kwargs["dtype"] != "auto":
         # Convert string to torch.dtype
         # We allow either "bfloat16" or "torch.bfloat16"
-        torch_dtype_str = model_kwargs["torch_dtype"]
-        torch_dtype_str = torch_dtype_str.removeprefix("torch.")
-        model_kwargs["torch_dtype"] = getattr(torch, torch_dtype_str)
-        if not isinstance(model_kwargs["torch_dtype"], torch.dtype):
-            msg = f"Invalid torch_dtype: {model_kwargs['torch_dtype']}"
+        dtype_str = model_kwargs["dtype"]
+        dtype_str = dtype_str.removeprefix("torch.")
+        model_kwargs["dtype"] = getattr(torch, dtype_str)
+        if not isinstance(model_kwargs["dtype"], torch.dtype):
+            msg = f"Invalid dtype: {model_kwargs['dtype']}"
             raise ValueError(msg)
     return model_kwargs

{flexeval-0.17.3 → flexeval-0.18.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "flexeval"
-version = "0.17.3" # This will be automatically set from git tag by poetry-dynamic-versioning
+version = "0.18.0" # This will be automatically set from git tag by poetry-dynamic-versioning
 description = ""
 authors = ["ryokan-ri <ryokan.ri@sbintuitions.co.jp>"]
 readme = "README.md"
@@ -16,10 +16,10 @@ flexeval_presets = "flexeval.scripts.flexeval_presets:main"
 
 [tool.poetry.dependencies]
 python = ">=3.10,<3.13"  # set version bound because we get "Unable to find installation candidates" with torch 2.5.1
-transformers = {extras = ["ja", "sentencepiece", "torch"], version = "^4.34.1"}
+transformers = {extras = ["ja", "sentencepiece", "torch"], version = ">=5.5.3,<6"}
 datasets = "^2.14.6"
-evaluate = "^0.4.1"
-peft = "^0.10.0"
+evaluate = "^0.4.6"  # 0.4.1-0.4.5 call huggingface_hub's removed `HfFolder` API and break with huggingface_hub>=1.0
+peft = "^0.19.1"
 jsonargparse = {extras = ["jsonnet"], version = "^4.26.1"}
 jinja2 = "^3.1.2"
 fuzzywuzzy = "^0.18.0"
@@ -27,9 +27,9 @@ python-levenshtein = "^0.23.0"
 rouge = "^1.0.1"
 sacrebleu = {extras = ["ja"], version = "^2.4.1"}
 jiwer = "^3.0.4"
-openai = "^1.52.2"
+openai = ">=2.0.0,<3.0.0"
 google-api-python-client = "^2.131.0"
-vllm = {version = "0.17.1", optional = true }
+vllm = {version = "0.24.0", optional = true }
 loguru = "^0.7.2"
 wandb = {version = "^0.17.2", optional = true}
 pyarrow = "16.1.0"  # set the version because we get "Unable to find installation candidates" with 17.0.0