transformers 5.0.0rc3__py3-none-any.whl → 5.1.0__py3-none-any.whl

transformers/utils/generic.py

@@ -210,6 +210,31 @@ def is_mlx_array(x):
     return False if not _is_mlx_available else _is_mlx(x)
 
 
+def is_flash_attention_requested(config=None, requested_attention_implementation: str | None = None):
+    """
+    Checks whether some flavor of flash attention is requested or not.
+
+    This is checked against one of the two arguments, i.e. either the `config` or the directly passed value
+    `requested_attention_implementation`. Otherwise, an error will be raised (ambiguity).
+
+    The different versions of flash attention are usually
+    - Implementations based on the original flash attention repo: https://github.com/Dao-AILab/flash-attention
+    - Kernels implementations such as: https://huggingface.co/kernels-community/vllm-flash-attn3
+    """
+    if config is not None and requested_attention_implementation is not None:
+        raise ValueError(
+            "Requested attention implementation is ambiguous: "
+            "Please pass either the config or the name of the attention implementation, not both."
+        )
+
+    if config is not None:
+        checked_attention_implementation = config._attn_implementation
+    else:
+        checked_attention_implementation = requested_attention_implementation
+
+    return "flash" in checked_attention_implementation
+
+
 def to_py_obj(obj):
     """
     Convert a PyTorch tensor, Numpy array or python list to a python list.
@@ -219,12 +244,11 @@ def to_py_obj(obj):
     elif isinstance(obj, (dict, UserDict)):
         return {k: to_py_obj(v) for k, v in obj.items()}
     elif isinstance(obj, (list, tuple)):
-        try:
-            arr = np.array(obj)
-            if np.issubdtype(arr.dtype, np.integer) or np.issubdtype(arr.dtype, np.floating):
-                return arr.tolist()
-        except Exception:
-            pass
+        # Only convert directly if all elements are numeric scalars
+        if all(isinstance(x, (int, float, np.number)) for x in obj):
+            return list(obj)
+
+        # Otherwise recurse element-wise
         return [to_py_obj(o) for o in obj]
 
     framework_to_py_obj = {
@@ -850,34 +874,46 @@ def check_model_inputs(func=None, *, tie_last_hidden_states=True):
     def wrapped_fn(func):
         @wraps(func)
         def wrapper(self, *args, **kwargs):
-            use_cache_arg_index = None
-            if "use_cache" in func.__code__.co_varnames:
-                use_cache_arg_index = func.__code__.co_varnames.index("use_cache") - 1  # -1 for self
-
-            if (
-                use_cache_arg_index is not None
-                and len(args) > use_cache_arg_index
-                and args[use_cache_arg_index] is not None
-            ):
-                use_cache = args[use_cache_arg_index]
-            elif kwargs.get("use_cache") is not None:
-                use_cache = kwargs["use_cache"]
-            else:
-                use_cache = getattr(self.config, "use_cache", None)
-
-            if use_cache is not None:
-                if getattr(self, "gradient_checkpointing", False) and self.training and use_cache:
-                    logger.warning_once(
-                        "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`."
-                    )
-                    use_cache = False
-
-                if use_cache_arg_index is not None and len(args) > use_cache_arg_index:
-                    args = list(args)
-                    args[use_cache_arg_index] = use_cache
-                    args = tuple(args)
+            args_with_config_defaults = [
+                "use_cache",
+                "vision_feature_layer",
+                "vision_feature_select_strategy",
+                "vision_aspect_ratio",
+            ]
+            for arg_name in args_with_config_defaults:
+                arg_index = None
+                if arg_name in func.__code__.co_varnames:
+                    arg_index = func.__code__.co_varnames.index(arg_name) - 1  # -1 for self
+
+                if arg_index is not None and len(args) > arg_index and args[arg_index] is not None:
+                    arg_value = args[arg_index]
+                elif kwargs.get(arg_name) is not None:
+                    arg_value = kwargs[arg_name]
                 else:
-                    kwargs["use_cache"] = use_cache
+                    arg_value = getattr(self.config, arg_name, None)
+
+                if arg_value is not None:
+                    # Arg-specific handling
+                    if arg_name == "use_cache":
+                        if getattr(self, "gradient_checkpointing", False) and self.training and arg_value:
+                            logger.warning_once(
+                                "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`."
+                            )
+                            arg_value = False
+                    elif arg_name == "vision_feature_select_strategy":
+                        valid_strategies = ["default", "full"]
+                        if arg_value not in valid_strategies:
+                            raise ValueError(
+                                f"`Unexpected select feature strategy: {arg_value}. "
+                                f"Please select from {valid_strategies}."
+                            )
+
+                    if arg_index is not None and len(args) > arg_index:
+                        args = list(args)
+                        args[arg_index] = arg_value
+                        args = tuple(args)
+                    else:
+                        kwargs[arg_name] = arg_value
 
             return_dict = kwargs.pop("return_dict", None)
             if return_dict is None:

transformers/utils/import_utils.py

@@ -112,9 +112,9 @@ def is_torch_available() -> bool:
     try:
         is_available, torch_version = _is_package_available("torch", return_version=True)
         parsed_version = version.parse(torch_version)
-        if is_available and parsed_version < version.parse("2.2.0"):
-            logger.warning_once(f"Disabling PyTorch because PyTorch >= 2.2 is required but found {torch_version}")
-        return is_available and version.parse(torch_version) >= version.parse("2.2.0")
+        if is_available and parsed_version < version.parse("2.4.0"):
+            logger.warning_once(f"Disabling PyTorch because PyTorch >= 2.4 is required but found {torch_version}")
+        return is_available and version.parse(torch_version) >= version.parse("2.4.0")
     except packaging.version.InvalidVersion:
         return False
 
@@ -241,25 +241,21 @@ def is_torch_npu_available(check_device=False) -> bool:
 @lru_cache
 def is_torch_xpu_available(check_device: bool = False) -> bool:
     """
-    Checks if XPU acceleration is available either via native PyTorch (>=2.6),
-    `intel_extension_for_pytorch` or via stock PyTorch (>=2.4) and potentially
-    if a XPU is in the environment.
+    Checks if XPU acceleration is available via stock PyTorch (>=2.6) and
+    potentially if a XPU is in the environment.
     """
     if not is_torch_available():
         return False
 
     torch_version = version.parse(get_torch_version())
     if torch_version.major == 2 and torch_version.minor < 6:
-        if is_ipex_available():
-            import intel_extension_for_pytorch  # noqa: F401
-        elif torch_version.major == 2 and torch_version.minor < 4:
-            return False
+        return False
 
     import torch
 
     if check_device:
         try:
-            # Will raise a RuntimeError if no XPU  is found
+            # Will raise a RuntimeError if no XPU is found
             _ = torch.xpu.device_count()
             return torch.xpu.is_available()
         except RuntimeError:
@@ -750,11 +746,6 @@ def is_flute_available() -> bool:
     return is_available and version.parse(flute_version) >= version.parse("0.4.1")
 
 
-@lru_cache
-def is_ftfy_available() -> bool:
-    return _is_package_available("ftfy")
-
-
 @lru_cache
 def is_g2p_en_available() -> bool:
     return _is_package_available("g2p_en")
@@ -853,29 +844,6 @@ def is_ninja_available() -> bool:
         return True
 
 
-@lru_cache
-def is_ipex_available(min_version: str = "") -> bool:
-    def get_major_and_minor_from_version(full_version):
-        return str(version.parse(full_version).major) + "." + str(version.parse(full_version).minor)
-
-    ipex_available, ipex_version = _is_package_available("intel_extension_for_pytorch", return_version=True)
-
-    if not is_torch_available() or not ipex_available:
-        return False
-
-    torch_major_and_minor = get_major_and_minor_from_version(get_torch_version())
-    ipex_major_and_minor = get_major_and_minor_from_version(ipex_version)
-    if torch_major_and_minor != ipex_major_and_minor:
-        logger.warning_once(
-            f"Intel Extension for PyTorch {ipex_major_and_minor} needs to work with PyTorch {ipex_major_and_minor}.*,"
-            f" but PyTorch {get_torch_version()} is found. Please switch to the matching version and run again."
-        )
-        return False
-    if min_version:
-        return version.parse(ipex_version) >= version.parse(min_version)
-    return True
-
-
 @lru_cache
 def is_bitsandbytes_available(min_version: str = BITSANDBYTES_MIN_VERSION) -> bool:
     is_available, bitsandbytes_version = _is_package_available("bitsandbytes", return_version=True)
@@ -1166,11 +1134,6 @@ def is_uroman_available() -> bool:
     return _is_package_available("uroman")
 
 
-@lru_cache
-def is_ccl_available() -> bool:
-    return _is_package_available("torch_ccl") or _is_package_available("oneccl_bindings_for_pytorch")
-
-
 @lru_cache
 def is_sudachi_available() -> bool:
     return _is_package_available("sudachipy")
@@ -1314,15 +1277,10 @@ def is_torchdynamo_exporting() -> bool:
 
         return torch.compiler.is_exporting()
     except Exception:
-        try:
-            import torch._dynamo as dynamo
-
-            return dynamo.is_exporting()
-        except Exception:
-            return False
+        return False
 
 
-def is_torch_fx_proxy(x):
+def is_torch_fx_proxy(x) -> bool:
     try:
         import torch.fx
 
@@ -1331,6 +1289,15 @@ def is_torch_fx_proxy(x):
         return False
 
 
+def is_fake_tensor(x) -> bool:
+    try:
+        import torch
+
+        return isinstance(x, torch._subclasses.FakeTensor)
+    except Exception:
+        return False
+
+
 def is_jax_jitting(x):
     """returns True if we are inside of `jax.jit` context, False otherwise.
 
@@ -1379,16 +1346,58 @@ def is_cuda_stream_capturing() -> bool:
 
 def is_tracing(tensor=None) -> bool:
     """Checks whether we are tracing a graph with dynamo (compile or export), torch.jit, torch.fx, jax.jit (with torchax) or
-    CUDA stream capturing"""
+    CUDA stream capturing or FakeTensor"""
+
     # Note that `is_torchdynamo_compiling` checks both compiling and exporting (the export check is stricter and
     # only checks export)
     _is_tracing = is_torchdynamo_compiling() or is_jit_tracing() or is_cuda_stream_capturing()
     if tensor is not None:
         _is_tracing |= is_torch_fx_proxy(tensor)
+        _is_tracing |= is_fake_tensor(tensor)
         _is_tracing |= is_jax_jitting(tensor)
+
     return _is_tracing
 
 
+def torch_compilable_check(cond: Any, msg: str | Callable[[], str], error_type: type[Exception] = ValueError) -> None:
+    """
+    Combines the functionalities of `torch._check`, `torch._check_with` and `torch._check_tensor_all_with` to provide a
+    unified way to perform checks that are compatible with TorchDynamo (torch.compile & torch.export).
+
+    The advantage of using `torch._check(cond, msg, error_type)` over `if cond: raise error_type(msg)` is that the former
+    works as a truthfulness hint for TorchDynamo, instead of failing with a data-dependent control flow error during compilation.
+
+    All checks using this method can be disabled in production environments by setting `TRANSFORMERS_DISABLE_TORCH_CHECK=1`.
+
+    Args:
+        cond (`bool`, `torch.Tensor` or `Callable[[], bool | torch.Tensor]`): The condition to check.
+        msg (`str` or `Callable[[], str]`): The error message to display if the condition is not met.
+        error_type (`type[Exception]`, *optional*, defaults to `ValueError`): The type of error to raise if the condition is not met.
+
+    Raises:
+        error_type: If the condition is not met.
+    """
+    if os.getenv("TRANSFORMERS_DISABLE_TORCH_CHECK", "0") == "1":
+        return
+
+    import torch
+
+    if not callable(msg):
+        # torch._check requires msg to be a callable but we want to keep the API simple for users
+        def msg_callable():
+            return msg
+    else:
+        msg_callable = msg
+
+    if callable(cond):
+        cond = cond()
+
+    if isinstance(cond, torch.Tensor):
+        torch._check_tensor_all_with(error_type, cond, msg_callable)
+    else:
+        torch._check_with(error_type, cond, msg_callable)
+
+
 @lru_cache
 def is_in_notebook() -> bool:
     try:
@@ -1596,13 +1605,6 @@ that match your environment. Please note that you may need to restart your runti
 """
 
 
-# docstyle-ignore
-FTFY_IMPORT_ERROR = """
-{0} requires the ftfy library but it was not found in your environment. Check out the instructions on the
-installation section: https://github.com/rspeer/python-ftfy/tree/master#installing and follow the ones
-that match your environment. Please note that you may need to restart your runtime after installation.
-"""
-
 LEVENSHTEIN_IMPORT_ERROR = """
 {0} requires the python-Levenshtein library but it was not found in your environment. You can install it with pip: `pip
 install python-Levenshtein`. Please note that you may need to restart your runtime after installation.
@@ -1740,13 +1742,6 @@ You can install or update it with pip: `pip install --upgrade accelerate`. Pleas
 runtime after installation.
 """
 
-# docstyle-ignore
-CCL_IMPORT_ERROR = """
-{0} requires the torch ccl library but it was not found in your environment. You can install it with pip:
-`pip install oneccl_bind_pt -f https://developer.intel.com/ipex-whl-stable`
-Please note that you may need to restart your runtime after installation.
-"""
-
 # docstyle-ignore
 ESSENTIA_IMPORT_ERROR = """
 {0} requires essentia library. But that was not found in your environment. You can install them with pip:
@@ -1809,7 +1804,6 @@ BACKENDS_MAPPING = OrderedDict(
         ("detectron2", (is_detectron2_available, DETECTRON2_IMPORT_ERROR)),
         ("essentia", (is_essentia_available, ESSENTIA_IMPORT_ERROR)),
         ("faiss", (is_faiss_available, FAISS_IMPORT_ERROR)),
-        ("ftfy", (is_ftfy_available, FTFY_IMPORT_ERROR)),
         ("g2p_en", (is_g2p_en_available, G2P_EN_IMPORT_ERROR)),
         ("pandas", (is_pandas_available, PANDAS_IMPORT_ERROR)),
         ("phonemizer", (is_phonemizer_available, PHONEMIZER_IMPORT_ERROR)),
@@ -1836,7 +1830,6 @@ BACKENDS_MAPPING = OrderedDict(
         ("vision", (is_vision_available, VISION_IMPORT_ERROR)),
         ("scipy", (is_scipy_available, SCIPY_IMPORT_ERROR)),
         ("accelerate", (is_accelerate_available, ACCELERATE_IMPORT_ERROR)),
-        ("oneccl_bind_pt", (is_ccl_available, CCL_IMPORT_ERROR)),
         ("cython", (is_cython_available, CYTHON_IMPORT_ERROR)),
         ("rjieba", (is_rjieba_available, RJIEBA_IMPORT_ERROR)),
         ("peft", (is_peft_available, PEFT_IMPORT_ERROR)),
@@ -2103,18 +2096,22 @@ class _LazyModule(ModuleType):
                                                 module = importlib.import_module(module_path)
                                                 base_tokenizer_class = getattr(module, candidate_name)
                                             except Exception:
-                                                pass
+                                                logger.debug(f"{module_path} does not have {candidate_name} defined.")
 
                                         # Fallback: try via _class_to_module
                                         if base_tokenizer_class is None and candidate_name in self._class_to_module:
                                             try:
-                                                alias_module = self._get_module(self._class_to_module[candidate_name])
+                                                alias_module_name = self._class_to_module[candidate_name]
+                                                alias_module = self._get_module(alias_module_name)
                                                 base_tokenizer_class = getattr(alias_module, candidate_name)
                                             except Exception:
-                                                continue
+                                                logger.debug(
+                                                    f"{alias_module_name} does not have {candidate_name} defined"
+                                                )
 
                                         # If we still don't have base_tokenizer_class, skip this candidate
                                         if base_tokenizer_class is None:
+                                            logger.debug(f"skipping candidate {candidate_name}")
                                             continue
 
                                         # If we got here, we have base_tokenizer_class
@@ -2125,8 +2122,8 @@ class _LazyModule(ModuleType):
                                             setattr(self, lookup_name, value)
                                         setattr(self, name, value)
                                         break
-                            except Exception:
-                                pass
+                            except Exception as e:
+                                logger.debug(f"Could not create tokenizer alias: {e}")
 
                         if value is None:
                             raise ModuleNotFoundError(
@@ -2156,8 +2153,8 @@ class _LazyModule(ModuleType):
                         setattr(self, fallback_name, value)
                         setattr(self, name, value)
                         return value
-                    except Exception:
-                        pass
+                    except Exception as e:
+                        logger.debug(f"Could not load fallback {fallback_name}: {e}")
             # V5: If a tokenizer class doesn't exist, check if it should alias to another tokenizer
             # via the converter mapping (e.g., FNetTokenizer -> AlbertTokenizer via AlbertConverter)
             value = None