transformers 5.0.0rc2__py3-none-any.whl → 5.1.0__py3-none-any.whl

transformers/backbone_utils.py

@@ -0,0 +1,326 @@
+# Copyright 2026 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Collection of utils to be used by backbones and their components."""
+
+import enum
+import inspect
+
+from huggingface_hub import repo_exists
+
+from .utils import logging
+
+
+logger = logging.get_logger(__name__)
+
+
+class BackboneType(enum.Enum):
+    TIMM = "timm"
+    TRANSFORMERS = "transformers"
+
+
+class BackboneConfigMixin:
+    """
+    A Mixin to support handling the `out_features` and `out_indices` attributes for the backbone configurations.
+    """
+
+    def set_output_features_output_indices(
+        self,
+        out_features: list | None,
+        out_indices: list | None,
+    ):
+        """
+        Sets output indices and features to new values and aligns them with the given `stage_names`.
+        If one of the inputs is not given, find the corresponding `out_features` or `out_indices`
+        for the given `stage_names`.
+
+        Args:
+            out_features (`list[str]`, *optional*):
+                The names of the features for the backbone to output. Defaults to `config._out_features` if not provided.
+            out_indices (`list[int]` or `tuple[int]`, *optional*):
+                The indices of the features for the backbone to output. Defaults to `config._out_indices` if not provided.
+        """
+        self._out_features = out_features
+        self._out_indices = list(out_indices) if isinstance(out_indices, tuple) else out_indices
+
+        # First verify that the out_features and out_indices are valid
+        self.verify_out_features_out_indices()
+
+        # Align output features with indices
+        out_features, out_indices = self._out_features, self._out_indices
+        if out_indices is None and out_features is None:
+            out_indices = [len(self.stage_names) - 1]
+            out_features = [self.stage_names[-1]]
+        elif out_indices is None and out_features is not None:
+            out_indices = [self.stage_names.index(layer) for layer in out_features]
+        elif out_features is None and out_indices is not None:
+            out_features = [self.stage_names[idx] for idx in out_indices]
+
+        # Update values and verify that the aligned out_features and out_indices are valid
+        self._out_features, self._out_indices = out_features, out_indices
+        self.verify_out_features_out_indices()
+
+    def verify_out_features_out_indices(self):
+        """
+        Verify that out_indices and out_features are valid for the given stage_names.
+        """
+        if self.stage_names is None:
+            raise ValueError("Stage_names must be set for transformers backbones")
+
+        if self._out_features is not None:
+            if not isinstance(self._out_features, (list,)):
+                raise ValueError(f"out_features must be a list got {type(self._out_features)}")
+            if any(feat not in self.stage_names for feat in self._out_features):
+                raise ValueError(
+                    f"out_features must be a subset of stage_names: {self.stage_names} got {self._out_features}"
+                )
+            if len(self._out_features) != len(set(self._out_features)):
+                raise ValueError(f"out_features must not contain any duplicates, got {self._out_features}")
+            if self._out_features != (
+                sorted_feats := [feat for feat in self.stage_names if feat in self._out_features]
+            ):
+                raise ValueError(
+                    f"out_features must be in the same order as stage_names, expected {sorted_feats} got {self._out_features}"
+                )
+
+        if self._out_indices is not None:
+            if not isinstance(self._out_indices, list):
+                raise ValueError(f"out_indices must be a list, got {type(self._out_indices)}")
+            # Convert negative indices to their positive equivalent: [-1,] -> [len(stage_names) - 1,]
+            positive_indices = tuple(idx % len(self.stage_names) if idx < 0 else idx for idx in self._out_indices)
+            if any(idx for idx in positive_indices if idx not in range(len(self.stage_names))):
+                raise ValueError(
+                    f"out_indices must be valid indices for stage_names {self.stage_names}, got {self._out_indices}"
+                )
+            if len(positive_indices) != len(set(positive_indices)):
+                msg = f"out_indices must not contain any duplicates, got {self._out_indices}"
+                msg += f"(equivalent to {positive_indices}))" if positive_indices != self._out_indices else ""
+                raise ValueError(msg)
+            if positive_indices != tuple(sorted(positive_indices)):
+                sorted_negative = [
+                    idx for _, idx in sorted(zip(positive_indices, self._out_indices), key=lambda x: x[0])
+                ]
+                raise ValueError(
+                    f"out_indices must be in the same order as stage_names, expected {sorted_negative} got {self._out_indices}"
+                )
+
+        if self._out_features is not None and self._out_indices is not None:
+            if len(self._out_features) != len(self._out_indices):
+                raise ValueError("out_features and out_indices should have the same length if both are set")
+            if self._out_features != [self.stage_names[idx] for idx in self._out_indices]:
+                raise ValueError("out_features and out_indices should correspond to the same stages if both are set")
+
+    @property
+    def out_features(self):
+        return self._out_features
+
+    @out_features.setter
+    def out_features(self, out_features: list[str]):
+        """
+        Set the out_features attribute. This will also update the out_indices attribute to match the new out_features.
+        """
+        self.set_output_features_output_indices(out_features=out_features, out_indices=None)
+
+    @property
+    def out_indices(self):
+        return self._out_indices
+
+    @out_indices.setter
+    def out_indices(self, out_indices: tuple[int, ...] | list[int]):
+        """
+        Set the out_indices attribute. This will also update the out_features attribute to match the new out_indices.
+        """
+        out_indices = list(out_indices) if out_indices is not None else out_indices
+        self.set_output_features_output_indices(out_features=None, out_indices=out_indices)
+
+    def to_dict(self):
+        """
+        Serializes this instance to a Python dictionary. Override the default `to_dict()` from `PreTrainedConfig` to
+        include the `out_features` and `out_indices` attributes.
+        """
+        output = super().to_dict()
+        output["out_features"] = output.pop("_out_features", None)
+        output["out_indices"] = output.pop("_out_indices", None)
+        return output
+
+
+class BackboneMixin:
+    backbone_type: BackboneType | None = None
+
+    # Attribute to indicate if the backbone has attention and can return attention outputs.
+    # Should be set to `False` for conv-based models to be able to run `forward_with_filtered_kwargs`
+    has_attentions: bool = True
+
+    def __init__(self, *args, **kwargs) -> None:
+        """
+        Method to initialize the backbone. This method is called by the constructor of the base class after the
+        pretrained model weights have been loaded.
+        """
+        super().__init__(*args, **kwargs)
+        timm_backbone = kwargs.pop("timm_backbone", None)
+        if timm_backbone is not None:
+            self.backbone_type = BackboneType.TIMM
+        else:
+            self.backbone_type = BackboneType.TRANSFORMERS
+
+        if self.backbone_type == BackboneType.TIMM:
+            self._init_timm_backbone(backbone=timm_backbone)
+        elif self.backbone_type == BackboneType.TRANSFORMERS:
+            self._init_transformers_backbone()
+        else:
+            raise ValueError(f"backbone_type {self.backbone_type} not supported.")
+
+    def _init_timm_backbone(self, backbone) -> None:
+        """
+        Initialize the backbone model from timm. The backbone must already be loaded to backbone
+        """
+
+        # These will disagree with the defaults for the transformers models e.g. for resnet50
+        # the transformer model has out_features = ['stem', 'stage1', 'stage2', 'stage3', 'stage4']
+        # the timm model has out_features = ['act', 'layer1', 'layer2', 'layer3', 'layer4']
+        self.stage_names = [stage["module"] for stage in backbone.feature_info.info]
+        self.num_features = [stage["num_chs"] for stage in backbone.feature_info.info]
+
+        self.config._out_indices = list(backbone.feature_info.out_indices)
+        self.config._out_features = backbone.feature_info.module_name()
+        self.config.stage_names = self.stage_names
+
+        # We verify the out indices and out features are valid
+        self.config.verify_out_features_out_indices()
+
+    def _init_transformers_backbone(self) -> None:
+        self.stage_names = self.config.stage_names
+        self.config.verify_out_features_out_indices()
+        # Number of channels for each stage. This is set in the transformer backbone model init
+        self.num_features = None
+
+    @property
+    def out_features(self):
+        return self.config._out_features
+
+    @out_features.setter
+    def out_features(self, out_features: list[str]):
+        """
+        Set the out_features attribute. This will also update the out_indices attribute to match the new out_features.
+        """
+        self.config.out_features = out_features
+
+    @property
+    def out_indices(self):
+        return self.config._out_indices
+
+    @out_indices.setter
+    def out_indices(self, out_indices: tuple[int] | list[int]):
+        """
+        Set the out_indices attribute. This will also update the out_features attribute to match the new out_indices.
+        """
+        self.config.out_indices = out_indices
+
+    @property
+    def out_feature_channels(self):
+        # the current backbones will output the number of channels for each stage
+        # even if that stage is not in the out_features list.
+        return {stage: self.num_features[i] for i, stage in enumerate(self.stage_names)}
+
+    @property
+    def channels(self):
+        return [self.out_feature_channels[name] for name in self.out_features]
+
+    def forward_with_filtered_kwargs(self, *args, **kwargs):
+        if not self.has_attentions:
+            kwargs.pop("output_attentions", None)
+        if self.backbone_type == BackboneType.TIMM:
+            signature = dict(inspect.signature(self.forward).parameters)
+            kwargs = {k: v for k, v in kwargs.items() if k in signature}
+        return self(*args, **kwargs)
+
+    def forward(
+        self,
+        pixel_values,
+        output_hidden_states: bool | None = None,
+        output_attentions: bool | None = None,
+        return_dict: bool | None = None,
+    ):
+        raise NotImplementedError("This method should be implemented by the derived class.")
+
+
+def consolidate_backbone_kwargs_to_config(
+    backbone_config,
+    default_backbone: str | None = None,
+    default_config_type: str | None = None,
+    default_config_kwargs: dict | None = None,
+    timm_default_kwargs: dict | None = None,
+    **kwargs,
+):
+    # Lazy import to avoid circular import issues. Can be imported properly
+    # after deleting ref to `BackboneMixin` in `utils/backbone_utils.py`
+    from .configuration_utils import PreTrainedConfig
+    from .models.auto import CONFIG_MAPPING
+
+    use_timm_backbone = kwargs.pop("use_timm_backbone", True)
+    backbone_kwargs = kwargs.pop("backbone_kwargs", {})
+    backbone = kwargs.pop("backbone") if kwargs.get("backbone") is not None else default_backbone
+    kwargs.pop("use_pretrained_backbone", None)
+
+    # Init timm backbone with hardcoded values for BC. If everything is set to `None` and there is
+    # a default timm config, we use it to init the backbone.
+    if (
+        timm_default_kwargs is not None
+        and use_timm_backbone
+        and backbone is not None
+        and backbone_config is None
+        and not backbone_kwargs
+    ):
+        backbone_config = CONFIG_MAPPING["timm_backbone"](backbone=backbone, **timm_default_kwargs)
+    elif backbone is not None and backbone_config is None:
+        if repo_exists(backbone):
+            config_dict, _ = PreTrainedConfig.get_config_dict(backbone)
+            config_class = CONFIG_MAPPING[config_dict["model_type"]]
+            config_dict.update(backbone_kwargs)
+            backbone_config = config_class(**config_dict)
+        else:
+            backbone_config = CONFIG_MAPPING["timm_backbone"](backbone=backbone, **backbone_kwargs)
+    elif backbone_config is None and default_config_type is not None:
+        logger.info(
+            f"`backbone_config` is `None`. Initializing the config with the default `{default_config_type}` vision config."
+        )
+        default_config_kwargs = default_config_kwargs or {}
+        backbone_config = CONFIG_MAPPING[default_config_type](**default_config_kwargs)
+    elif isinstance(backbone_config, dict):
+        backbone_model_type = backbone_config.get("model_type")
+        config_class = CONFIG_MAPPING[backbone_model_type]
+        backbone_config = config_class.from_dict(backbone_config)
+
+    return backbone_config, kwargs
+
+
+def load_backbone(config):
+    """
+    Loads the backbone model from a config object.
+
+    If the config is from the backbone model itself, then we return a backbone model with randomly initialized
+    weights.
+
+    If the config is from the parent model of the backbone model itself, then we load the pretrained backbone weights
+    if specified.
+    """
+    from transformers import AutoBackbone
+
+    backbone_config = getattr(config, "backbone_config", None)
+
+    if backbone_config is None:
+        backbone = AutoBackbone.from_config(config=config)
+    else:
+        backbone = AutoBackbone.from_config(config=backbone_config)
+    return backbone

transformers/cache_utils.py

@@ -1,12 +1,13 @@
 from abc import ABC, abstractmethod
 from collections.abc import Iterable
-from typing import Any, Optional
+from typing import Any
 
 import torch
 
 from .configuration_utils import PreTrainedConfig
 from .utils import (
     is_hqq_available,
+    is_optimum_quanto_available,
     is_quanto_greater,
     is_torch_greater_or_equal,
     is_torchdynamo_compiling,
@@ -29,8 +30,8 @@ class CacheLayerMixin(ABC):
     is_compileable = False
 
     def __init__(self):
-        self.keys: Optional[torch.Tensor] = None
-        self.values: Optional[torch.Tensor] = None
+        self.keys: torch.Tensor | None = None
+        self.values: torch.Tensor | None = None
         self.is_initialized = False
 
     def __repr__(self):
@@ -41,7 +42,7 @@ class CacheLayerMixin(ABC):
 
     @abstractmethod
     def update(
-        self, key_states: torch.Tensor, value_states: torch.Tensor, cache_kwargs: Optional[dict[str, Any]] = None
+        self, key_states: torch.Tensor, value_states: torch.Tensor, cache_kwargs: dict[str, Any] | None = None
     ) -> tuple[torch.Tensor, torch.Tensor]: ...
 
     @abstractmethod
@@ -99,7 +100,7 @@ class DynamicLayer(CacheLayerMixin):
         self,
         key_states: torch.Tensor,
         value_states: torch.Tensor,
-        cache_kwargs: Optional[dict[str, Any]] = None,
+        cache_kwargs: dict[str, Any] | None = None,
     ) -> tuple[torch.Tensor, torch.Tensor]:
         """
         Update the key and value caches in-place, and return the necessary keys and value states.
@@ -186,7 +187,7 @@ class DynamicSlidingWindowLayer(DynamicLayer):
         self,
         key_states: torch.Tensor,
         value_states: torch.Tensor,
-        cache_kwargs: Optional[dict[str, Any]] = None,
+        cache_kwargs: dict[str, Any] | None = None,
     ) -> tuple[torch.Tensor, torch.Tensor]:
         """
         Update the key and value caches in-place, and return the necessary keys and value states.
@@ -310,7 +311,7 @@ class StaticLayer(CacheLayerMixin):
         self,
         key_states: torch.Tensor,
         value_states: torch.Tensor,
-        cache_kwargs: Optional[dict[str, Any]] = None,
+        cache_kwargs: dict[str, Any] | None = None,
     ) -> tuple[torch.Tensor, torch.Tensor]:
         """
         Update the key and value caches in-place, and return the necessary keys and value states.
@@ -385,7 +386,7 @@ class StaticSlidingWindowLayer(StaticLayer):
         self,
         key_states: torch.Tensor,
         value_states: torch.Tensor,
-        cache_kwargs: Optional[dict[str, Any]] = None,
+        cache_kwargs: dict[str, Any] | None = None,
     ) -> tuple[torch.Tensor, torch.Tensor]:
         """
         Update the key and value caches in-place, and return the necessary keys and value states.
@@ -518,7 +519,7 @@ class QuantizedLayer(DynamicLayer):
         self,
         key_states: torch.Tensor,
         value_states: torch.Tensor,
-        cache_kwargs: Optional[dict[str, Any]] = None,
+        cache_kwargs: dict[str, Any] | None = None,
     ) -> tuple[torch.Tensor, torch.Tensor]:
         """
         Update the key and value caches in-place, and return the necessary keys and value states.
@@ -584,11 +585,16 @@ class QuantoQuantizedLayer(QuantizedLayer):
         )
 
         # We need to import quanto here to avoid circular imports due to optimum/quanto/models/transformers_models.py
-        if is_quanto_greater("0.2.5", accept_dev=True):
+        if not is_optimum_quanto_available():
+            raise ImportError(
+                "You need to install optimum-quanto in order to use KV cache quantization with optimum-quanto "
+                "backend. Please install it via  with `pip install optimum-quanto`"
+            )
+        elif is_quanto_greater("0.2.5", accept_dev=True):
             from optimum.quanto import MaxOptimizer, qint2, qint4
         else:
             raise ImportError(
-                "You need optimum-quanto package version to be greater or equal than 0.2.5 to use `QuantoQuantizedCache`. "
+                "You need optimum-quanto package version to be greater or equal than 0.2.5 to use `QuantoQuantizedLayer`. "
             )
 
         if self.nbits not in [2, 4]:
@@ -634,7 +640,10 @@ class HQQQuantizedLayer(QuantizedLayer):
         )
 
         if not is_hqq_available():
-            raise ImportError("You need to install `hqq` to use `HQQQuantizedLayer`")
+            raise ImportError(
+                "You need to install `HQQ` in order to use KV cache quantization with HQQ backend. "
+                "Please install it via  with `pip install hqq`"
+            )
 
         if self.nbits not in [1, 2, 3, 4, 8]:
             raise ValueError(
@@ -692,8 +701,8 @@ class Cache:
 
     def __init__(
         self,
-        layers: Optional[list[CacheLayerMixin]] = None,
-        layer_class_to_replicate: Optional[type[CacheLayerMixin]] = None,
+        layers: list[CacheLayerMixin] | None = None,
+        layer_class_to_replicate: type[CacheLayerMixin] | None = None,
         offloading: bool = False,
         offload_only_non_sliding: bool = True,
     ):
@@ -751,7 +760,7 @@ class Cache:
         key_states: torch.Tensor,
         value_states: torch.Tensor,
         layer_idx: int,
-        cache_kwargs: Optional[dict[str, Any]] = None,
+        cache_kwargs: dict[str, Any] | None = None,
     ) -> tuple[torch.Tensor, torch.Tensor]:
         """
         Updates the cache with the new `key_states` and `value_states` for the layer `layer_idx`.
@@ -939,8 +948,8 @@ class DynamicCache(Cache):
 
     def __init__(
         self,
-        ddp_cache_data: Optional[Iterable[tuple[Optional[torch.Tensor], ...]]] = None,
-        config: Optional[PreTrainedConfig] = None,
+        ddp_cache_data: Iterable[tuple[torch.Tensor | None, ...]] | None = None,
+        config: PreTrainedConfig | None = None,
         offloading: bool = False,
         offload_only_non_sliding: bool = False,
     ):
@@ -1293,112 +1302,3 @@ class EncoderDecoderCache(Cache):
     @property
     def is_compileable(self) -> bool:
         return self.self_attention_cache.is_compileable
-
-
-### Deprecated classes
-
-
-class SlidingWindowLayer(StaticSlidingWindowLayer):
-    def __init__(self, max_cache_len: int, sliding_window: int):
-        logger.warning_once(
-            "`SlidingWindowLayer` is deprecated and will be removed in version v4.59 "
-            "Use `StaticSlidingWindowLayer` instead, which is a better name for it."
-        )
-        super().__init__(max_cache_len, sliding_window)
-
-
-class ChunkedSlidingLayer(StaticSlidingWindowLayer):
-    def __init__(self, max_cache_len: int, sliding_window: int):
-        logger.warning_once(
-            "`ChunkedSlidingLayer` is deprecated and will be removed in version v4.59 "
-            "Use `StaticSlidingWindowLayer` instead, which has the exact same functionalities."
-        )
-        super().__init__(max_cache_len, sliding_window)
-
-
-class OffloadedCache(DynamicCache):
-    def __init__(self) -> None:
-        logger.warning_once(
-            "`OffloadedCache` is deprecated and will be removed in version v4.59 "
-            "Use `DynamicCache(offloading=True)` instead"
-        )
-        super().__init__(offloading=True)
-
-
-class OffloadedStaticCache(StaticCache):
-    def __init__(self, config: PreTrainedConfig, max_cache_len: int, *args, **kwargs):
-        logger.warning_once(
-            "`OffloadedStaticCache` is deprecated and will be removed in version v4.59 "
-            "Use `StaticCache(..., offloading=True)` instead"
-        )
-        super().__init__(config=config, max_cache_len=max_cache_len, offloading=True)
-
-
-class SlidingWindowCache(StaticCache):
-    def __init__(self, config: PreTrainedConfig, max_cache_len: int, *args, **kwargs):
-        logger.warning_once(
-            "`SlidingWindowCache` is deprecated and will be removed in version v4.59 "
-            "Use `StaticCache(...)` instead which will correctly infer the type of each layer."
-        )
-        super().__init__(config=config, max_cache_len=max_cache_len)
-
-
-class HybridCache(StaticCache):
-    def __init__(self, config: PreTrainedConfig, max_cache_len: int, *args, **kwargs):
-        logger.warning_once(
-            "`HybridCache` is deprecated and will be removed in version v4.59 "
-            "Use `StaticCache(...)` instead which will correctly infer the type of each layer."
-        )
-        super().__init__(config=config, max_cache_len=max_cache_len)
-
-
-class HybridChunkedCache(StaticCache):
-    def __init__(self, config: PreTrainedConfig, max_cache_len: int, *args, **kwargs):
-        logger.warning_once(
-            "`HybridChunkedCache` is deprecated and will be removed in version v4.59 "
-            "Use `StaticCache(...)` instead which will correctly infer the type of each layer."
-        )
-        super().__init__(config=config, max_cache_len=max_cache_len)
-
-
-class OffloadedHybridCache(StaticCache):
-    def __init__(self, config: PreTrainedConfig, max_cache_len: int, *args, **kwargs):
-        logger.warning_once(
-            "`OffloadedHybridCache` is deprecated and will be removed in version v4.59 "
-            "Use `StaticCache(..., offload=True)` instead which will correctly infer the type of each layer."
-        )
-        super().__init__(config=config, max_cache_len=max_cache_len, offloading=True)
-
-
-class QuantoQuantizedCache(QuantizedCache):
-    def __init__(
-        self,
-        config: PreTrainedConfig,
-        nbits: int = 4,
-        axis_key: int = 0,
-        axis_value: int = 0,
-        q_group_size: int = 64,
-        residual_length: int = 128,
-    ):
-        logger.warning_once(
-            "`QuantoQuantizedCache` is deprecated and will be removed in version v4.59 "
-            "Use `QuantizedCache(backend='quanto', ...)` instead."
-        )
-        super().__init__("quanto", config, nbits, axis_key, axis_value, q_group_size, residual_length)
-
-
-class HQQQuantizedCache(QuantizedCache):
-    def __init__(
-        self,
-        config: PreTrainedConfig,
-        nbits: int = 4,
-        axis_key: int = 0,
-        axis_value: int = 0,
-        q_group_size: int = 64,
-        residual_length: int = 128,
-    ):
-        logger.warning_once(
-            "`HQQQuantizedCache` is deprecated and will be removed in version v4.59 "
-            "Use `QuantizedCache(backend='hqq', ...)` instead."
-        )
-        super().__init__("hqq", config, nbits, axis_key, axis_value, q_group_size, residual_length)

transformers/cli/chat.py

@@ -19,7 +19,7 @@ import re
 import string
 import time
 from collections.abc import AsyncIterator
-from typing import Annotated, Any, Optional
+from typing import Annotated, Any
 from urllib.parse import urljoin, urlparse
 
 import httpx
@@ -206,7 +206,7 @@ class Chat:
         self,
         model_id: Annotated[str, typer.Argument(help="ID of the model to use (e.g. 'HuggingFaceTB/SmolLM3-3B').")],
         base_url: Annotated[
-            Optional[str], typer.Argument(help="Base url to connect to (e.g. http://localhost:8000/v1).")
+            str | None, typer.Argument(help="Base url to connect to (e.g. http://localhost:8000/v1).")
         ] = f"http://{DEFAULT_HTTP_ENDPOINT['hostname']}:{DEFAULT_HTTP_ENDPOINT['port']}",
         generate_flags: Annotated[
             list[str] | None,
@@ -374,7 +374,7 @@ class Chat:
         config = self.config
 
         async with AsyncInferenceClient(base_url=self.base_url) as client:
-            pending_user_input: Optional[str] = None
+            pending_user_input: str | None = None
             while True:
                 try:
                     if pending_user_input is not None:

transformers/cli/serve.py

@@ -387,7 +387,7 @@ class Serve:
             ),
         ] = None,
         quantization: Annotated[
-            Optional[str],
+            str | None,
             typer.Option(help="Which quantization method to use. choices: 'bnb-4bit', 'bnb-8bit'"),
         ] = None,
         host: Annotated[str, typer.Option(help="Interface the server will listen to.")] = "localhost",
@@ -839,6 +839,17 @@ class Serve:
                 for result in self.running_continuous_batching_manager.request_id_iter(request_id):
                     n_tokens_generated += 1
 
+                    # Always yield the token content (even for the final FINISHED token)
+                    if result.generated_tokens:
+                        token_id = result.generated_tokens[-1]
+                        yield self.build_chat_completion_chunk(
+                            request_id=request_id,
+                            content=token_id,
+                            model=model_id_and_revision,
+                            decode_stream=decode_stream,
+                            tokenizer=tokenizer,
+                        )
+
                     if result.status == RequestStatus.FINISHED:
                         generated_all_tokens = n_tokens_generated >= generation_config.max_new_tokens
 
@@ -855,14 +866,6 @@ class Serve:
                             model=model_id_and_revision,
                         )
                         break
-                    else:
-                        yield self.build_chat_completion_chunk(
-                            request_id=request_id,
-                            content=result.generated_tokens[-1],
-                            model=model_id_and_revision,
-                            decode_stream=decode_stream,
-                            tokenizer=tokenizer,
-                        )
 
             except Exception as e:
                 logger.error(str(e))
@@ -1728,7 +1731,7 @@ class Serve:
         self.last_messages = messages
         return req_continues_last_messages
 
-    def get_quantization_config(self) -> Optional[BitsAndBytesConfig]:
+    def get_quantization_config(self) -> BitsAndBytesConfig | None:
         """
         Returns the quantization config for the given CLI arguments.
 

transformers/cli/transformers.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 """Transformers CLI."""
 
-from huggingface_hub import typer_factory
+from huggingface_hub import check_cli_update, typer_factory
 
 from transformers.cli.add_fast_image_processor import add_fast_image_processor
 from transformers.cli.add_new_model_like import add_new_model_like
@@ -35,6 +35,7 @@ app.command()(version)
 
 
 def main():
+    check_cli_update("transformers")
     app()