transformers 5.0.0__py3-none-any.whl → 5.0.0rc0__py3-none-any.whl

transformers/models/sam3/modular_sam3.py

@@ -1,3 +1,4 @@
+# coding=utf-8
 # Copyright 2025 The Meta AI Authors and The HuggingFace Team. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,6 +14,8 @@
 # limitations under the License.
 
 
+from typing import Optional
+
 import torch
 
 from ...image_utils import (
@@ -57,7 +60,7 @@ class Sam3ImageProcessorFast(Sam2ImageProcessorFast):
     mask_size = {"height": 288, "width": 288}
 
     def post_process_semantic_segmentation(
-        self, outputs, target_sizes: list[tuple] | None = None, threshold: float = 0.5
+        self, outputs, target_sizes: Optional[list[tuple]] = None, threshold: float = 0.5
     ):
         """
         Converts the output of [`Sam3Model`] into semantic segmentation maps.
@@ -115,7 +118,9 @@ class Sam3ImageProcessorFast(Sam2ImageProcessorFast):
 
         return semantic_segmentation
 
-    def post_process_object_detection(self, outputs, threshold: float = 0.3, target_sizes: list[tuple] | None = None):
+    def post_process_object_detection(
+        self, outputs, threshold: float = 0.3, target_sizes: Optional[list[tuple]] = None
+    ):
         """
         Converts the raw output of [`Sam3Model`] into final bounding boxes in (top_left_x, top_left_y,
         bottom_right_x, bottom_right_y) format.
@@ -171,7 +176,7 @@ class Sam3ImageProcessorFast(Sam2ImageProcessorFast):
         outputs,
         threshold: float = 0.3,
         mask_threshold: float = 0.5,
-        target_sizes: list[tuple] | None = None,
+        target_sizes: Optional[list[tuple]] = None,
     ):
         """
         Converts the raw output of [`Sam3Model`] into instance segmentation predictions with bounding boxes and masks.

transformers/models/sam3/processing_sam3.py

@@ -1,3 +1,4 @@
+# coding=utf-8
 # Copyright 2025 The HuggingFace Inc. team.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,13 +17,14 @@ Processor class for SAM3.
 """
 
 from copy import deepcopy
+from typing import Optional, Union
 
 import numpy as np
 
 from ...image_utils import ImageInput
 from ...processing_utils import ProcessorMixin
 from ...tokenization_utils_base import BatchEncoding, PreTokenizedInput, TextInput
-from ...utils import TensorType, auto_docstring, is_torch_available, logging
+from ...utils import TensorType, is_torch_available, logging
 from ...utils.import_utils import requires
 
 
@@ -83,46 +85,63 @@ def box_area(boxes):
 
 
 @requires(backends=("torch",))
-@auto_docstring
 class Sam3Processor(ProcessorMixin):
-    def __init__(
-        self, image_processor, tokenizer, target_size: int | None = None, point_pad_value: int = -10, **kwargs
-    ):
-        r"""
+    r"""
+    Constructs a SAM3 processor which wraps a SAM3 image processor and bounding boxes processing into a
+    single processor.
+
+    [`Sam2Processor`] offers all the functionalities of [`Sam2ImageProcessorFast`] and [`Sam2VideoProcessor`]. See the docstring of
+    [`~Sam2ImageProcessorFast.__call__`] and [`~Sam2VideoProcessor.__call__`] for more information.
+
+    Args:
+        image_processor (`Sam2ImageProcessorFast`):
+            An instance of [`Sam2ImageProcessorFast`].
+        tokenizer ([`PreTrainedTokenizer`, `PreTrainedTokenizerFast`]):
+            An instance of [`PreTrainedTokenizer`, `PreTrainedTokenizerFast`]. The tokenizer is a required input.
         target_size (`int`, *optional*):
             The target size (target_size, target_size) to which the image will be resized.
         point_pad_value (`int`, *optional*, defaults to -10):
             The value used for padding input boxes.
-        """
+    """
+
+    def __init__(
+        self, image_processor, tokenizer, target_size: Optional[int] = None, point_pad_value: int = -10, **kwargs
+    ):
         super().__init__(image_processor, tokenizer, **kwargs)
         self.point_pad_value = point_pad_value
         self.target_size = target_size if target_size is not None else self.image_processor.size["height"]
 
-    @auto_docstring
     def __call__(
         self,
-        images: ImageInput | None = None,
-        text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput] | None = None,
-        segmentation_maps: ImageInput | None = None,
-        input_boxes: list[list[list[float]]] | torch.Tensor | None = None,
-        input_boxes_labels: list[list[list[int]]] | torch.Tensor | None = None,
-        original_sizes: list[list[float]] | torch.Tensor | None = None,
-        return_tensors: str | TensorType | None = None,
+        images: Optional[ImageInput] = None,
+        text: Optional[Union[TextInput, PreTokenizedInput, list[TextInput], list[PreTokenizedInput]]] = None,
+        segmentation_maps: Optional[ImageInput] = None,
+        input_boxes: Optional[Union[list[list[list[float]]], torch.Tensor]] = None,
+        input_boxes_labels: Optional[Union[list[list[list[int]]], torch.Tensor]] = None,
+        original_sizes: Optional[Union[list[list[float]], torch.Tensor]] = None,
+        return_tensors: Optional[Union[str, TensorType]] = None,
         **kwargs,
     ) -> BatchEncoding:
         r"""
-        images (`ImageInput`, *optional*):
-            The image(s) to process.
-        text (`str`, `list[str]`, `list[list[str]]`, *optional*):
-            The text to process.
-        segmentation_maps (`ImageInput`, *optional*):
-            The segmentation maps to process.
-        input_boxes (`list[list[list[float]]]`, `torch.Tensor`, *optional*):
-            The bounding boxes to process.
-        input_boxes_labels (`list[list[int]]`, `torch.Tensor`, *optional*):
-            The labels for the bounding boxes.
-        original_sizes (`list[list[float]]`, `torch.Tensor`, *optional*):
-            The original sizes of the images.
+        This method uses [`Sam3ImageProcessorFast.__call__`] method to prepare image(s) for the model. It also prepares bounding boxes for the model if they are provided.
+
+        Args:
+            images (`ImageInput`, *optional*):
+                The image(s) to process.
+            text (`str`, `list[str]`, `list[list[str]]`, *optional*):
+                The text to process.
+            segmentation_maps (`ImageInput`, *optional*):
+                The segmentation maps to process.
+            input_boxes (`list[list[list[float]]]`, `torch.Tensor`, *optional*):
+                The bounding boxes to process.
+            input_boxes_labels (`list[list[int]]`, `torch.Tensor`, *optional*):
+                The labels for the bounding boxes.
+            original_sizes (`list[list[float]]`, `torch.Tensor`, *optional*):
+                The original sizes of the images.
+            return_tensors (`str` or `TensorType`, *optional*):
+                The type of tensors to return.
+            **kwargs:
+                Additional keyword arguments to pass to the image processor.
 
         Returns:
             A [`BatchEncoding`] with the following fields:
@@ -444,11 +463,11 @@ class Sam3Processor(ProcessorMixin):
 
     def _validate_single_input(
         self,
-        data: torch.Tensor | np.ndarray | list,
+        data: Union[torch.Tensor, np.ndarray, list],
         expected_depth: int,
         input_name: str,
         expected_format: str,
-        expected_coord_size: int | None = None,
+        expected_coord_size: Optional[int] = None,
     ) -> list:
         """
                 Validate a single input by ensuring proper nesting and raising an error if the input is not valid.
@@ -572,15 +591,13 @@ class Sam3Processor(ProcessorMixin):
         ```python
         >>> from transformers import AutoModel, AutoProcessor
         >>> from PIL import Image
-        >>> import httpx
-        >>> from io import BytesIO
+        >>> import requests
 
         >>> model = AutoModel.from_pretrained("facebook/sam3-base")
         >>> processor = AutoProcessor.from_pretrained("facebook/sam3-base")
 
         >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
-        >>> with httpx.stream("GET", url) as response:
-        ...     image = Image.open(BytesIO(response.read()))
+        >>> image = Image.open(requests.get(url, stream=True).raw)
         >>> inputs = processor(images=image, text="cat", return_tensors="pt")
         >>> outputs = model(**inputs)
 
@@ -628,15 +645,13 @@ class Sam3Processor(ProcessorMixin):
         ```python
         >>> from transformers import AutoModel, AutoProcessor
         >>> from PIL import Image
-        >>> import httpx
-        >>> from io import BytesIO
+        >>> import requests
 
         >>> model = AutoModel.from_pretrained("facebook/sam3-base")
         >>> processor = AutoProcessor.from_pretrained("facebook/sam3-base")
 
         >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
-        >>> with httpx.stream("GET", url) as response:
-        ...     image = Image.open(BytesIO(response.read()))
+        >>> image = Image.open(requests.get(url, stream=True).raw)
         >>> inputs = processor(images=image, text="cat", return_tensors="pt")
         >>> outputs = model(**inputs)
 

transformers/models/sam3_tracker/__init__.py

@@ -1,3 +1,4 @@
+# coding=utf-8
 # Copyright 2025 the HuggingFace Team. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

transformers/models/sam3_tracker/configuration_sam3_tracker.py

@@ -4,6 +4,7 @@
 #             the file from the modular. If any change should be done, please apply the change to the
 #                          modular_sam3_tracker.py file directly. One of our CI enforces this.
 #                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# coding=utf-8
 # Copyright 2025 the HuggingFace Team. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,6 +19,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 from ...configuration_utils import PreTrainedConfig
 from ..auto import CONFIG_MAPPING, AutoConfig
 
@@ -178,7 +180,7 @@ class Sam3TrackerConfig(PreTrainedConfig):
     ... )
 
     >>> # Initializing a Sam3TrackerConfig with `"facebook/sam3_tracker.1_hiera_tiny"` style configuration
-    >>> configuration = Sam3TrackerConfig()
+    >>> configuration = Sam3Trackerconfig()
 
     >>> # Initializing a Sam3TrackerModel (with random weights) from the `"facebook/sam3_tracker.1_hiera_tiny"` style configuration
     >>> model = Sam3TrackerModel(configuration)

transformers/models/sam3_tracker/modeling_sam3_tracker.py

@@ -4,6 +4,7 @@
 #             the file from the modular. If any change should be done, please apply the change to the
 #                          modular_sam3_tracker.py file directly. One of our CI enforces this.
 #                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# coding=utf-8
 # Copyright 2025 the HuggingFace Team. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,8 +19,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 from collections.abc import Callable
 from dataclasses import dataclass
+from typing import Optional, Union
 
 import numpy as np
 import torch
@@ -31,18 +34,15 @@ from transformers.utils.generic import OutputRecorder
 
 from ... import initialization as init
 from ...activations import ACT2FN
-from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling
+from ...modeling_outputs import BaseModelOutput
 from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
 from ...processing_utils import Unpack
-from ...utils import ModelOutput, auto_docstring, can_return_tuple, logging
-from ...utils.generic import TransformersKwargs, check_model_inputs, is_flash_attention_requested
+from ...utils import ModelOutput, auto_docstring
+from ...utils.generic import TransformersKwargs, check_model_inputs
 from ..auto import AutoModel
 from .configuration_sam3_tracker import Sam3TrackerConfig, Sam3TrackerMaskDecoderConfig, Sam3TrackerPromptEncoderConfig
 
 
-logger = logging.get_logger(__name__)
-
-
 @dataclass
 @auto_docstring(custom_intro="Base class for the Sam3Tracker model's output.")
 class Sam3TrackerImageSegmentationOutput(ModelOutput):
@@ -68,13 +68,13 @@ class Sam3TrackerImageSegmentationOutput(ModelOutput):
         Attentions weights of the mask decoder.
     """
 
-    iou_scores: torch.FloatTensor | None = None
-    pred_masks: torch.FloatTensor | None = None
-    object_score_logits: torch.FloatTensor | None = None
+    iou_scores: Optional[torch.FloatTensor] = None
+    pred_masks: Optional[torch.FloatTensor] = None
+    object_score_logits: Optional[torch.FloatTensor] = None
     image_embeddings: tuple[torch.FloatTensor, ...] = None
-    vision_hidden_states: tuple[torch.FloatTensor, ...] | None = None
-    vision_attentions: tuple[torch.FloatTensor, ...] | None = None
-    mask_decoder_attentions: tuple[torch.FloatTensor, ...] | None = None
+    vision_hidden_states: Optional[tuple[torch.FloatTensor, ...]] = None
+    vision_attentions: Optional[tuple[torch.FloatTensor, ...]] = None
+    mask_decoder_attentions: Optional[tuple[torch.FloatTensor, ...]] = None
 
 
 class Sam3TrackerFeedForward(nn.Module):
@@ -107,19 +107,14 @@ class Sam3TrackerFeedForward(nn.Module):
         return hidden_states
 
 
-@auto_docstring(
-    custom_intro="""
-    Segment Anything Model 3 (SAM 3) for generating segmentation masks, given an input image and
-    input points and labels, boxes, or masks.
-    """
-)
+@auto_docstring
 class Sam3TrackerPreTrainedModel(PreTrainedModel):
     config_class = Sam3TrackerConfig
     base_model_prefix = "sam3_tracker"
     main_input_name = "pixel_values"
     input_modalities = ("image",)
     _supports_sdpa = True
-    _supports_flash_attn = True
+    _supports_flash_attn_2 = True
     _supports_attention_backend = True
 
     @torch.no_grad()
@@ -128,8 +123,6 @@ class Sam3TrackerPreTrainedModel(PreTrainedModel):
         if isinstance(module, Sam3TrackerModel):
             if module.no_memory_embedding is not None:
                 init.zeros_(module.no_memory_embedding)
-        elif isinstance(module, Sam3TrackerPositionalEmbedding):
-            init.normal_(module.positional_embedding, std=module.scale)
 
 
 class Sam3TrackerPositionalEmbedding(nn.Module):
@@ -238,10 +231,10 @@ class Sam3TrackerPromptEncoder(nn.Module):
 
     def forward(
         self,
-        input_points: tuple[torch.Tensor, torch.Tensor] | None,
-        input_labels: torch.Tensor | None,
-        input_boxes: torch.Tensor | None,
-        input_masks: torch.Tensor | None,
+        input_points: Optional[tuple[torch.Tensor, torch.Tensor]],
+        input_labels: Optional[torch.Tensor],
+        input_boxes: Optional[torch.Tensor],
+        input_masks: Optional[torch.Tensor],
     ) -> tuple[torch.Tensor, torch.Tensor]:
         """
         Embeds different types of prompts, returning both sparse and dense embeddings.
@@ -284,7 +277,7 @@ def eager_attention_forward(
     query: torch.Tensor,
     key: torch.Tensor,
     value: torch.Tensor,
-    attention_mask: torch.Tensor | None,
+    attention_mask: Optional[torch.Tensor],
     scaling: float,
     dropout: float = 0.0,
     **kwargs,
@@ -328,7 +321,7 @@ class Sam3TrackerAttention(nn.Module):
         query: torch.Tensor,
         key: torch.Tensor,
         value: torch.Tensor,
-        attention_similarity: torch.Tensor | None = None,
+        attention_similarity: Optional[torch.Tensor] = None,
         **kwargs: Unpack[TransformersKwargs],
     ) -> tuple[torch.Tensor, torch.Tensor]:
         # Input projections
@@ -343,15 +336,6 @@ class Sam3TrackerAttention(nn.Module):
         if self.config._attn_implementation != "eager":
             attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation]
 
-        if is_flash_attention_requested(self.config) and attention_similarity is not None:
-            # Target guided masks are represented as float masks and are incompatible with Flash Attention
-            # Fallback to SDPA for this call only so the rest of the model can still benefit from FA
-            attention_interface = ALL_ATTENTION_FUNCTIONS["sdpa"]
-            logger.warning_once(
-                "Falling back to SDPA for target-guided attention because "
-                "Flash Attention does not support additive bias masks."
-            )
-
         attn_output, attn_weights = attention_interface(
             self,
             query,
@@ -471,7 +455,7 @@ class Sam3TrackerTwoWayTransformer(nn.Module):
         attention_similarity: Tensor,
         target_embedding=None,
         **kwargs: Unpack[TransformersKwargs],
-    ) -> tuple | BaseModelOutput:
+    ) -> Union[tuple, BaseModelOutput]:
         if image_embeddings is None:
             raise ValueError("You have to specify an image_embedding")
 
@@ -582,8 +566,8 @@ class Sam3TrackerMaskDecoder(nn.Module):
         dense_prompt_embeddings: torch.Tensor,
         multimask_output: bool,
         high_resolution_features: list[torch.Tensor],
-        attention_similarity: torch.Tensor | None = None,
-        target_embedding: torch.Tensor | None = None,
+        attention_similarity: Optional[torch.Tensor] = None,
+        target_embedding: Optional[torch.Tensor] = None,
         **kwargs: Unpack[TransformersKwargs],
     ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
         """
@@ -737,10 +721,16 @@ class Sam3TrackerMaskDecoder(nn.Module):
 
 @dataclass
 @auto_docstring(custom_intro="Base class for the vision encoder's outputs.")
-class Sam3TrackerVisionEncoderOutput(BaseModelOutputWithPooling):
+class Sam3TrackerVisionEncoderOutput(ModelOutput):
     r"""
     last_hidden_state (`torch.FloatTensor` of shape `(batch_size, height, width, hidden_size)`):
         Sequence of hidden-states at the output of the last layer of the model.
+    fpn_hidden_states (`tuple(torch.FloatTensor)`):
+        Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape
+        `(batch_size, hidden_size, height, width)`. Feature maps from the Feature Pyramid Network neck.
+    fpn_position_encoding (`tuple(torch.FloatTensor)`):
+        Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape
+        `(batch_size, hidden_size, height, width)`. Positional encodings corresponding to the `fpn_hidden_states`.
     hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
         Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
         one for the output of each stage) of shape `(batch_size, height, width, hidden_size)`. Hidden-states of the
@@ -749,16 +739,13 @@ class Sam3TrackerVisionEncoderOutput(BaseModelOutputWithPooling):
         Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
         sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
         the self-attention heads.
-    fpn_hidden_states (`tuple(torch.FloatTensor)`):
-        Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape
-        `(batch_size, hidden_size, height, width)`. Feature maps from the Feature Pyramid Network neck.
-    fpn_position_encoding (`tuple(torch.FloatTensor)`):
-        Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape
-        `(batch_size, hidden_size, height, width)`. Positional encodings corresponding to the `fpn_hidden_states`.
     """
 
-    fpn_hidden_states: torch.FloatTensor | None = None
-    fpn_position_encoding: torch.FloatTensor | None = None
+    last_hidden_state: Optional[torch.FloatTensor] = None
+    fpn_hidden_states: Optional[torch.FloatTensor] = None
+    fpn_position_encoding: Optional[torch.FloatTensor] = None
+    hidden_states: Optional[tuple[torch.FloatTensor, ...]] = None
+    attentions: Optional[tuple[torch.FloatTensor, ...]] = None
 
 
 @auto_docstring(
@@ -770,7 +757,6 @@ class Sam3TrackerVisionEncoderOutput(BaseModelOutputWithPooling):
 class Sam3TrackerModel(Sam3TrackerPreTrainedModel):
     input_modalities = ("image", "text")
     _can_record_outputs = {"mask_decoder_attentions": OutputRecorder(Sam3TrackerTwoWayAttentionBlock, index=2)}
-    _tied_weights_keys = {}
     _keys_to_ignore_on_load_unexpected = [
         r"^detector_model.",
         r"^memory_.*",
@@ -838,8 +824,7 @@ class Sam3TrackerModel(Sam3TrackerPreTrainedModel):
                 Input pixel values
         """
         batch_size = pixel_values.shape[0]
-        image_outputs = self.get_image_features(pixel_values, return_dict=True, **kwargs)
-        feature_maps = image_outputs.fpn_hidden_states
+        feature_maps, _, _, _ = self.get_image_features(pixel_values, **kwargs)
 
         # add no memory embedding to the last feature map
         feature_maps[-1] = feature_maps[-1] + self.no_memory_embedding
@@ -855,10 +840,10 @@ class Sam3TrackerModel(Sam3TrackerPreTrainedModel):
     @torch.no_grad()
     def get_prompt_embeddings(
         self,
-        input_points: torch.FloatTensor | None = None,
-        input_labels: torch.LongTensor | None = None,
-        input_boxes: torch.FloatTensor | None = None,
-        input_masks: torch.LongTensor | None = None,
+        input_points: Optional[torch.FloatTensor] = None,
+        input_labels: Optional[torch.LongTensor] = None,
+        input_boxes: Optional[torch.FloatTensor] = None,
+        input_masks: Optional[torch.LongTensor] = None,
     ):
         r"""
         Returns the prompt embeddings by passing the input points, labels, boxes and masks through the prompt encoder.
@@ -889,15 +874,15 @@ class Sam3TrackerModel(Sam3TrackerPreTrainedModel):
     @auto_docstring
     def forward(
         self,
-        pixel_values: torch.FloatTensor | None = None,
-        input_points: torch.FloatTensor | None = None,
-        input_labels: torch.LongTensor | None = None,
-        input_boxes: torch.FloatTensor | None = None,
-        input_masks: torch.LongTensor | None = None,
-        image_embeddings: torch.FloatTensor | None = None,
+        pixel_values: Optional[torch.FloatTensor] = None,
+        input_points: Optional[torch.FloatTensor] = None,
+        input_labels: Optional[torch.LongTensor] = None,
+        input_boxes: Optional[torch.FloatTensor] = None,
+        input_masks: Optional[torch.LongTensor] = None,
+        image_embeddings: Optional[torch.FloatTensor] = None,
         multimask_output: bool = True,
-        attention_similarity: torch.FloatTensor | None = None,
-        target_embedding: torch.FloatTensor | None = None,
+        attention_similarity: Optional[torch.FloatTensor] = None,
+        target_embedding: Optional[torch.FloatTensor] = None,
         **kwargs: Unpack[TransformersKwargs],
     ) -> Sam3TrackerImageSegmentationOutput:
         r"""
@@ -958,16 +943,14 @@ class Sam3TrackerModel(Sam3TrackerPreTrainedModel):
 
         ```python
         >>> from PIL import Image
-        >>> import httpx
-        >>> from io import BytesIO
+        >>> import requests
         >>> from transformers import AutoModel, AutoProcessor
 
         >>> model = AutoModel.from_pretrained("danelcsb/sam3_tracker.1_hiera_tiny")
         >>> processor = AutoProcessor.from_pretrained("danelcsb/sam3_tracker.1_hiera_tiny")
 
-        >>> url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/model_doc/sam-car.png"
-        >>> with httpx.stream("GET", url) as response:
-        ...     raw_image = Image.open(BytesIO(response.read())).convert("RGB")
+        >>> img_url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/model_doc/sam-car.png"
+        >>> raw_image = Image.open(requests.get(img_url, stream=True).raw).convert("RGB")
         >>> input_points = [[[400, 650]]]  # 2D location of a window on the car
         >>> inputs = processor(images=raw_image, input_points=input_points, return_tensors="pt")
 
@@ -997,12 +980,10 @@ class Sam3TrackerModel(Sam3TrackerPreTrainedModel):
         vision_hidden_states = None
 
         if pixel_values is not None:
-            image_outputs: Sam3TrackerVisionEncoderOutput = self.get_image_features(
-                pixel_values, return_dict=True, **kwargs
+            feature_maps, _, vision_hidden_states, vision_attentions = self.get_image_features(
+                pixel_values,
+                **kwargs,
             )
-            feature_maps = image_outputs.fpn_hidden_states
-            vision_hidden_states = image_outputs.hidden_states
-            vision_attentions = image_outputs.attentions
 
             # add no memory embedding to the last feature map
             feature_maps[-1] = feature_maps[-1] + self.no_memory_embedding
@@ -1062,18 +1043,34 @@ class Sam3TrackerModel(Sam3TrackerPreTrainedModel):
             vision_attentions=vision_attentions,
         )
 
-    @can_return_tuple
-    @auto_docstring
     def get_image_features(
         self,
         pixel_values: torch.FloatTensor,
         **kwargs: Unpack[TransformersKwargs],
-    ) -> tuple | Sam3TrackerVisionEncoderOutput:
+    ) -> tuple[
+        list[torch.Tensor],
+        list[torch.Tensor],
+        Optional[tuple[torch.FloatTensor, ...]],
+        Optional[tuple[torch.FloatTensor, ...]],
+    ]:
         r"""
-        pixel_values (`torch.FloatTensor`):
-            Input pixel values of shape `(batch_size, num_channels, height, width)`.
+        Extract and preprocess image features using the vision encoder.
+
+        Args:
+            pixel_values (`torch.FloatTensor`):
+                Input pixel values of shape `(batch_size, num_channels, height, width)`.
+
+        Returns:
+            `tuple`: A tuple containing:
+                - feature_maps (`list[torch.Tensor]`): List of feature maps from different levels.
+                - feature_maps_position_embeddings (`list[torch.Tensor]`): List of positional embeddings for each feature level.
+                - vision_hidden_states (`tuple[torch.FloatTensor]`, *optional*): Hidden states from the vision encoder.
+                - vision_attentions (`tuple[torch.FloatTensor]`, *optional*): Attention weights from the vision encoder.
         """
-        vision_outputs: Sam3TrackerVisionEncoderOutput = self.vision_encoder(pixel_values, return_dict=True, **kwargs)
+        vision_outputs: Sam3TrackerVisionEncoderOutput = self.vision_encoder(
+            pixel_values,
+            **kwargs,
+        )
 
         feature_maps = vision_outputs.fpn_hidden_states
         feature_maps_position_embeddings = vision_outputs.fpn_position_encoding
@@ -1090,10 +1087,8 @@ class Sam3TrackerModel(Sam3TrackerPreTrainedModel):
             feature_map_position_embedding.flatten(2).permute(2, 0, 1)
             for feature_map_position_embedding in feature_maps_position_embeddings
         ]
-        vision_outputs.fpn_hidden_states = feature_maps
-        vision_outputs.fpn_position_encoding = feature_maps_position_embeddings
 
-        return vision_outputs
+        return feature_maps, feature_maps_position_embeddings, vision_outputs.hidden_states, vision_outputs.attentions
 
 
 __all__ = ["Sam3TrackerModel", "Sam3TrackerPreTrainedModel"]

transformers/models/sam3_tracker/modular_sam3_tracker.py

@@ -1,3 +1,4 @@
+# coding=utf-8
 # Copyright 2025 the HuggingFace Team. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 import torch
 
 from ... import initialization as init
@@ -134,12 +136,7 @@ class Sam3TrackerFeedForward(Sam2FeedForward):
     pass
 
 
-@auto_docstring(
-    custom_intro="""
-    Segment Anything Model 3 (SAM 3) for generating segmentation masks, given an input image and
-    input points and labels, boxes, or masks.
-    """
-)
+@auto_docstring
 class Sam3TrackerPreTrainedModel(Sam2PreTrainedModel):
     @torch.no_grad()
     def _init_weights(self, module):
@@ -147,8 +144,6 @@ class Sam3TrackerPreTrainedModel(Sam2PreTrainedModel):
         if isinstance(module, Sam3TrackerModel):
             if module.no_memory_embedding is not None:
                 init.zeros_(module.no_memory_embedding)
-        elif isinstance(module, Sam3TrackerPositionalEmbedding):
-            init.normal_(module.positional_embedding, std=module.scale)
 
 
 class Sam3TrackerPositionalEmbedding(Sam2PositionalEmbedding):