optimum-rbln 0.8.4a1__tar.gz → 0.8.4a3__tar.gz

{optimum_rbln-0.8.4a1 → optimum_rbln-0.8.4a3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: optimum-rbln
-Version: 0.8.4a1
+Version: 0.8.4a3
 Summary: Optimum RBLN is the interface between the HuggingFace Transformers and Diffusers libraries and RBLN accelerators. It provides a set of tools enabling easy model loading and inference on single and multiple rbln device settings for different downstream tasks.
 Project-URL: Homepage, https://rebellions.ai
 Project-URL: Documentation, https://docs.rbln.ai
@@ -23,7 +23,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: <3.13,>=3.9
 Requires-Dist: accelerate>=1.0.1
-Requires-Dist: diffusers==0.34.0
+Requires-Dist: diffusers==0.35.1
 Requires-Dist: packaging>=24.1
 Requires-Dist: torch==2.7.0
 Requires-Dist: torchaudio<=2.7.0

{optimum_rbln-0.8.4a1 → optimum_rbln-0.8.4a3}/pyproject.toml

@@ -33,7 +33,7 @@ dependencies = [
     "torchvision<=0.22.0",
     "accelerate>=1.0.1",
     "transformers==4.51.3",
-    "diffusers==0.34.0",
+    "diffusers==0.35.1",
     "packaging>=24.1",
 ]
 

{optimum_rbln-0.8.4a1 → optimum_rbln-0.8.4a3}/src/optimum/rbln/__init__.py

@@ -148,6 +148,10 @@ _import_structure = {
         "RBLNQwen3ForCausalLMConfig",
         "RBLNQwen3Model",
         "RBLNQwen3ModelConfig",
+        "RBLNQwen2VisionTransformerPretrainedModel",
+        "RBLNQwen2VisionTransformerPretrainedModelConfig",
+        "RBLNQwen2VLForConditionalGeneration",
+        "RBLNQwen2VLForConditionalGenerationConfig",
         "RBLNResNetForImageClassification",
         "RBLNResNetForImageClassificationConfig",
         "RBLNRobertaForMaskedLM",
@@ -430,6 +434,10 @@ if TYPE_CHECKING:
         RBLNQwen2ForCausalLMConfig,
         RBLNQwen2Model,
         RBLNQwen2ModelConfig,
+        RBLNQwen2VisionTransformerPretrainedModel,
+        RBLNQwen2VisionTransformerPretrainedModelConfig,
+        RBLNQwen2VLForConditionalGeneration,
+        RBLNQwen2VLForConditionalGenerationConfig,
         RBLNQwen3ForCausalLM,
         RBLNQwen3ForCausalLMConfig,
         RBLNQwen3Model,

{optimum_rbln-0.8.4a1 → optimum_rbln-0.8.4a3}/src/optimum/rbln/__version__.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.8.4a1'
-__version_tuple__ = version_tuple = (0, 8, 4, 'a1')
+__version__ = version = '0.8.4a3'
+__version_tuple__ = version_tuple = (0, 8, 4, 'a3')
 
 __commit_id__ = commit_id = None

{optimum_rbln-0.8.4a1 → optimum_rbln-0.8.4a3}/src/optimum/rbln/transformers/__init__.py

@@ -130,6 +130,10 @@ _import_structure = {
         "RBLNQwen2_5_VisionTransformerPretrainedModelConfig",
         "RBLNQwen2_5_VLForConditionalGeneration",
         "RBLNQwen2_5_VLForConditionalGenerationConfig",
+        "RBLNQwen2VisionTransformerPretrainedModel",
+        "RBLNQwen2VisionTransformerPretrainedModelConfig",
+        "RBLNQwen2VLForConditionalGeneration",
+        "RBLNQwen2VLForConditionalGenerationConfig",
         "RBLNQwen2Model",
         "RBLNQwen2ModelConfig",
         "RBLNQwen2ForCausalLM",
@@ -282,6 +286,10 @@ if TYPE_CHECKING:
         RBLNQwen2ForCausalLMConfig,
         RBLNQwen2Model,
         RBLNQwen2ModelConfig,
+        RBLNQwen2VisionTransformerPretrainedModel,
+        RBLNQwen2VisionTransformerPretrainedModelConfig,
+        RBLNQwen2VLForConditionalGeneration,
+        RBLNQwen2VLForConditionalGenerationConfig,
         RBLNQwen3ForCausalLM,
         RBLNQwen3ForCausalLMConfig,
         RBLNQwen3Model,

{optimum_rbln-0.8.4a1 → optimum_rbln-0.8.4a3}/src/optimum/rbln/transformers/models/__init__.py

@@ -85,6 +85,12 @@ _import_structure = {
         "RBLNQwen2_5_VLForConditionalGeneration",
         "RBLNQwen2_5_VLForConditionalGenerationConfig",
     ],
+    "qwen2_vl": [
+        "RBLNQwen2VisionTransformerPretrainedModel",
+        "RBLNQwen2VisionTransformerPretrainedModelConfig",
+        "RBLNQwen2VLForConditionalGeneration",
+        "RBLNQwen2VLForConditionalGenerationConfig",
+    ],
     "decoderonly": [
         "RBLNDecoderOnlyModelConfig",
         "RBLNDecoderOnlyModel",
@@ -281,6 +287,12 @@ if TYPE_CHECKING:
         RBLNQwen2_5_VLForConditionalGeneration,
         RBLNQwen2_5_VLForConditionalGenerationConfig,
     )
+    from .qwen2_vl import (
+        RBLNQwen2VisionTransformerPretrainedModel,
+        RBLNQwen2VisionTransformerPretrainedModelConfig,
+        RBLNQwen2VLForConditionalGeneration,
+        RBLNQwen2VLForConditionalGenerationConfig,
+    )
     from .qwen3 import RBLNQwen3ForCausalLM, RBLNQwen3ForCausalLMConfig, RBLNQwen3Model, RBLNQwen3ModelConfig
     from .resnet import RBLNResNetForImageClassification, RBLNResNetForImageClassificationConfig
     from .roberta import (

{optimum_rbln-0.8.4a1 → optimum_rbln-0.8.4a3}/src/optimum/rbln/transformers/models/decoderonly/decoderonly_architecture.py

@@ -579,7 +579,7 @@ class DecoderOnlyAttention(nn.Module):
         )
         self.head_dim = self._original_mod.head_dim
         self._phase = "prefill"
-        self.scale = torch.tensor(self.get_attn_scale())
+        self.scale = torch.nn.Parameter(torch.tensor(self.get_attn_scale()))
         self.quantization = rbln_config.quantization
 
         if hasattr(self._original_mod, "num_key_value_heads"):

{optimum_rbln-0.8.4a1 → optimum_rbln-0.8.4a3}/src/optimum/rbln/transformers/models/grounding_dino/grounding_dino_architecture.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import math
 from functools import wraps
 from typing import TYPE_CHECKING, List, Optional, Tuple
 
@@ -20,7 +21,6 @@ from torch import Tensor
 from transformers.models.grounding_dino.modeling_grounding_dino import (
     GroundingDinoDecoder,
     GroundingDinoEncoder,
-    get_sine_pos_embed,
 )
 
 
@@ -33,31 +33,46 @@ def monkey_patch():
         GroundingDinoBiMultiHeadAttention,
         GroundingDinoEncoderLayer,
         GroundingDinoMultiscaleDeformableAttention,
+        MultiScaleDeformableAttention,
     )
 
     original_forward = GroundingDinoMultiscaleDeformableAttention.forward
     original_bi_multihead_attention_forward = GroundingDinoBiMultiHeadAttention.forward
     original_encoder_layer_forward = GroundingDinoEncoderLayer.forward
+    original_multiscale_deform_attn = MultiScaleDeformableAttention.forward
 
     # Patch the methods with the custom implementations
     GroundingDinoMultiscaleDeformableAttention.forward = _GroundingDinoMultiscaleDeformableAttention.forward
     GroundingDinoBiMultiHeadAttention.forward = _GroundingDinoBiMultiHeadAttention.forward
     GroundingDinoEncoderLayer.forward = _GroundingDinoEncoderLayer.forward
+    MultiScaleDeformableAttention.forward = _MultiScaleDeformableAttention.forward
 
-    return (original_forward, original_bi_multihead_attention_forward, original_encoder_layer_forward)
+    return (
+        original_forward,
+        original_bi_multihead_attention_forward,
+        original_encoder_layer_forward,
+        original_multiscale_deform_attn,
+    )
 
 
-def restore_monkey_patch(original_forward, original_bi_multihead_attention_forward, original_encoder_layer_forward):
+def restore_monkey_patch(
+    original_forward,
+    original_bi_multihead_attention_forward,
+    original_encoder_layer_forward,
+    original_multiscale_deform_attn,
+):
     from transformers.models.grounding_dino.modeling_grounding_dino import (
         GroundingDinoBiMultiHeadAttention,
         GroundingDinoEncoderLayer,
         GroundingDinoMultiscaleDeformableAttention,
+        MultiScaleDeformableAttention,
     )
 
     # Restore the original methods
     GroundingDinoMultiscaleDeformableAttention.forward = original_forward
     GroundingDinoBiMultiHeadAttention.forward = original_bi_multihead_attention_forward
     GroundingDinoEncoderLayer.forward = original_encoder_layer_forward
+    MultiScaleDeformableAttention.forward = original_multiscale_deform_attn
 
 
 def monkey_patch_decorator(func):
@@ -76,6 +91,30 @@ def monkey_patch_decorator(func):
     return wrapper
 
 
+def get_sine_pos_embed(
+    pos_tensor: torch.Tensor, num_pos_feats: int = 128, temperature: int = 10000, exchange_xy: bool = True
+) -> Tensor:
+    scale = 2 * math.pi
+    dim_t = torch.arange(num_pos_feats, dtype=torch.float32, device=pos_tensor.device)
+    dim_t = temperature ** (2 * torch.div(dim_t, 2, rounding_mode="floor") / num_pos_feats)
+
+    scaled_pos = pos_tensor.unsqueeze(-1) * scale / dim_t
+    reshaped_pos = scaled_pos.view(*scaled_pos.shape[:-1], -1, 2)
+    sin_chunk, cos_chunk = torch.split(reshaped_pos, 1, dim=-1)
+    sin_embed = sin_chunk.squeeze(-1).sin()
+    cos_embed = cos_chunk.squeeze(-1).cos()
+
+    pos_embed = torch.stack((sin_embed, cos_embed), dim=-1).flatten(-2)
+
+    if exchange_xy and pos_tensor.shape[-1] >= 2:
+        swapped_embeds = torch.cat([pos_embed[..., 1:2, :], pos_embed[..., 0:1, :], pos_embed[..., 2:, :]], dim=-2)
+        pos_embed = swapped_embeds
+
+    position_embeddings = pos_embed.flatten(start_dim=-2)
+
+    return position_embeddings
+
+
 class _GroundingDinoEncoder(torch.nn.Module):
     def __init__(self, model: "GroundingDinoEncoder", rbln_config: "RBLNGroundingDinoEncoderConfig"):
         super().__init__()
@@ -386,16 +425,20 @@ class _GroundingDinoMultiscaleDeformableAttention(torch.nn.Module):
         # batch_size, num_queries, n_heads, n_levels, n_points, 2
         num_coordinates = reference_points.shape[-1]
         if num_coordinates == 2:
-            offset_normalizer = torch.stack([spatial_shapes[..., 1], spatial_shapes[..., 0]], -1)
-            sampling_locations = (
-                reference_points[:, :, None, :, None, :]
+            offset_normalizer = 0.5 * torch.stack([spatial_shapes[..., 1], spatial_shapes[..., 0]], -1)
+            sampling_grids = (
+                2 * reference_points[:, :, None, :, None, :]
+                - 1
                 + sampling_offsets / offset_normalizer[None, None, None, :, None, :]
             )
         elif num_coordinates == 4:
-            sampling_locations = (
-                reference_points[:, :, None, :, None, :2]
-                + sampling_offsets / self.n_points * reference_points[:, :, None, :, None, 2:] * 0.5
-            )
+            ref_points_xy, ref_points_wh = torch.split(reference_points, 2, dim=-1)
+            ref_points_xy = ref_points_xy[:, :, None, :, None, :]
+            ref_points_wh = ref_points_wh[:, :, None, :, None, :]
+            ref_points_grids = 2 * ref_points_xy - 1
+            offset_grids = sampling_offsets / self.n_points * ref_points_wh
+            sampling_grids = ref_points_grids + offset_grids
+
         else:
             raise ValueError(f"Last dim of reference_points must be 2 or 4, but got {reference_points.shape[-1]}")
 
@@ -404,7 +447,7 @@ class _GroundingDinoMultiscaleDeformableAttention(torch.nn.Module):
             spatial_shapes,
             spatial_shapes_list,
             level_start_index,
-            sampling_locations,
+            sampling_grids,
             attention_weights,
             self.im2col_step,
         )
@@ -456,15 +499,14 @@ class _GroundingDinoBiMultiHeadAttention(torch.nn.Module):
         # # Do not increase -50000/50000, data type half has quite limited range
         attn_weights = torch.clamp(attn_weights, min=-50000, max=50000)
 
-        attn_weights_transposed = attn_weights.transpose(1, 2)
         # RBLN FIX: max_values from scalar to vector
-        text_attn_weights = attn_weights_transposed - torch.max(attn_weights_transposed, dim=-1, keepdim=True)[
-            0
-        ].repeat(1, 1, tgt_len)
+        text_attn_weights = attn_weights - torch.max(attn_weights, dim=1, keepdim=True)[0].repeat(1, tgt_len, 1)
 
         # # Do not increase -50000/50000, data type half has quite limited range
         text_attn_weights = torch.clamp(text_attn_weights, min=-50000, max=50000)
 
+        text_attn_weights = text_attn_weights.transpose(1, 2)
+
         # mask vision for language
         if vision_attention_mask is not None:
             # RBLN FIX: bool tensor to float tensor
@@ -511,3 +553,47 @@ class _GroundingDinoBiMultiHeadAttention(torch.nn.Module):
         text_attn_output = self.out_text_proj(text_attn_output)
 
         return (vision_attn_output, vision_attn_weights), (text_attn_output, text_attn_weights)
+
+
+class _MultiScaleDeformableAttention(torch.nn.Module):
+    def forward(
+        self,
+        value: Tensor,
+        value_spatial_shapes: Tensor,
+        value_spatial_shapes_list: List[Tuple],
+        level_start_index: Tensor,
+        sampling_grids: Tensor,
+        attention_weights: Tensor,
+        im2col_step: int,
+    ):
+        batch_size, _, num_heads, hidden_dim = value.shape
+        _, num_queries, num_heads, num_levels, num_points, _ = sampling_grids.shape
+        value_list = value.split([height * width for height, width in value_spatial_shapes_list], dim=1)
+        sampling_value_list = []
+        sampling_grids_list = [t.squeeze(3) for t in torch.split(sampling_grids, 1, dim=3)]
+        for level_id, (height, width) in enumerate(value_spatial_shapes_list):
+            value_l_ = (
+                value_list[level_id].permute(0, 2, 3, 1).reshape(batch_size * num_heads, hidden_dim, height, width)
+            )
+            sampling_grid_l_ = sampling_grids_list[level_id].transpose(1, 2).flatten(0, 1)
+            sampling_value_l_ = torch.nn.functional.grid_sample(
+                value_l_,
+                sampling_grid_l_,
+                mode="bilinear",
+                padding_mode="zeros",
+                align_corners=False,
+            )
+            sampling_value_list.append(sampling_value_l_)
+
+        sampling_values = torch.cat(sampling_value_list, dim=-1)
+        attention_weights_prep = attention_weights.transpose(1, 2)
+        values_permuted = sampling_values.permute(0, 2, 3, 1)
+
+        weights_for_matmul = attention_weights_prep.reshape(
+            batch_size * num_heads, num_queries, 1, num_levels * num_points
+        )
+        output_before_permute = torch.matmul(weights_for_matmul, values_permuted)
+        output_before_view = output_before_permute.squeeze(2).permute(0, 2, 1)
+        output = output_before_view.reshape(batch_size, num_heads * hidden_dim, num_queries)
+
+        return output.transpose(1, 2).contiguous()

optimum_rbln-0.8.4a3/src/optimum/rbln/transformers/models/qwen2_vl/__init__.py

@@ -0,0 +1,19 @@
+# Copyright 2025 Rebellions Inc. All rights reserved.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .configuration_qwen2_vl import (
+    RBLNQwen2VisionTransformerPretrainedModelConfig,
+    RBLNQwen2VLForConditionalGenerationConfig,
+)
+from .modeling_qwen2_vl import RBLNQwen2VisionTransformerPretrainedModel, RBLNQwen2VLForConditionalGeneration

optimum_rbln-0.8.4a3/src/optimum/rbln/transformers/models/qwen2_vl/configuration_qwen2_vl.py

@@ -0,0 +1,88 @@
+# Copyright 2025 Rebellions Inc. All rights reserved.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Dict, List, Optional, Union
+
+from ....configuration_utils import RBLNModelConfig
+from ..decoderonly.configuration_decoderonly import RBLNDecoderOnlyModelForCausalLMConfig
+
+
+class RBLNQwen2VLForConditionalGenerationConfig(RBLNDecoderOnlyModelForCausalLMConfig):
+    submodules = ["visual"]
+
+    def __init__(
+        self,
+        use_inputs_embeds: bool = True,
+        visual: Optional[RBLNModelConfig] = None,
+        **kwargs: Dict[str, Any],
+    ):
+        """
+        Args:
+            use_inputs_embeds (bool): Whether or not to use `inputs_embeds` as input. Defaults to `True`.
+            visual (Optional[RBLNModelConfig]): Configuration for the vision encoder component.
+            **kwargs: Additional arguments passed to the parent `RBLNDecoderOnlyModelForCausalLMConfig`.
+
+        Raises:
+            ValueError: If `use_inputs_embeds` is False.
+            ValueError: If the visual configuration is provided but contains invalid settings, such as an invalid max_seq_lens (e.g., not a positive integer or insufficient for the expected resolution).
+            ValueError: If visual is None and no default vision configuration can be inferred for the model architecture.
+            ValueError: If any inherited parameters violate constraints defined in the parent class, such as batch_size not being a positive integer, prefill_chunk_size not being divisible by 64, or max_seq_len not meeting requirements for Flash Attention.
+        """
+        super().__init__(use_inputs_embeds=use_inputs_embeds, **kwargs)
+        if not self.use_inputs_embeds:
+            raise ValueError(
+                "RBLNQwen2VLForConditionalGenerationConfig does not allow `use_inputs_embeds` to be set to False, "
+                "as RBLNQwen2VLForConditionalGeneration accepts only `inputs_embeds` as input."
+            )
+        self.visual = visual
+
+
+class RBLNQwen2VisionTransformerPretrainedModelConfig(RBLNModelConfig):
+    def __init__(self, max_seq_lens: Union[int, List[int]] = None, **kwargs: Dict[str, Any]):
+        """
+        Args:
+            max_seq_lens (Optional[Union[int, List[int]]]): Maximum sequence lengths for Vision
+                Transformer attention. Can be an integer or list of integers, each indicating
+                the number of patches in a sequence for an image or video. For example, an image
+                of 224x224 pixels with patch size 14 results in (224/14) * (224/14) = 256 patches,
+                so `max_seq_lens` must be at least 256. RBLN optimization runs inference per image
+                or video frame, so set `max_seq_lens` to match the maximum expected resolution to
+                optimize computation. If not provided, a `ValueError` is raised.
+            **kwargs: Additional arguments passed to the parent RBLNModelConfig.
+
+        Raises:
+            ValueError: If batch_size is not a positive integer.
+            ValueError: If `max_seq_lens` (or any value in the list) is not a positive integer.
+            ValueError: If `max_seq_lens` is insufficient for the expected image/video resolution.
+            ValueError: If `batch_size` (inherited from RBLNModelConfig) is not a positive integer.
+
+        Max Seq Lens:
+            Since `Qwen2VLForConditionalGeneration` performs inference on a per-image or per-frame basis,
+            `max_seq_lens` should be set based on the maximum expected resolution of the input images or video frames.
+
+            The value must be greater than or equal to the number of patches generated from the input image.
+            For example, a 224x224 image with a patch size of 14 results in (224 / 14) * (224 / 14) = 256 patches.
+            Therefore, `max_seq_lens` must be at least 256.
+        """
+        super().__init__(**kwargs)
+
+        if max_seq_lens is not None:
+            if isinstance(max_seq_lens, int):
+                max_seq_lens = [max_seq_lens]
+            elif isinstance(max_seq_lens, list):
+                max_seq_lens.sort(reverse=True)
+        else:
+            raise ValueError("'max_seq_lens' must be specified.")
+
+        self.max_seq_lens = max_seq_lens