optimum-rbln 0.9.2a4__tar.gz → 0.9.2a5__tar.gz

optimum_rbln-0.9.2a5/.github/version.yaml

@@ -0,0 +1 @@
+rebel_compiler_version: 0.9.2.dev155+g1ab49983.prod

optimum_rbln-0.9.2a5/.github/workflows/rbln_scheduled_test.yaml

@@ -0,0 +1,186 @@
+name: Optimum-rbln / Scheduled Test
+
+on:
+  schedule:
+    # Run every day at 2am (17:00 UTC, 2:00am KST)
+    - cron: '0 17 * * *'
+
+env:
+  HF_USER_ID: ${{ secrets.HF_USER_ID }}
+  HF_AUTH_TOKEN: ${{ secrets.HF_AUTH_TOKEN }}
+
+jobs:
+  check-code-quality:
+    uses: ./.github/workflows/check_code_quality.yml
+    
+  test-docstrings:
+    uses: ./.github/workflows/test-docstrings.yml
+    with:
+      test_all_files: true
+
+  load-version:
+    runs-on: rebel-k8s-runner
+    outputs:
+      compiler_version: ${{ steps.get_version.outputs.compiler_version }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      
+      - name: Get compiler version
+        id: get_version
+        run: |
+          VERSION=$(grep rebel_compiler_version .github/version.yaml | cut -d ':' -f2 | tr -d ' ')
+          echo "compiler_version=$VERSION" >> $GITHUB_OUTPUT
+
+  check-compiler:
+    needs: load-version
+    uses: ./.github/workflows/rbln_check_compiler.yaml
+    with:
+      compiler_version: ${{ needs.load-version.outputs.compiler_version }}
+    secrets: inherit
+
+  optimum-rbln-pytest:
+    needs: [load-version, check-compiler]
+    if: ${{ needs.check-compiler.outputs.compiler_version_check == 'true' }}
+    uses: ./.github/workflows/rbln_optimum_pytest.yaml
+    with:
+      ref: main
+      rebel_compiler_version: ${{ needs.check-compiler.outputs.compiler_version }}
+      test_level: "full"
+      enable_hf_hub_tests: true
+      fail_fast: false
+    secrets: inherit
+
+  optimum-rbln-inference-test:
+    needs: check-compiler
+    if: ${{ needs.check-compiler.outputs.compiler_version_check == 'true' }}
+    uses: ./.github/workflows/rbln_optimum_inference_test.yaml
+    with:
+      ref: main
+      rebel_compiler_version: ${{ needs.check-compiler.outputs.compiler_version }}
+    secrets: inherit
+
+  summary_and_report:
+    needs: [load-version, check-compiler, optimum-rbln-pytest]
+    if: always()
+    runs-on: rebel-k8s-runner
+    steps:
+      - name: Get failed test details
+        id: get_failed_tests
+        if: needs.optimum-rbln-pytest.result == 'failure'
+        run: |
+          # Get the workflow run jobs
+          JOBS=$(curl -s -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
+            "https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs")
+          
+          # Extract failed pytest job names step by step for readability
+          FAILED_JOB_NAMES=$(echo "$JOBS" | jq -r '.jobs[] | select(.conclusion == "failure" and (.name | contains("Pytest"))) | .name')
+          # Remove "Pytest (...)" wrapper to get the test name
+          FAILED_TEST_NAMES=$(echo "$FAILED_JOB_NAMES" | sed 's/Pytest (\(.*\))/\1/')
+          # Join names with comma and space
+          FAILED_TESTS=$(echo "$FAILED_TEST_NAMES" | tr '\n' ', ' | sed 's/,$//')
+          
+          if [ -z "$FAILED_TESTS" ]; then
+            echo "failed_tests=Unknown" >> $GITHUB_OUTPUT
+          else
+            echo "failed_tests=$FAILED_TESTS" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Determine test results
+        id: test_results
+        run: |
+          # Determine overall status
+          if [ "${{ needs.optimum-rbln-pytest.result }}" == "success" ]; then
+            echo "pytest_status=✅ Success - All tests passed" >> $GITHUB_OUTPUT
+            echo "pytest_emoji=✅" >> $GITHUB_OUTPUT
+          elif [ "${{ needs.optimum-rbln-pytest.result }}" == "failure" ]; then
+            FAILED="${{ steps.get_failed_tests.outputs.failed_tests }}"
+            if [ -n "$FAILED" ] && [ "$FAILED" != "Unknown" ]; then
+              echo "pytest_status=❌ Failed - Tests: \`$FAILED\`" >> $GITHUB_OUTPUT
+            else
+              echo "pytest_status=❌ Failed" >> $GITHUB_OUTPUT
+            fi
+            echo "pytest_emoji=❌" >> $GITHUB_OUTPUT
+          elif [ "${{ needs.optimum-rbln-pytest.result }}" == "skipped" ]; then
+            echo "pytest_status=⏭️ Skipped" >> $GITHUB_OUTPUT
+            echo "pytest_emoji=⏭️" >> $GITHUB_OUTPUT
+          else
+            echo "pytest_status=⚠️ Cancelled" >> $GITHUB_OUTPUT
+            echo "pytest_emoji=⚠️" >> $GITHUB_OUTPUT
+          fi
+
+          # Determine compiler check status
+          if [ "${{ needs.check-compiler.result }}" == "success" ]; then
+            echo "compiler_status=✅ Available" >> $GITHUB_OUTPUT
+          else
+            echo "compiler_status=❌ Not Available" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Notify Slack
+        if: always()
+        run: |
+          # Determine overall workflow status emoji
+          if [ "${{ needs.optimum-rbln-pytest.result }}" == "success" ]; then
+            title="✅ Optimum-RBLN Scheduled Pytest Results"
+          elif [ "${{ needs.check-compiler.result }}" != "success" ]; then
+            title="⚠️ Optimum-RBLN Scheduled Pytest Results - Compiler Check Failed"
+          else
+            title="❌ Optimum-RBLN Scheduled Pytest Results"
+          fi
+
+          commit="*Commit*\n<https://github.com/rebellions-sw/optimum-rbln/commit/${{github.sha}}|${{github.sha}}>"
+          action_link="*CI Report*\n<https://github.com/rebellions-sw/optimum-rbln/actions/runs/${{ github.run_id }}|View Details>"
+          compiler_version="${{ needs.load-version.outputs.compiler_version }}"
+
+          payload=$(jq -n \
+            --arg channel "${{ secrets.SLACK_CI_REPORTER_CHANNEL }}" \
+            --arg title "$title" \
+            --arg commit "$commit" \
+            --arg action_link "$action_link" \
+            --arg compiler_version "$compiler_version" \
+            --arg compiler_status "${{ steps.test_results.outputs.compiler_status }}" \
+            --arg pytest_status "${{ steps.test_results.outputs.pytest_status }}" \
+            '{
+              channel: $channel,
+              text: "Optimum-RBLN Scheduled Test Results",
+              blocks: [
+                {
+                  type: "header",
+                  text: {
+                    type: "plain_text",
+                    text: $title
+                  }
+                },
+                {
+                  type: "section",
+                  fields: [
+                    { type: "mrkdwn", text: $commit },
+                    { type: "mrkdwn", text: $action_link }
+                  ]
+                },
+                {
+                  type: "section",
+                  fields: [
+                    { type: "mrkdwn", text: "*Compiler Version*" },
+                    { type: "mrkdwn", text: ("`" + $compiler_version + "`") }
+                  ]
+                },
+                { type: "divider" },
+                {
+                  type: "section",
+                  fields: [
+                    { type: "mrkdwn", text: "*Compiler Check*" },
+                    { type: "mrkdwn", text: $compiler_status }
+                  ]
+                },
+                {
+                  type: "section",
+                  fields: [
+                    { type: "mrkdwn", text: "*Pytest Results*" },
+                    { type: "mrkdwn", text: $pytest_status }
+                  ]
+                }
+              ]
+            }'
+          )
+          curl -X POST -H 'Authorization: Bearer ${{ secrets.SLACK_BOT_USER_OAUTH_ACCESS_TOKEN }}' -H 'Content-type: application/json; charset=utf-8' --data "$payload" https://slack.com/api/chat.postMessage

{optimum_rbln-0.9.2a4 → optimum_rbln-0.9.2a5}/.github/workflows/rbln_trigger_on_pr.yaml

@@ -92,4 +92,4 @@ jobs:
       ref: ${{ github.event.pull_request.head.sha }}
       rebel_compiler_version: ${{ needs.check-compiler.outputs.compiler_version }}
       test_level: "default"
-    secrets: inherit
+    secrets: inherit

{optimum_rbln-0.9.2a4 → optimum_rbln-0.9.2a5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: optimum-rbln
-Version: 0.9.2a4
+Version: 0.9.2a5
 Summary: Optimum RBLN is the interface between the HuggingFace Transformers and Diffusers libraries and RBLN accelerators. It provides a set of tools enabling easy model loading and inference on single and multiple rbln device settings for different downstream tasks.
 Project-URL: Homepage, https://rebellions.ai
 Project-URL: Documentation, https://docs.rbln.ai
@@ -26,10 +26,10 @@ Requires-Python: <3.14,>=3.9
 Requires-Dist: accelerate>=1.0.1
 Requires-Dist: diffusers==0.35.1
 Requires-Dist: packaging>=24.1
-Requires-Dist: torch==2.7.0
-Requires-Dist: torchaudio<=2.7.0
-Requires-Dist: torchvision<=0.22.0
-Requires-Dist: transformers==4.53.1
+Requires-Dist: torch==2.8.0
+Requires-Dist: torchaudio<=2.8.0
+Requires-Dist: torchvision<=0.23.0
+Requires-Dist: transformers==4.57.1
 Description-Content-Type: text/markdown
 
 

{optimum_rbln-0.9.2a4 → optimum_rbln-0.9.2a5}/pyproject.toml

@@ -29,11 +29,11 @@ classifiers = [
 ]
 keywords = ["transformers", "diffusers", "inference", "rbln", "atom", "rebel"]
 dependencies = [
-    "torch==2.7.0",
-    "torchaudio<=2.7.0",
-    "torchvision<=0.22.0",
+    "torch==2.8.0",
+    "torchaudio<=2.8.0",
+    "torchvision<=0.23.0",
     "accelerate>=1.0.1",
-    "transformers==4.53.1",
+    "transformers==4.57.1",
     "diffusers==0.35.1",
     "packaging>=24.1",
 ]

{optimum_rbln-0.9.2a4 → optimum_rbln-0.9.2a5}/src/optimum/rbln/__version__.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.9.2a4'
-__version_tuple__ = version_tuple = (0, 9, 2, 'a4')
+__version__ = version = '0.9.2a5'
+__version_tuple__ = version_tuple = (0, 9, 2, 'a5')
 
 __commit_id__ = commit_id = None

{optimum_rbln-0.9.2a4 → optimum_rbln-0.9.2a5}/src/optimum/rbln/modeling.py

@@ -34,6 +34,49 @@ if TYPE_CHECKING:
 logger = get_logger(__name__)
 
 
+def _get_dtype(
+    cls,
+    dtype: Optional[Union[str, torch.dtype, dict]],
+    config: PretrainedConfig,
+) -> tuple[PretrainedConfig, Optional[torch.dtype], Optional[torch.dtype]]:
+    dtype_orig = None
+
+    if dtype is not None:
+        if isinstance(dtype, str):
+            if dtype == "auto":
+                if hasattr(config, "dtype") and config.dtype is not None:
+                    dtype = config.dtype
+                else:
+                    dtype = torch.get_default_dtype()
+            elif hasattr(torch, dtype):
+                dtype = getattr(torch, dtype)
+                config.dtype = dtype
+        elif isinstance(dtype, torch.dtype):
+            config.dtype = dtype
+        elif isinstance(dtype, dict):
+            for key, curr_dtype in dtype.items():
+                if hasattr(config, key):
+                    value = getattr(config, key)
+                    curr_dtype = curr_dtype if not isinstance(curr_dtype, str) else getattr(torch, curr_dtype)
+                    value.dtype = curr_dtype
+            # main torch dtype for modules that aren't part of any sub-config
+            dtype = dtype.get("")
+            dtype = dtype if not isinstance(dtype, str) else getattr(torch, dtype)
+            config.dtype = dtype
+            if dtype is None:
+                dtype = torch.float32
+        else:
+            raise ValueError(f"Invalid dtype: {dtype}")
+
+        dtype_orig = cls._set_default_dtype(dtype)
+    else:
+        # Use default dtype
+        default_dtype = torch.get_default_dtype()
+        config.dtype = default_dtype
+
+    return config, dtype, dtype_orig
+
+
 class RBLNModel(RBLNBaseModel):
     @classmethod
     def update_kwargs(cls, kwargs):
@@ -206,10 +249,37 @@ class RBLNModel(RBLNBaseModel):
         trust_remote_code: bool = False,
         # Some rbln-config should be applied before loading torch module (i.e. quantized llm)
         rbln_config: Optional[RBLNModelConfig] = None,
+        dtype: Optional[Union[str, torch.dtype, dict]] = None,
         **kwargs,
     ) -> "PreTrainedModel":
         kwargs = cls.update_kwargs(kwargs)
-        return cls.get_hf_class().from_pretrained(
+
+        hf_class = cls.get_hf_class()
+
+        if dtype is not None:
+            config = hf_class.config_class.from_pretrained(
+                model_id,
+                subfolder=subfolder,
+                revision=revision,
+                cache_dir=cache_dir,
+                use_auth_token=use_auth_token,
+                local_files_only=local_files_only,
+                force_download=force_download,
+                trust_remote_code=trust_remote_code,
+            )
+
+            config, processed_dtype, dtype_orig = _get_dtype(
+                cls=hf_class,
+                dtype=dtype,
+                config=config,
+            )
+
+            kwargs["torch_dtype"] = processed_dtype
+
+            if dtype_orig is not None:
+                hf_class._set_default_dtype(dtype_orig)
+
+        return hf_class.from_pretrained(
             model_id,
             subfolder=subfolder,
             revision=revision,

{optimum_rbln-0.9.2a4 → optimum_rbln-0.9.2a5}/src/optimum/rbln/transformers/modeling_generic.py

@@ -23,6 +23,7 @@ different model architectures.
 import inspect
 from typing import TYPE_CHECKING, Optional, Union
 
+from torch import nn
 from transformers import (
     AutoModel,
     AutoModelForAudioClassification,
@@ -57,6 +58,28 @@ class RBLNTransformerEncoder(RBLNModel):
     rbln_model_input_names = ["input_ids", "attention_mask", "token_type_ids"]
     rbln_dtype = "int64"
 
+    @classmethod
+    def wrap_model_if_needed(cls, model: "PreTrainedModel", rbln_config: RBLNTransformerEncoderConfig) -> nn.Module:
+        class TransformerEncoderWrapper(nn.Module):
+            # Parameters to disable for RBLN compilation
+            DISABLED_PARAMS = {"return_dict", "use_cache"}
+
+            def __init__(self, model: "PreTrainedModel", rbln_config: RBLNTransformerEncoderConfig):
+                super().__init__()
+                self.model = model
+                self.rbln_config = rbln_config
+                self._forward_signature = inspect.signature(model.forward)
+
+            def forward(self, *args, **kwargs):
+                # Disable parameters that are not compatible with RBLN compilation
+                for param_name in self.DISABLED_PARAMS:
+                    if param_name in self._forward_signature.parameters:
+                        kwargs[param_name] = False
+
+                return self.model(*args, **kwargs)
+
+        return TransformerEncoderWrapper(model, rbln_config).eval()
+
     @classmethod
     def _update_rbln_config(
         cls,
@@ -208,7 +231,6 @@ class RBLNModelForQuestionAnswering(RBLNTransformerEncoder):
 
     def _prepare_output(self, output, return_dict):
         # Prepare QuestionAnswering specific output format.
-
         start_logits, end_logits = output
 
         if not return_dict:

{optimum_rbln-0.9.2a4 → optimum_rbln-0.9.2a5}/src/optimum/rbln/transformers/models/blip_2/modeling_blip_2.py

@@ -31,6 +31,7 @@ from transformers.utils import logging
 from ....configuration_utils import RBLNCompileConfig, RBLNModelConfig
 from ....modeling import RBLNModel
 from ...utils.rbln_runtime_wrapper import LoopProcessor
+from ..decoderonly.generation_decoderonly import RBLNDecoderOnlyGenerationMixin
 
 
 logger = logging.get_logger(__name__)
@@ -265,7 +266,7 @@ class RBLNBlip2QFormerModel(RBLNModel):
         )
 
 
-class RBLNBlip2ForConditionalGeneration(RBLNModel):
+class RBLNBlip2ForConditionalGeneration(RBLNModel, RBLNDecoderOnlyGenerationMixin):
     """
     RBLNBlip2ForConditionalGeneration is a multi-modal model that integrates vision and language processing capabilities,
     optimized for RBLN NPUs. It is designed for conditional generation tasks that involve both image and text inputs.
@@ -433,3 +434,66 @@ class RBLNBlip2ForConditionalGeneration(RBLNModel):
             )
 
         return inputs_embeds
+
+    @torch.no_grad()
+    def generate(
+        self,
+        pixel_values: torch.FloatTensor,
+        input_ids: Optional[torch.LongTensor] = None,
+        attention_mask: Optional[torch.LongTensor] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        interpolate_pos_encoding: bool = False,
+        **generate_kwargs,
+    ) -> torch.LongTensor:
+        batch_size = pixel_values.shape[0]
+        image_embeds = self.vision_model(
+            pixel_values,
+            return_dict=True,
+            interpolate_pos_encoding=interpolate_pos_encoding,
+        ).last_hidden_state
+        image_attention_mask = torch.ones(image_embeds.size()[:-1], dtype=torch.long, device=image_embeds.device)
+
+        query_tokens = self.query_tokens.expand(image_embeds.shape[0], -1, -1)
+        query_outputs = self.qformer(
+            query_embeds=query_tokens,
+            encoder_hidden_states=image_embeds,
+            encoder_attention_mask=image_attention_mask,
+            return_dict=True,
+        )
+        query_output = query_outputs.last_hidden_state
+
+        if query_output.dtype != image_embeds.dtype:
+            query_output = query_output.to(image_embeds.dtype)
+
+        language_model_inputs = self.language_projection(query_output)
+
+        if inputs_embeds is None:
+            if input_ids is None:
+                image_tokens = [self.config.image_token_index] * self.config.num_query_tokens
+                start_tokens = image_tokens + [self.config.text_config.bos_token_id]
+                input_ids = torch.tensor([start_tokens], dtype=torch.long, device=image_embeds.device)
+                input_ids = input_ids.repeat(batch_size, 1)
+            inputs_embeds = self.get_input_embeddings()(input_ids)
+
+        if attention_mask is None:
+            attention_mask = torch.ones_like(input_ids)
+
+        if input_ids is None:
+            special_image_mask = inputs_embeds == self.get_input_embeddings()(
+                torch.tensor(self.config.image_token_id, dtype=torch.long, device=inputs_embeds.device)
+            )
+            special_image_mask = special_image_mask.all(-1)
+        else:
+            special_image_mask = input_ids == self.config.image_token_id
+
+        special_image_mask = special_image_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device)
+        language_model_inputs = language_model_inputs.to(inputs_embeds.device, inputs_embeds.dtype)
+        inputs_embeds = inputs_embeds.masked_scatter(special_image_mask, language_model_inputs)
+
+        inputs = {"inputs_embeds": inputs_embeds, "attention_mask": attention_mask}
+        if not self.language_model.config.is_encoder_decoder:
+            inputs["input_ids"] = input_ids
+
+        outputs = self.language_model.generate(**inputs, **generate_kwargs)
+
+        return outputs

{optimum_rbln-0.9.2a4 → optimum_rbln-0.9.2a5}/src/optimum/rbln/transformers/models/decoderonly/modeling_decoderonly.py

@@ -19,7 +19,7 @@ from typing import TYPE_CHECKING, Any, Callable, List, Optional, Tuple, Union
 import rebel
 import torch
 from rebel.compile_context import CompileContext
-from transformers import AutoModel, AutoModelForCausalLM, PretrainedConfig, PreTrainedModel
+from transformers import AutoConfig, AutoModel, AutoModelForCausalLM, PretrainedConfig, PreTrainedModel
 from transformers.modeling_outputs import BaseModelOutputWithPast
 from transformers.modeling_utils import no_init_weights
 
@@ -317,12 +317,27 @@ class RBLNDecoderOnlyModel(RBLNModel, RBLNDecoderOnlyFlashAttentionMixin):
 
     @classmethod
     def get_pytorch_model(
-        cls, *args, rbln_config: Optional[RBLNDecoderOnlyModelConfig] = None, **kwargs
+        cls,
+        model_id: str,
+        *args,
+        rbln_config: Optional[RBLNDecoderOnlyModelConfig] = None,
+        num_hidden_layers: Optional[int] = None,
+        **kwargs,
     ) -> PreTrainedModel:
         if rbln_config and rbln_config.quantization:
-            model = cls.get_quantized_model(*args, rbln_config=rbln_config, **kwargs)
+            model = cls.get_quantized_model(model_id, *args, rbln_config=rbln_config, **kwargs)
         else:
-            model = super().get_pytorch_model(*args, **kwargs)
+            if num_hidden_layers is not None:
+                trust_remote_code = kwargs.get("trust_remote_code", None)
+                config, kwargs = AutoConfig.from_pretrained(
+                    model_id, return_unused_kwargs=True, num_hidden_layers=num_hidden_layers, **kwargs
+                )
+                if hasattr(config, "layer_types"):
+                    config.layer_types = config.layer_types[:num_hidden_layers]
+                kwargs["config"] = config
+                kwargs["trust_remote_code"] = trust_remote_code
+
+            model = super().get_pytorch_model(model_id, *args, **kwargs)
 
         return model
 

{optimum_rbln-0.9.2a4 → optimum_rbln-0.9.2a5}/src/optimum/rbln/transformers/models/gemma3/modeling_gemma3.py

@@ -28,6 +28,7 @@ from ....modeling import RBLNModel
 from ...modeling_outputs import RBLNDecoderOnlyOutput
 from ...utils.rbln_runtime_wrapper import LoopProcessor
 from ..decoderonly.decoderonly_runtime_utils import RBLNPageTableManager
+from ..decoderonly.generation_decoderonly import RBLNDecoderOnlyGenerationMixin
 from ..decoderonly.modeling_decoderonly import (
     RBLNDecoderOnlyModelForCausalLM,
 )
@@ -77,7 +78,7 @@ class LoopProjector(LoopProcessor):
         return output[0]
 
 
-class RBLNGemma3ForConditionalGeneration(RBLNModel):
+class RBLNGemma3ForConditionalGeneration(RBLNModel, RBLNDecoderOnlyGenerationMixin):
     auto_model_class = AutoModelForImageTextToText
     _rbln_submodules = [
         {"name": "vision_tower"},

{optimum_rbln-0.9.2a4 → optimum_rbln-0.9.2a5}/src/optimum/rbln/transformers/models/idefics3/modeling_idefics3.py

@@ -35,6 +35,7 @@ from ....configuration_utils import RBLNCompileConfig, RBLNModelConfig
 from ....modeling import RBLNModel
 from ....utils.runtime_utils import RBLNPytorchRuntime
 from ...modeling_outputs import RBLNDecoderOnlyOutput
+from ..decoderonly.generation_decoderonly import RBLNDecoderOnlyGenerationMixin
 
 
 if TYPE_CHECKING:
@@ -120,9 +121,6 @@ class RBLNIdefics3VisionTransformer(RBLNModel):
                 encoder_outputs = self.encoder(
                     inputs_embeds=hidden_states,
                     attention_mask=patch_attention_mask,
-                    output_attentions=None,
-                    output_hidden_states=None,
-                    return_dict=False,
                 )
                 last_hidden_state = encoder_outputs[0]
                 last_hidden_state = self.post_layernorm(last_hidden_state)
@@ -185,7 +183,7 @@ class RBLNIdefics3VisionTransformer(RBLNModel):
             return BaseModelOutput(last_hidden_state=last_hidden_state)
 
 
-class RBLNIdefics3ForConditionalGeneration(RBLNModel):
+class RBLNIdefics3ForConditionalGeneration(RBLNModel, RBLNDecoderOnlyGenerationMixin):
     """
     RBLNIdefics3ForConditionalGeneration is a multi-modal model that integrates vision and language processing capabilities,
     optimized for RBLN NPUs. It is designed for conditional generation tasks that involve both image and text inputs.

{optimum_rbln-0.9.2a4 → optimum_rbln-0.9.2a5}/src/optimum/rbln/transformers/models/llava/modeling_llava.py

@@ -27,6 +27,7 @@ from ....modeling import RBLNModel
 from ....utils.logging import get_logger
 from ...modeling_outputs import RBLNDecoderOnlyOutput
 from ...utils.rbln_runtime_wrapper import LoopProcessor
+from ..decoderonly.generation_decoderonly import RBLNDecoderOnlyGenerationMixin
 
 
 logger = get_logger(__name__)
@@ -103,7 +104,7 @@ class LoopProjector(LoopProcessor):
         return output[0]
 
 
-class RBLNLlavaForConditionalGeneration(RBLNModel):
+class RBLNLlavaForConditionalGeneration(RBLNModel, RBLNDecoderOnlyGenerationMixin):
     """
     RBLNLlavaForConditionalGeneration is a multi-modal model that combines vision and language processing capabilities,
     optimized for RBLN NPUs. It is designed for conditional generation tasks that involve both image and text inputs.

{optimum_rbln-0.9.2a4 → optimum_rbln-0.9.2a5}/src/optimum/rbln/transformers/models/llava_next/modeling_llava_next.py

@@ -32,6 +32,7 @@ from ....configuration_utils import RBLNCompileConfig, RBLNModelConfig
 from ....modeling import RBLNModel
 from ....utils.logging import get_logger
 from ...utils.rbln_runtime_wrapper import LoopProcessor
+from ..decoderonly.generation_decoderonly import RBLNDecoderOnlyGenerationMixin
 from ..decoderonly.modeling_decoderonly import RBLNDecoderOnlyOutput
 
 
@@ -87,7 +88,7 @@ class LoopProjector(LoopProcessor):
         return output[0]
 
 
-class RBLNLlavaNextForConditionalGeneration(RBLNModel):
+class RBLNLlavaNextForConditionalGeneration(RBLNModel, RBLNDecoderOnlyGenerationMixin):
     """
     RBLNLlavaNextForConditionalGeneration is a multi-modal model that combines vision and language processing capabilities,
     optimized for RBLN NPUs. It is designed for conditional generation tasks that involve both image and text inputs.

{optimum_rbln-0.9.2a4 → optimum_rbln-0.9.2a5}/src/optimum/rbln/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py

@@ -400,15 +400,6 @@ class RBLNQwen2_5_VLForConditionalGeneration(RBLNDecoderOnlyModelForCausalLM):
         del model.lm_head
         return model
 
-    @classmethod
-    def update_kwargs(cls, kwargs):
-        kwargs.update(
-            {
-                "_attn_implementation": "eager",
-            }
-        )
-        return super().update_kwargs(kwargs)
-
     @classmethod
     def get_input_info(
         cls,

{optimum_rbln-0.9.2a4 → optimum_rbln-0.9.2a5}/src/optimum/rbln/transformers/models/seq2seq/modeling_seq2seq.py

@@ -20,6 +20,7 @@ import rebel
 import torch
 from rebel.compile_context import CompileContext
 from transformers import AutoModelForSeq2SeqLM, PretrainedConfig, PreTrainedModel
+from transformers.generation.utils import GenerationMixin
 from transformers.modeling_outputs import BaseModelOutput, Seq2SeqLMOutput
 
 from ....configuration_utils import RBLNCompileConfig
@@ -101,7 +102,7 @@ class RBLNRuntimeDecoder(RBLNPytorchRuntime):
         return Seq2SeqLMOutput(logits=lm_logits)
 
 
-class RBLNModelForSeq2SeqLM(RBLNModel, ABC):
+class RBLNModelForSeq2SeqLM(RBLNModel, GenerationMixin, ABC):
     """
     This is a generic model class that will be instantiated as one of the model classes of the library (with a sequence-to-sequence language modeling head) when created with the from_pretrained() class method.
     This model inherits from [`RBLNModel`]. Check the superclass documentation for the generic methods the library implements for all its models.
@@ -117,6 +118,7 @@ class RBLNModelForSeq2SeqLM(RBLNModel, ABC):
     main_input_name = "input_ids"
     auto_model_class = AutoModelForSeq2SeqLM
     support_causal_attn = None
+    _is_stateful = False
 
     def __post_init__(self, **kwargs):
         batch_size = self.rbln_config.batch_size

{optimum_rbln-0.9.2a4 → optimum_rbln-0.9.2a5}/src/optimum/rbln/transformers/models/whisper/modeling_whisper.py

@@ -150,7 +150,8 @@ class RBLNWhisperForConditionalGeneration(RBLNModel, RBLNWhisperGenerationMixin)
     """
 
     auto_model_class = AutoModelForSpeechSeq2Seq
-    main_input_name = "input_ids"
+    main_input_name = "input_features"
+    _is_stateful = False
 
     def __post_init__(self, **kwargs):
         super().__post_init__(**kwargs)