optimum-rbln 0.7.4a6__tar.gz → 0.7.4a8__tar.gz

{optimum_rbln-0.7.4a6 → optimum_rbln-0.7.4a8}/.github/ISSUE_TEMPLATE/model_request.md

@@ -8,7 +8,7 @@ assignees: ''
 
 ### Model Information
 - Model name: 
-- Model link (Hugging Face/Paper/GitHub): 
+- Model link (HuggingFace/Paper/GitHub): 
 - Task type (e.g., Text Generation, Image Classification): 
 
 ### Model Details

{optimum_rbln-0.7.4a6 → optimum_rbln-0.7.4a8}/.github/scripts/auto_code_review.py

@@ -36,7 +36,7 @@ def get_pr_diff():
 
 
 def get_prompt(diff, pr):
-    system_prompt = """You are an experienced software engineer specializing in code reviews for deep learning libraries. Your task is to review code changes and related pull request (PR) information for `optimum-rbln`, a Python library that optimizes Hugging Face models for execution on RBLN NPUs.
+    system_prompt = """You are an experienced software engineer specializing in code reviews for deep learning libraries. Your task is to review code changes and related pull request (PR) information for `optimum-rbln`, a Python library that optimizes HuggingFace models for execution on RBLN NPUs.
 
 Focus on providing actionable and constructive feedback. Don't make generalized suggestions."""
 

{optimum_rbln-0.7.4a6 → optimum_rbln-0.7.4a8}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: optimum-rbln
-Version: 0.7.4a6
-Summary: Optimum RBLN is the interface between the Hugging Face Transformers and Diffusers libraries and RBLN accelerators. It provides a set of tools enabling easy model loading and inference on single and multiple rbln device settings for different downstream tasks.
+Version: 0.7.4a8
+Summary: Optimum RBLN is the interface between the HuggingFace Transformers and Diffusers libraries and RBLN accelerators. It provides a set of tools enabling easy model loading and inference on single and multiple rbln device settings for different downstream tasks.
 Project-URL: Homepage, https://rebellions.ai
 Project-URL: Documentation, https://docs.rbln.ai
 Project-URL: Repository, https://github.com/rebellions-sw/optimum-rbln

optimum_rbln-0.7.4a8/examples/image-to-text/run_idefics3.py

@@ -0,0 +1,67 @@
+import os
+import typing
+
+import fire
+from datasets import load_dataset
+from transformers import AutoProcessor
+
+from optimum.rbln import RBLNIdefics3ForConditionalGeneration
+
+
+def main(
+    model_id: str = "HuggingFaceM4/Idefics3-8B-Llama3",
+    batch_size: int = 1,
+    from_transformers: bool = False,
+    prompt: typing.Optional[str] = None,
+    max_seq_len: typing.Optional[int] = None,
+    tensor_parallel_size: typing.Optional[int] = 4,
+):
+    processor = AutoProcessor.from_pretrained(model_id)
+
+    if from_transformers:
+        model = RBLNIdefics3ForConditionalGeneration.from_pretrained(
+            model_id,
+            export=True,
+            rbln_config={
+                "text_model": {
+                    "attn_impl": "flash_attn",
+                    "max_seq_len": max_seq_len,
+                    "use_inputs_embeds": True,
+                    "tensor_parallel_size": tensor_parallel_size,
+                    "batch_size": batch_size,
+                }
+            },
+        )
+        model.save_pretrained(os.path.basename(model_id))
+    else:
+        model = RBLNIdefics3ForConditionalGeneration.from_pretrained(
+            os.path.basename(model_id),
+            export=False,
+        )
+
+    ds = load_dataset("HuggingFaceM4/the_cauldron", "ai2d", split="train")
+    samples = ds.select(range(batch_size))
+    images = []
+    prompts = []
+
+    for sample in samples:
+        img = sample["images"]
+        images.append(img)
+
+        message = [{"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "Describe this image."}]}]
+        prompt = processor.apply_chat_template(message, add_generation_prompt=True)
+        prompts.append(prompt)
+
+    inputs = processor(text=prompts, images=images, return_tensors="pt", padding=True)
+    inputs = dict(inputs)
+    # Generate
+
+    generated_ids = model.generate(**inputs, max_new_tokens=500)
+    generated_texts = processor.batch_decode(generated_ids, skip_special_tokens=True)
+
+    for i, text in enumerate(generated_texts):
+        print(f"Sample {i + 1} generate:\n{text}\n")
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

{optimum_rbln-0.7.4a6 → optimum_rbln-0.7.4a8}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "optimum-rbln"
 description = """
-Optimum RBLN is the interface between the Hugging Face Transformers and Diffusers libraries and RBLN accelerators.
+Optimum RBLN is the interface between the HuggingFace Transformers and Diffusers libraries and RBLN accelerators.
 It provides a set of tools enabling easy model loading and inference on single and multiple rbln device settings for different downstream tasks.
 """
 authors = [

{optimum_rbln-0.7.4a6 → optimum_rbln-0.7.4a8}/src/optimum/rbln/__init__.py

@@ -74,6 +74,10 @@ _import_structure = {
         "RBLNGemmaForCausalLMConfig",
         "RBLNGPT2LMHeadModel",
         "RBLNGPT2LMHeadModelConfig",
+        "RBLNIdefics3VisionTransformer",
+        "RBLNIdefics3ForConditionalGeneration",
+        "RBLNIdefics3ForConditionalGenerationConfig",
+        "RBLNIdefics3VisionTransformerConfig",
         "RBLNLlamaForCausalLM",
         "RBLNLlamaForCausalLMConfig",
         "RBLNLlavaNextForConditionalGeneration",
@@ -281,6 +285,10 @@ if TYPE_CHECKING:
         RBLNGemmaForCausalLMConfig,
         RBLNGPT2LMHeadModel,
         RBLNGPT2LMHeadModelConfig,
+        RBLNIdefics3ForConditionalGeneration,
+        RBLNIdefics3ForConditionalGenerationConfig,
+        RBLNIdefics3VisionTransformer,
+        RBLNIdefics3VisionTransformerConfig,
         RBLNLlamaForCausalLM,
         RBLNLlamaForCausalLMConfig,
         RBLNLlavaNextForConditionalGeneration,

{optimum_rbln-0.7.4a6 → optimum_rbln-0.7.4a8}/src/optimum/rbln/__version__.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.7.4a6'
-__version_tuple__ = version_tuple = (0, 7, 4, 'a6')
+__version__ = version = '0.7.4a8'
+__version_tuple__ = version_tuple = (0, 7, 4, 'a8')

{optimum_rbln-0.7.4a6 → optimum_rbln-0.7.4a8}/src/optimum/rbln/configuration_utils.py

@@ -174,6 +174,14 @@ class RBLNAutoConfig:
         cls = getattr(importlib.import_module("optimum.rbln"), cls_name)
         return cls(**kwargs)
 
+    @staticmethod
+    def load_from_dict(config_dict: Dict[str, Any]) -> "RBLNModelConfig":
+        cls_name = config_dict.get("cls_name")
+        if cls_name is None:
+            raise ValueError("`cls_name` is required.")
+        cls = getattr(importlib.import_module("optimum.rbln"), cls_name)
+        return cls(**config_dict)
+
     @staticmethod
     def load(
         path: str,
@@ -195,8 +203,9 @@ class RBLNAutoConfig:
         cls, config_file = load_config(path)
 
         rbln_keys = [key for key in kwargs.keys() if key.startswith("rbln_")]
-
         rbln_runtime_kwargs = {key[5:]: kwargs.pop(key) for key in rbln_keys if key[5:] in RUNTIME_KEYWORDS}
+        rbln_submodule_kwargs = {key[5:]: kwargs.pop(key) for key in rbln_keys if key[5:] in cls.submodules}
+
         rbln_kwargs = {
             key[5:]: kwargs.pop(key)
             for key in rbln_keys
@@ -206,6 +215,14 @@ class RBLNAutoConfig:
         if len(rbln_kwargs) > 0:
             raise ValueError(f"Cannot set the following arguments: {list(rbln_kwargs.keys())}")
 
+        # Process submodule's rbln_config
+        for submodule in cls.submodules:
+            if submodule not in config_file:
+                raise ValueError(f"Submodule {submodule} not found in rbln_config.json.")
+            submodule_config = config_file[submodule]
+            submodule_config.update(rbln_submodule_kwargs.pop(submodule, {}))
+            config_file[submodule] = RBLNAutoConfig.load_from_dict(submodule_config)
+
         if passed_rbln_config is not None:
             config_file.update(passed_rbln_config._runtime_options)
             # TODO(jongho): Reject if the passed_rbln_config has different attributes from the config_file

{optimum_rbln-0.7.4a6 → optimum_rbln-0.7.4a8}/src/optimum/rbln/diffusers/modeling_diffusers.py

@@ -47,17 +47,11 @@ class RBLNDiffusionMixin:
 
     1. Create a new pipeline class that inherits from both this mixin and the original StableDiffusionPipeline.
     2. Define the required _submodules class variable listing the components to be compiled.
-    3. If needed, implement get_default_rbln_config for custom configuration of submodules.
 
     Example:
         ```python
         class RBLNStableDiffusionPipeline(RBLNDiffusionMixin, StableDiffusionPipeline):
             _submodules = ["text_encoder", "unet", "vae"]
-
-            @classmethod
-            def get_default_rbln_config(cls, model, submodule_name, rbln_config):
-                # Configuration for other submodules...
-                pass
         ```
 
     Class Variables:
@@ -77,14 +71,6 @@ class RBLNDiffusionMixin:
     _prefix = {}
     _rbln_config_class = None
 
-    @classmethod
-    def is_img2img_pipeline(cls):
-        return "Img2Img" in cls.__name__
-
-    @classmethod
-    def is_inpaint_pipeline(cls):
-        return "Inpaint" in cls.__name__
-
     @staticmethod
     def _maybe_apply_and_fuse_lora(
         model: torch.nn.Module,
@@ -151,7 +137,47 @@ class RBLNDiffusionMixin:
         lora_weights_names: Optional[Union[str, List[str]]] = None,
         lora_scales: Optional[Union[float, List[float]]] = None,
         **kwargs,
-    ) -> RBLNModel:
+    ) -> "RBLNDiffusionMixin":
+        """
+        Load a pretrained diffusion pipeline from a model checkpoint, with optional compilation for RBLN NPUs.
+
+        This method has two distinct operating modes:
+        - When `export=True`: Takes a PyTorch-based diffusion model, compiles it for RBLN NPUs, and loads the compiled model
+        - When `export=False`: Loads an already compiled RBLN model from `model_id` without recompilation
+
+        It supports various diffusion pipelines including Stable Diffusion, Kandinsky, ControlNet, and other diffusers-based models.
+
+        Args:
+            model_id (`str`):
+                The model ID or path to the pretrained model to load. Can be either:
+                - A model ID from the HuggingFace Hub
+                - A local path to a saved model directory
+            export (`bool`, *optional*, defaults to `False`):
+                If True, takes a PyTorch model from `model_id` and compiles it for RBLN NPU execution.
+                If False, loads an already compiled RBLN model from `model_id` without recompilation.
+            model_save_dir (`os.PathLike`, *optional*):
+                Directory to save the compiled model artifacts. Only used when `export=True`.
+                If not provided and `export=True`, a temporary directory is used.
+            rbln_config (`Dict[str, Any]`, *optional*, defaults to `{}`):
+                Configuration options for RBLN compilation. Can include settings for specific submodules
+                such as `text_encoder`, `unet`, and `vae`. Configuration can be tailored to the specific
+                pipeline being compiled.
+            lora_ids (`str` or `List[str]`, *optional*):
+                LoRA adapter ID(s) to load and apply before compilation. LoRA weights are fused
+                into the model weights during compilation. Only used when `export=True`.
+            lora_weights_names (`str` or `List[str]`, *optional*):
+                Names of specific LoRA weight files to load, corresponding to lora_ids. Only used when `export=True`.
+            lora_scales (`float` or `List[float]`, *optional*):
+                Scaling factor(s) to apply to the LoRA adapter(s). Only used when `export=True`.
+            **kwargs:
+                Additional arguments to pass to the underlying diffusion pipeline constructor or the
+                RBLN compilation process. These may include parameters specific to individual submodules
+                or the particular diffusion pipeline being used.
+
+        Returns:
+            `RBLNDiffusionMixin`: A compiled or loaded diffusion pipeline that can be used for inference on RBLN NPU.
+            The returned object is an instance of the class that called this method, inheriting from RBLNDiffusionMixin.
+        """
         rbln_config, kwargs = cls.get_rbln_config_class().initialize_from_kwargs(rbln_config, **kwargs)
 
         if export:

{optimum_rbln-0.7.4a6 → optimum_rbln-0.7.4a8}/src/optimum/rbln/modeling_base.py

@@ -216,6 +216,7 @@ class RBLNBaseModel(SubModulesMixin, PushToHubMixin, PreTrainedModel):
             if isinstance(rbln_config, dict):
                 rbln_config_as_kwargs = {f"rbln_{key}": value for key, value in rbln_config.items()}
                 kwargs.update(rbln_config_as_kwargs)
+                rbln_config = None
             elif isinstance(rbln_config, RBLNModelConfig) and rbln_config.rbln_model_cls_name != cls.__name__:
                 raise ValueError(
                     f"Cannot use the passed rbln_config. Its model class name ({rbln_config.rbln_model_cls_name}) "
@@ -392,13 +393,13 @@ class RBLNBaseModel(SubModulesMixin, PushToHubMixin, PreTrainedModel):
     @classmethod
     def get_hf_class(cls):
         """
-        Lazily loads and caches the corresponding Hugging Face model class.
+        Lazily loads and caches the corresponding HuggingFace model class.
         Removes 'RBLN' prefix from the class name to get the original class name
         (e.g., RBLNLlamaForCausalLM -> LlamaForCausalLM) and imports it from
         the transformers/diffusers module.
 
         Returns:
-            type: The original Hugging Face model class
+            type: The original HuggingFace model class
         """
         if cls._hf_class is None:
             hf_cls_name = cls.__name__[4:]
@@ -478,7 +479,7 @@ class RBLNBaseModel(SubModulesMixin, PushToHubMixin, PreTrainedModel):
             save_directory (`Union[str, Path]`):
                 Directory where to save the model file.
             push_to_hub (`bool`, *optional*, defaults to `False`):
-                Whether or not to push your model to the Hugging Face model hub after saving it.
+                Whether or not to push your model to the HuggingFace model hub after saving it.
 
         """
         if os.path.isfile(save_directory):

{optimum_rbln-0.7.4a6 → optimum_rbln-0.7.4a8}/src/optimum/rbln/transformers/__init__.py

@@ -68,6 +68,10 @@ _import_structure = {
         "RBLNGemmaForCausalLMConfig",
         "RBLNGPT2LMHeadModel",
         "RBLNGPT2LMHeadModelConfig",
+        "RBLNIdefics3VisionTransformer",
+        "RBLNIdefics3ForConditionalGeneration",
+        "RBLNIdefics3ForConditionalGenerationConfig",
+        "RBLNIdefics3VisionTransformerConfig",
         "RBLNLlamaForCausalLM",
         "RBLNLlamaForCausalLMConfig",
         "RBLNLlavaNextForConditionalGeneration",
@@ -169,6 +173,10 @@ if TYPE_CHECKING:
         RBLNGemmaForCausalLMConfig,
         RBLNGPT2LMHeadModel,
         RBLNGPT2LMHeadModelConfig,
+        RBLNIdefics3ForConditionalGeneration,
+        RBLNIdefics3ForConditionalGenerationConfig,
+        RBLNIdefics3VisionTransformer,
+        RBLNIdefics3VisionTransformerConfig,
         RBLNLlamaForCausalLM,
         RBLNLlamaForCausalLMConfig,
         RBLNLlavaNextForConditionalGeneration,

{optimum_rbln-0.7.4a6 → optimum_rbln-0.7.4a8}/src/optimum/rbln/transformers/models/__init__.py

@@ -73,6 +73,12 @@ _import_structure = {
     "exaone": ["RBLNExaoneForCausalLM", "RBLNExaoneForCausalLMConfig"],
     "gemma": ["RBLNGemmaForCausalLM", "RBLNGemmaForCausalLMConfig"],
     "gpt2": ["RBLNGPT2LMHeadModel", "RBLNGPT2LMHeadModelConfig"],
+    "idefics3": [
+        "RBLNIdefics3VisionTransformer",
+        "RBLNIdefics3ForConditionalGeneration",
+        "RBLNIdefics3ForConditionalGenerationConfig",
+        "RBLNIdefics3VisionTransformerConfig",
+    ],
     "llama": ["RBLNLlamaForCausalLM", "RBLNLlamaForCausalLMConfig"],
     "llava_next": ["RBLNLlavaNextForConditionalGeneration", "RBLNLlavaNextForConditionalGenerationConfig"],
     "midm": ["RBLNMidmLMHeadModel", "RBLNMidmLMHeadModelConfig"],
@@ -144,6 +150,12 @@ if TYPE_CHECKING:
     from .exaone import RBLNExaoneForCausalLM, RBLNExaoneForCausalLMConfig
     from .gemma import RBLNGemmaForCausalLM, RBLNGemmaForCausalLMConfig
     from .gpt2 import RBLNGPT2LMHeadModel, RBLNGPT2LMHeadModelConfig
+    from .idefics3 import (
+        RBLNIdefics3ForConditionalGeneration,
+        RBLNIdefics3ForConditionalGenerationConfig,
+        RBLNIdefics3VisionTransformer,
+        RBLNIdefics3VisionTransformerConfig,
+    )
     from .llama import RBLNLlamaForCausalLM, RBLNLlamaForCausalLMConfig
     from .llava_next import RBLNLlavaNextForConditionalGeneration, RBLNLlavaNextForConditionalGenerationConfig
     from .midm import RBLNMidmLMHeadModel, RBLNMidmLMHeadModelConfig

{optimum_rbln-0.7.4a6 → optimum_rbln-0.7.4a8}/src/optimum/rbln/transformers/models/auto/auto_factory.py

@@ -48,7 +48,7 @@ class _BaseAutoModelClass:
 
         Args:
             pretrained_model_name_or_path (str): Identifier or path to the pretrained model.
-            export (bool): Whether to infer the class based on Hugging Face (HF) architecture.
+            export (bool): Whether to infer the class based on HuggingFace (HF) architecture.
             kwargs: Additional arguments for configuration and loading.
 
         Returns:
@@ -86,14 +86,14 @@ class _BaseAutoModelClass:
         **kwargs,
     ):
         """
-        Infer the Hugging Face model class based on the configuration or model name.
+        Infer the HuggingFace model class based on the configuration or model name.
 
         Args:
             pretrained_model_name_or_path (str): Identifier or path to the pretrained model.
             kwargs: Additional arguments for configuration and loading.
 
         Returns:
-            PretrainedModel: The inferred Hugging Face model class.
+            PretrainedModel: The inferred HuggingFace model class.
         """
 
         # Try to load configuration if provided or retrieve it from the model ID

{optimum_rbln-0.7.4a6 → optimum_rbln-0.7.4a8}/src/optimum/rbln/transformers/models/decoderonly/decoderonly_architecture.py

@@ -184,6 +184,7 @@ class DecoderOnlyWrapper(nn.Module):
 
     def convert_to_rbln_causal_lm(self, causal_lm: PreTrainedModel, max_seq_len: int):
         new_layers = []
+
         for layer in causal_lm.model.layers:
             if self.attn_impl == "eager":
                 new_self_attn = DecoderOnlyAttention(
@@ -201,6 +202,7 @@ class DecoderOnlyWrapper(nn.Module):
 
             new_layer = DecoderOnlyLayer(layer, new_self_attn)
             new_layers.append(new_layer)
+
         new_model = DecoderOnlyModel(
             causal_lm.model,
             new_layers,

{optimum_rbln-0.7.4a6 → optimum_rbln-0.7.4a8}/src/optimum/rbln/transformers/models/decoderonly/modeling_decoderonly.py

@@ -451,6 +451,12 @@ class RBLNDecoderOnlyModelForCausalLM(RBLNModel):
     def get_input_embeddings(self):
         return self.embed_tokens
 
+    def get_attn_impl(self) -> str:
+        return self.rbln_config.attn_impl
+
+    def get_kvcache_num_blocks(self) -> int:
+        return self.rbln_config.kvcache_num_blocks
+
     @classmethod
     def get_quantized_model(
         cls,

optimum_rbln-0.7.4a8/src/optimum/rbln/transformers/models/idefics3/__init__.py

@@ -0,0 +1,16 @@
+# Copyright 2025 Rebellions Inc. All rights reserved.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .configuration_idefics3 import RBLNIdefics3ForConditionalGenerationConfig, RBLNIdefics3VisionTransformerConfig
+from .modeling_idefics3 import RBLNIdefics3ForConditionalGeneration, RBLNIdefics3VisionTransformer

optimum_rbln-0.7.4a8/src/optimum/rbln/transformers/models/idefics3/configuration_idefics3.py

@@ -0,0 +1,51 @@
+# Copyright 2025 Rebellions Inc. All rights reserved.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Optional
+
+from ....configuration_utils import RBLNModelConfig
+
+
+class RBLNIdefics3VisionTransformerConfig(RBLNModelConfig):
+    pass
+
+
+class RBLNIdefics3ForConditionalGenerationConfig(RBLNModelConfig):
+    submodules = ["vision_model", "text_model"]
+
+    def __init__(
+        self,
+        batch_size: Optional[int] = None,
+        vision_model: Optional[RBLNModelConfig] = None,
+        text_model: Optional[RBLNModelConfig] = None,
+        **kwargs,
+    ):
+        """
+        Args:
+            batch_size (Optional[int]): The batch size for inference. Defaults to 1.
+            vision_model (Optional[RBLNModelConfig]): Configuration for the vision transformer component.
+            text_model (Optional[RBLNModelConfig]): Configuration for the text model component.
+            **kwargs: Additional arguments passed to the parent RBLNModelConfig.
+
+        Raises:
+            ValueError: If batch_size is not a positive integer.
+        """
+
+        super().__init__(**kwargs)
+        self.batch_size = batch_size or 1
+        if not isinstance(self.batch_size, int) or self.batch_size < 0:
+            raise ValueError(f"batch_size must be a positive integer, got {self.batch_size}")
+
+        self.vision_model = vision_model
+        self.text_model = text_model