optimum-rbln 0.7.4a5__py3-none-any.whl → 0.7.4a6__py3-none-any.whl

optimum/rbln/transformers/models/qwen2_5_vl/qwen2_5_vl_architecture.py

@@ -0,0 +1,214 @@
+import math
+from typing import Tuple
+
+import torch
+import torch.nn as nn
+
+from ..decoderonly.decoderonly_architecture import (
+    DecoderOnlyWrapper,
+    apply_rotary_pos_emb,
+)
+
+
+class Qwen2_5_VisionTransformerWrapper(nn.Module):
+    def __init__(self, model: torch.nn.Module):
+        super().__init__()
+        self._original_mod = model
+        self.fullatt_block_indexes = model.fullatt_block_indexes
+        self.merger = model.merger
+        window_seq_len = (model.window_size // model.patch_size) ** 2
+        self.blocks = self.wrap_vision_blocks(model.blocks, window_seq_len)
+
+    def wrap_vision_blocks(self, blocks: torch.nn.ModuleList, window_seq_len: int):
+        wrapped_blocks = []
+        for i, block in enumerate(blocks):
+            is_full_attn = True if i in self.fullatt_block_indexes else False
+            wrapped_blocks.append(Qwen2_5_VLVisionBlock(block, is_full_attn, window_seq_len))
+        return nn.ModuleList(wrapped_blocks)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        full_attn_masks: torch.Tensor,
+        window_attn_masks: torch.Tensor,
+        cos: torch.Tensor,
+        sin: torch.Tensor,
+    ):
+        full_attn_masks = (1 - full_attn_masks) * torch.finfo(torch.float32).min
+        window_attn_masks = (1 - window_attn_masks) * torch.finfo(torch.float32).min
+
+        for i, block in enumerate(self.blocks):
+            attn_masks = full_attn_masks if i in self.fullatt_block_indexes else window_attn_masks
+            hidden_states = block(hidden_states, attn_masks, [cos, sin])
+
+        hidden_states = self.merger(hidden_states)
+
+        return hidden_states
+
+
+class Qwen2_5_VLVisionBlock(torch.nn.Module):
+    def __init__(self, model: torch.nn.Module, is_full_attn: bool, window_seq_len: int):
+        super().__init__()
+        self._origin_model = model
+        self.norm1 = model.norm1
+        self.norm2 = model.norm2
+
+        if is_full_attn:
+            self.attn = Qwen2_5_VLVisionFullAttention(model.attn)
+        else:
+            self.attn = Qwen2_5_VLVisionWindowAttention(model.attn, window_seq_len)
+        self.mlp = model.mlp
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attn_masks: torch.Tensor,
+        position_embeddings: Tuple[torch.Tensor, torch.Tensor],
+    ) -> torch.Tensor:
+        hidden_states = hidden_states + self.attn(
+            self.norm1(hidden_states),
+            attn_masks,
+            position_embeddings,
+        )
+        hidden_states = hidden_states + self.mlp(self.norm2(hidden_states))
+        return hidden_states
+
+
+class Qwen2_5_VLVisionFullAttention(nn.Module):
+    def __init__(self, model: nn.Module) -> None:
+        super().__init__()
+        self._origin_model = model
+        self.num_heads = model.num_heads
+        self.head_dim = model.head_dim
+        self.qkv = model.qkv
+        self.proj = model.proj
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attn_masks: torch.Tensor,
+        position_embeddings: Tuple[torch.Tensor, torch.Tensor],
+    ) -> torch.Tensor:
+        seq_length = hidden_states.shape[0]
+        hidden_states = hidden_states.unsqueeze(0)
+        q, k, v = (
+            self.qkv(hidden_states).reshape(1, seq_length, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4).unbind(0)
+        )
+
+        cos, sin = position_embeddings
+        q, k = apply_rotary_pos_emb(q, k, cos, sin)
+
+        attn_weights = torch.matmul(q, k.transpose(2, 3)) / math.sqrt(self.head_dim)
+        attn_weights = attn_weights + attn_masks
+        attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32)
+        attn_output = torch.matmul(attn_weights, v)
+        attn_output = attn_output.transpose(1, 2)
+        attn_output = attn_output.reshape(1, seq_length, -1)
+        attn_output = self.proj(attn_output).squeeze(0)
+
+        return attn_output
+
+
+class Qwen2_5_VLVisionWindowAttention(nn.Module):
+    def __init__(self, model: nn.Module, window_seq_len: int) -> None:
+        super().__init__()
+        self._origin_model = model
+        self.num_heads = model.num_heads
+        self.head_dim = model.head_dim
+        self.qkv = model.qkv
+        self.proj = model.proj
+        self.window_seq_len = window_seq_len
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attn_masks: torch.Tensor,
+        position_embeddings: Tuple[torch.Tensor, torch.Tensor],
+    ) -> torch.Tensor:
+        seq_length = hidden_states.shape[0]
+        num_windows = seq_length // self.window_seq_len
+
+        window_hidden_states = []
+        for i in range(0, seq_length, self.window_seq_len):
+            window_hidden_states.append(hidden_states[i : i + self.window_seq_len])
+        hidden_states = torch.stack(window_hidden_states)
+
+        q, k, v = (
+            self.qkv(hidden_states)
+            .reshape(num_windows, self.window_seq_len, 3, self.num_heads, -1)
+            .permute(2, 0, 3, 1, 4)
+            .unbind(0)
+        )
+        cos, sin = position_embeddings
+        cos = cos.reshape(num_windows, 1, seq_length // num_windows, -1)
+        sin = sin.reshape(num_windows, 1, seq_length // num_windows, -1)
+        q, k = apply_rotary_pos_emb(q, k, cos, sin)
+
+        attn_weights = torch.matmul(q, k.transpose(2, 3)) / math.sqrt(self.head_dim)
+
+        attn_weights = attn_weights + attn_masks
+        attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32)
+        attn_output = torch.matmul(attn_weights, v)
+        attn_output = attn_output.transpose(1, 2)
+        attn_output = attn_output.reshape(1, seq_length, -1)
+        attn_output = self.proj(attn_output).squeeze(0)
+
+        return attn_output
+
+
+class Qwen2_5_VL_LanguageModelWrapper(DecoderOnlyWrapper):
+    def forward(self, *args):
+        if self.phase == "decode":
+            if self.use_attention_mask:
+                (
+                    input_ids_or_inputs_embeds,
+                    cache_position,
+                    attention_mask,
+                    block_tables,
+                    position_emb,
+                    *past_key_values,
+                ) = args
+            else:
+                (
+                    input_ids_or_inputs_embeds,
+                    cache_position,
+                    block_tables,
+                    position_emb,
+                    *past_key_values,
+                ) = args
+                attention_mask = None
+            query_position = None
+        elif self.phase == "prefill":
+            if self.use_attention_mask:
+                (
+                    input_ids_or_inputs_embeds,
+                    cache_position,
+                    attention_mask,
+                    query_position,
+                    block_tables,
+                    position_emb,
+                    *past_key_values,
+                ) = args
+            else:
+                (
+                    input_ids_or_inputs_embeds,
+                    cache_position,
+                    query_position,
+                    block_tables,
+                    position_emb,
+                    *past_key_values,
+                ) = args
+                attention_mask = None
+
+        else:
+            raise ValueError(f"Unknown phase: {self.phase}")
+
+        return self.forward_common(
+            input_ids_or_inputs_embeds,
+            cache_position,
+            attention_mask,
+            query_position,
+            block_tables,
+            position_emb,
+            *past_key_values,
+        )

optimum/rbln/utils/runtime_utils.py

@@ -45,23 +45,54 @@ class RBLNPytorchRuntime:
 
 
 class UnavailableRuntime:
+    """
+    A placeholder class used when model runtimes are not created.
+
+    This class is returned by RBLNBaseModel._from_compiled_models when rbln_config.create_runtimes=False.
+    It provides proper error messages when users attempt to use a model that was loaded without
+    runtime creation.
+
+    Usage:
+        1. When compiling models on machines without NPU hardware
+        2. When preparing models for later deployment
+        3. When only model compilation is needed, not inference
+
+    To use a model with runtimes, either:
+        - Load the model with from_pretrained(..., rbln_create_runtimes=True)
+        - Or set rbln_config={"create_runtimes": True} during loading
+    """
+
     def __call__(self, *args: Any, **kwargs: Any) -> Any:
+        """Raises a RuntimeError when the model is called without runtimes."""
         raise self.forward(*args, **kwargs)
 
     def __len__(self) -> int:
+        """Returns 0 since no runtimes are available."""
         return 0
 
     def __getitem__(self, idx: int) -> Any:
+        """Returns self for any index, allowing iteration to work with appropriate errors."""
         return self
 
     def __iter__(self):
+        """Returns an iterator with self as the only item."""
         return iter([self])
 
     def forward(self, *args: List["torch.Tensor"], **kwargs: Dict[str, "torch.Tensor"]):
-        raise RuntimeError("The model can't run because the runtime hasn't been created.")
+        """Raises a detailed RuntimeError explaining why inference cannot be performed."""
+        raise RuntimeError(
+            "Cannot perform inference: RBLN runtime is not available.\n\n"
+            "This model was loaded with create_runtimes=False. To use this model for inference:\n"
+            "1. Load the model with runtime creation enabled:\n"
+            "   model = RBLNModel.from_pretrained(..., rbln_create_runtimes=True)\n"
+            "2. Ensure your NPU hardware is properly configured (check with 'rbln-stat' command)\n"
+            "3. If you're on a machine without NPU hardware, you need to transfer the model files\n"
+            "   to a compatible system with NPU support."
+        )
 
     def __repr__(self) -> str:
-        return "UnavailableRuntime"
+        """Returns a detailed string representation of the UnavailableRuntime."""
+        return "<UnavailableRuntime: Model loaded without runtime creation (create_runtimes=False)>"
 
 
 class ContextRblnConfig:

{optimum_rbln-0.7.4a5.dist-info → optimum_rbln-0.7.4a6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: optimum-rbln
-Version: 0.7.4a5
+Version: 0.7.4a6
 Summary: Optimum RBLN is the interface between the Hugging Face Transformers and Diffusers libraries and RBLN accelerators. It provides a set of tools enabling easy model loading and inference on single and multiple rbln device settings for different downstream tasks.
 Project-URL: Homepage, https://rebellions.ai
 Project-URL: Documentation, https://docs.rbln.ai

{optimum_rbln-0.7.4a5.dist-info → optimum_rbln-0.7.4a6.dist-info}/RECORD

@@ -1,8 +1,8 @@
-optimum/rbln/__init__.py,sha256=I1L4sFUTPi3og3Dxx3kmWEq4tza7PhQpR6Kt3ExUQs8,12374
-optimum/rbln/__version__.py,sha256=tbsOVqydiZKxDMzmIcNOjsjuX2tONzvUiapW5PDnVtU,513
+optimum/rbln/__init__.py,sha256=gwlRrI10tkLlpXt-N3FbmMeVqEJRVExtfVrxsqdcCpM,12814
+optimum/rbln/__version__.py,sha256=BD2gZveYjqvIGZe_xk184XdKxoYxjbIrrR0xZAj20HQ,519
 optimum/rbln/configuration_utils.py,sha256=1IX8AHmQbnJvhNm3yD6ZJABP7Oi40WvVSNlThty-ASk,29167
 optimum/rbln/modeling.py,sha256=qDXB69Oq0jx9hfONebDiSNe2_DgKYhnAGLTbGAtwYVw,9677
-optimum/rbln/modeling_base.py,sha256=WPA5yhgc8YHzxcicJeQ1vzs3Bxrl5dQWGKB_Inb6W3A,23090
+optimum/rbln/modeling_base.py,sha256=3JQhXDh_Rz1f6bMNxgFk-Xd-lQcAVeh-q1KVO0ijSaA,24141
 optimum/rbln/diffusers/__init__.py,sha256=XL6oKPHbPCV6IVCw3fu0-M9mD2KO_x6unx5kJdAtpVY,6180
 optimum/rbln/diffusers/modeling_diffusers.py,sha256=mOP352aYok7DQIQj1dKq1uTZwIkjmUIm-pmtlu6rIUI,16987
 optimum/rbln/diffusers/configurations/__init__.py,sha256=Sk_sQVTuTl01RVgYViWknQSLmulxKaISS0w-oPdNoBQ,1164
@@ -60,13 +60,13 @@ optimum/rbln/ops/attn.py,sha256=x02yFLk7FcONFqfow0ROmVy9fmxo5Pw0SPCiDY3AZNg,9012
 optimum/rbln/ops/flash_attn.py,sha256=NmCqUdMTzgJ4sbYGj8IWXJEsLWvbuCMponR01w5DK6w,4121
 optimum/rbln/ops/kv_cache_update.py,sha256=HjnHBR-oFrJQibsVnkYb0P5_-wEma8jl0mkjkylwakU,1270
 optimum/rbln/ops/linear.py,sha256=1_7Hg-9wXxhu97fqPobotLQx17k7VPeSSL91_9Z7EDg,1018
-optimum/rbln/transformers/__init__.py,sha256=Ey2Et6TFYzwCncC3m1rC-UHTL4Zb-HC3jZfgvhfOssI,7272
+optimum/rbln/transformers/__init__.py,sha256=9IzUXwbMZ1cGFNa25wvoBs42UkQrvZNgvlbZ8zjOTOs,7712
 optimum/rbln/transformers/configuration_alias.py,sha256=qFVfg6ohsR7a6b-CBgxjBUPDrk9MyiJwtO8AQah_RTU,1505
 optimum/rbln/transformers/configuration_generic.py,sha256=XIiZ1-5p1CMHhG7Sr2qR4SLYKcYw9aph7eGlga3Opx0,5056
 optimum/rbln/transformers/modeling_alias.py,sha256=yx7FnZQWAnrWzivaO5hI7T6i-fyLzt2tMIXG2oDNbPo,1657
 optimum/rbln/transformers/modeling_generic.py,sha256=nT_lytAILkYtwBVJKxXg0dxmh0UpjGYO6zOdLoMs1uU,12891
 optimum/rbln/transformers/modeling_rope_utils.py,sha256=3zwkhYUyTZhxCJUSmwCc88iiY1TppRWEY9ShwUqNB2k,14293
-optimum/rbln/transformers/models/__init__.py,sha256=N2xll74hQvQMD2AIaiDSIqL7CHeR8UxknpRjFCx8KQU,6287
+optimum/rbln/transformers/models/__init__.py,sha256=9f2fJgdchqM_CJbx5BogQQ_xvCVtgYQ8bgyARt4lFu4,6790
 optimum/rbln/transformers/models/auto/__init__.py,sha256=GvGbb3ZpMv-h6euXeZ42jSizoOfrL2O1uvpAnfKxYEo,1034
 optimum/rbln/transformers/models/auto/auto_factory.py,sha256=jS33v5H6AY7yHZMAQPq94MBLer2wpb33IVMYcb1w57A,7047
 optimum/rbln/transformers/models/auto/modeling_auto.py,sha256=Un9qoqdy3dO8JBza_bTJF_6_fRVNM9QisihSgTRFI-o,3933
@@ -82,8 +82,8 @@ optimum/rbln/transformers/models/clip/configuration_clip.py,sha256=wgfZeVvcVdSzr
 optimum/rbln/transformers/models/clip/modeling_clip.py,sha256=UslcDN6otyQ_psou7F_YcdK5vCImEtgIdcbwmexSfOM,7256
 optimum/rbln/transformers/models/decoderonly/__init__.py,sha256=vQYZDDdoddwA7yKc5zzrq2Zs9sax-0p8rNF_aYfF4bk,1006
 optimum/rbln/transformers/models/decoderonly/configuration_decoderonly.py,sha256=b1W7zS0MUmeDd048bLp5AkZMrWd3LIhHaVy8NvlwdCw,4116
-optimum/rbln/transformers/models/decoderonly/decoderonly_architecture.py,sha256=tOW8C9_HNtve5xXPYfrbiWLJ0rjpRHTWz80oJ3j75hs,40948
-optimum/rbln/transformers/models/decoderonly/modeling_decoderonly.py,sha256=LS3ByYNJaFfwX9zEDH2b1tsbu5DraTSPV9XLGSIG0F0,38264
+optimum/rbln/transformers/models/decoderonly/decoderonly_architecture.py,sha256=BDATZTGF0J2zfc3u-zHXPKbkvsVHOwkNR10smB9Zuo8,41746
+optimum/rbln/transformers/models/decoderonly/modeling_decoderonly.py,sha256=uzw1nOz_jdi3lrBiLkkHy8QiOn-M5bCwNtyNs01a-d8,42815
 optimum/rbln/transformers/models/dpt/__init__.py,sha256=Nzep9mlzKyL1kV726IBqY8DnLp1DkH9JzFeknWSRhok,714
 optimum/rbln/transformers/models/dpt/configuration_dpt.py,sha256=4fW6bzVhaAxym4wGV3F785rvUOoWPyw_gdEMqB08Leg,755
 optimum/rbln/transformers/models/dpt/modeling_dpt.py,sha256=oKLX7MQZvfk1QB8wOtcdi7AmZH2fOIVbypa9A3RA9MI,733
@@ -122,6 +122,10 @@ optimum/rbln/transformers/models/qwen2/__init__.py,sha256=Tu4_AXy3ktTvxGwxED3kew
 optimum/rbln/transformers/models/qwen2/configuration_qwen2.py,sha256=sQBu4UjM8Ctiy696FLjKeck1t49MR31zWTCN_bMvBl4,773
 optimum/rbln/transformers/models/qwen2/modeling_qwen2.py,sha256=9-aFDvjMzPNUyGOz0qo33RE18bUFGYZ3Wt_68zb5uJY,1530
 optimum/rbln/transformers/models/qwen2/qwen2_architecture.py,sha256=XlNAMYAcDLohnSAhIFGKOPuCB5XLgzYs5ABWdeQSaZs,720
+optimum/rbln/transformers/models/qwen2_5_vl/__init__.py,sha256=rAW3DKQUzGL6EMwa5r1iLu94yhpiZpk6zfoD7TtYXrc,865
+optimum/rbln/transformers/models/qwen2_5_vl/configuration_qwen2_5_vl.py,sha256=dPcGNaLwJf61PIvVbyt-lvBflp_dvK0hubhNoA3len0,3123
+optimum/rbln/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py,sha256=sKsDAiMGWtbquTw6_PITK4ijL0aHBZqgs2nPObEiFN8,24951
+optimum/rbln/transformers/models/qwen2_5_vl/qwen2_5_vl_architecture.py,sha256=YRy7Ylm-UQLovt5BmxhayJMKzF3rj0_HIc4tUXaiPO0,7474
 optimum/rbln/transformers/models/seq2seq/__init__.py,sha256=6WKstWiS1kW0oFDn_jyrKMW5QEJAWkmsSRAaadNedDM,715
 optimum/rbln/transformers/models/seq2seq/configuration_seq2seq2.py,sha256=vSNP1eILfL32cbiLOAD58Ocz6lk3hYFnhIRLDVqlSoI,2624
 optimum/rbln/transformers/models/seq2seq/modeling_seq2seq.py,sha256=7MN6CNVyYg4JsULYXDfQ_KkDd3w-1TXuxndSrM-CX8w,16980
@@ -153,10 +157,10 @@ optimum/rbln/utils/hub.py,sha256=bNmOJGEO9Jfux4Cg8Xli-898I4mxk20KuwQOhP0Zs1U,419
 optimum/rbln/utils/import_utils.py,sha256=uMldLJmDVMj5uHvxBfb96uV29bfGEDvlksLY26GOHAs,4389
 optimum/rbln/utils/logging.py,sha256=VKKBmlQSdg6iZCGmAXaWYiW67K84jyp1QJhLQSSjPPE,3453
 optimum/rbln/utils/model_utils.py,sha256=DfD_Z2qvZHqcddXqnzTM1AN8khanj3-DXK2lJvVxDvs,1278
-optimum/rbln/utils/runtime_utils.py,sha256=5-DYniyP59nx-mrrbi7AqA77L85b4Cm5oLpaxidSyss,3699
+optimum/rbln/utils/runtime_utils.py,sha256=LoKNK3AQNV_BSScstIZWjICkJf265MnUgy360BOocVI,5454
 optimum/rbln/utils/save_utils.py,sha256=hG5uOtYmecSXZuGTvCXsTM-SiyZpr5q3InUGCCq_jzQ,3619
 optimum/rbln/utils/submodule.py,sha256=_qX15LE2nHzOFVUp5U5DqdJx11RzPBR848PR0nsjQBc,4293
-optimum_rbln-0.7.4a5.dist-info/METADATA,sha256=AaElWAA8uJqR_z1U6NQ59b0Cx9o3FJI5jlz6tOoJwQo,5300
-optimum_rbln-0.7.4a5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-optimum_rbln-0.7.4a5.dist-info/licenses/LICENSE,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
-optimum_rbln-0.7.4a5.dist-info/RECORD,,
+optimum_rbln-0.7.4a6.dist-info/METADATA,sha256=m4NMptBdyrOQ746DO16TCGBCSHeidkLBpXQQC8_EEhY,5300
+optimum_rbln-0.7.4a6.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+optimum_rbln-0.7.4a6.dist-info/licenses/LICENSE,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
+optimum_rbln-0.7.4a6.dist-info/RECORD,,