optimum-rbln 0.8.2a4__py3-none-any.whl → 0.9.3rc0__py3-none-any.whl

optimum/rbln/transformers/models/qwen3/qwen3_architecture.py

@@ -12,16 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import torch
-import torch.nn as nn
-from transformers import PreTrainedModel
 
-from ..decoderonly.decoderonly_architecture import (
-    DecoderOnlyAttention,
-    DecoderOnlyLayer,
-    DecoderOnlyWrapper,
-    RotaryEmbedding,
-)
+from ..decoderonly.decoderonly_architecture import DecoderOnlyAttention, DecoderOnlyWrapper
 
 
 class Qwen3Wrapper(DecoderOnlyWrapper):
@@ -37,239 +29,3 @@ class Qwen3Attention(DecoderOnlyAttention):
         self.o_proj = self._original_mod.o_proj
         self.q_norm = self._original_mod.q_norm
         self.k_norm = self._original_mod.k_norm
-
-
-class Qwen3ModelWrapper(nn.Module):
-    def __init__(
-        self,
-        model,
-        attn_impl=None,
-        use_inputs_embeds=None,
-        use_attention_mask=None,
-        use_rotary_emb=None,
-        cache_impl=None,
-        kvcache_partition_len=None,
-        max_seq_len=None,
-        kvcache_block_size=None,
-        sliding_window=None,
-        sliding_window_layers=None,
-    ):
-        super().__init__()
-        self.config = model.config
-
-        if use_rotary_emb:
-            rotary_embs = self.get_rotary_emb(max_seq_len=max_seq_len)
-            if isinstance(rotary_embs, tuple):
-                self.rotary_emb_global, self.rotary_emb_local = rotary_embs
-            else:
-                self.rotary_emb = rotary_embs
-        else:
-            self.rotary_emb = None
-
-        self._original_mod = model
-        self.use_inputs_embeds = use_inputs_embeds
-        self.attn_impl = attn_impl
-        self.cache_impl = cache_impl
-        self.use_attention_mask = use_attention_mask
-        self.kvcache_partition_len = kvcache_partition_len
-        self.kvcache_block_size = kvcache_block_size
-        self.max_seq_len = max_seq_len
-        self.sliding_window = sliding_window
-        self.sliding_window_layers = sliding_window_layers
-        self.model = self.convert_to_rbln_model(model)
-
-    def get_rotary_emb(self, max_seq_len):
-        return RotaryEmbedding(config=self.config, max_seq_len_cached=max_seq_len)
-
-    def convert_to_rbln_model(self, base_model: PreTrainedModel):
-        for layer_idx, layer in enumerate(base_model.layers):
-            is_sliding = layer_idx in self.sliding_window_layers
-            new_self_attn = Qwen3Attention(
-                layer.self_attn,
-                self.use_attention_mask if not is_sliding else True,
-                use_position_ids=None,
-                kvcache_block_size=self.sliding_window
-                if layer_idx in self.sliding_window_layers
-                else self.kvcache_block_size,
-                is_sliding=is_sliding,
-                attn_impl=self.attn_impl if not is_sliding else "eager",
-                kvcache_partition_len=self.kvcache_partition_len,
-            )
-            base_model.layers[layer_idx] = DecoderOnlyLayer(layer, new_self_attn)
-
-        return base_model
-
-    @property
-    def hidden_multiplier(self):
-        return 1
-
-    def get_last_layernorm(self) -> nn.LayerNorm:
-        return self._original_mod.norm
-
-    def get_embedding(self) -> nn.Embedding:
-        return self._original_mod.embed_tokens
-
-    def get_pos_embedding(self) -> nn.Embedding:
-        raise NotImplementedError(
-            "The 'get_pos_embedding' method is not implemented. Please define this method in a subclass."
-        )
-
-    def convert_sequence_positions_for_flash_attn(self, seq_positions, max_seq_len):
-        if self.attn_impl not in ["flash_attn"]:
-            raise NotImplementedError(f"Unknown attn_impl ({self.attn_impl}).")
-        partition_len = self.kvcache_partition_len
-        num_partition = max_seq_len // partition_len
-
-        cs = seq_positions.repeat(num_partition, 1).transpose(0, 1)
-        pidx = torch.arange(num_partition)
-        cache_pos_for_partitions = torch.clamp(cs - pidx * partition_len, 0, partition_len)
-        return cache_pos_for_partitions
-
-    def get_local_cache_positions(self, position_ids, query_position):
-        max_cache_len = self.model.config.sliding_window
-        valid_input_len = 1 if query_position is None else query_position + 1
-        cache_seq_len = torch.clamp(position_ids, max=max_cache_len)[:, :1]  # past seen tokens
-        cache_offset = (
-            torch.clamp(position_ids, max=max_cache_len)[:, :1] + valid_input_len
-        )  # cache offset for next steps
-
-        return cache_seq_len, cache_offset
-
-    def prepare_forward_args(self, *args):
-        args = list(args)
-        input_ids = None if self.use_inputs_embeds else args.pop(0)
-        inputs_embeds = args.pop(0) if self.use_inputs_embeds else None
-        cache_position = args.pop(0)
-        global_block_tables = args.pop(0) if self.cache_impl in ["hybrid", "static"] else None
-        local_block_tables = args.pop(0) if self.cache_impl in ["hybrid", "sliding_window"] else None
-        query_position = args.pop(0) if self.sliding_window else None
-        attention_mask = args.pop(0) if self.use_attention_mask else None
-        position_ids = None
-        past_key_values = args
-
-        if len(past_key_values) != 2 * self.config.num_hidden_layers:
-            raise ValueError(
-                f"Different past_key_values to model's config. {len(past_key_values)} != {2 * self.config.num_hidden_layers}"
-            )
-
-        # [key, value] * n_layer -> ( (key, value) ) * n_layer
-        # cache shape : batch, n_heads, 1, max_seq_len, head_dim
-        _past_key_values = []
-        for i in range(self.config.num_hidden_layers):
-            key_states = past_key_values[i * 2]
-            value_states = past_key_values[i * 2 + 1]
-            past_key_value = [key_states, value_states]
-            _past_key_values.append(past_key_value)
-        past_key_values = _past_key_values
-
-        if hasattr(self, "rotary_emb_global") and hasattr(self, "rotary_emb_local"):
-            rotary_emb = (self.rotary_emb_global, self.rotary_emb_local)
-        else:
-            rotary_emb = self.rotary_emb
-
-        return (
-            input_ids,
-            inputs_embeds,
-            cache_position,
-            global_block_tables,
-            local_block_tables,
-            attention_mask,
-            position_ids,
-            query_position,
-            past_key_values,
-            rotary_emb,
-        )
-
-    def forward(self, *args):
-        (
-            input_ids,
-            inputs_embeds,
-            cache_position,
-            global_block_tables,
-            local_block_tables,
-            attention_mask,
-            position_ids,
-            query_position,
-            past_key_values,
-            rotary_emb,
-        ) = self.prepare_forward_args(*args)
-
-        # retrieve input_ids and inputs_embeds
-        if (input_ids is None) ^ (inputs_embeds is not None):
-            raise ValueError(
-                "You cannot specify both input_ids and inputs_embeds at the same time, and must specify either one"
-            )
-
-        # embed positions
-        if inputs_embeds is None:
-            inputs_embeds = self.get_embedding()(input_ids)
-
-        hidden_states = inputs_embeds * self.hidden_multiplier
-
-        # get cos,sin vector if needed
-        position_ids = position_ids if position_ids is not None else cache_position
-        if rotary_emb is not None:
-            if isinstance(rotary_emb, torch.Tensor):
-                cos = rotary_emb[0]
-                sin = rotary_emb[1]
-            else:
-                cos, sin = rotary_emb(hidden_states, self.max_seq_len)  # dtype carrier, max_seq_len
-                cos, sin = slice_and_unsqueeze_cos_sin(cos, sin, position_ids)
-        else:
-            batch_size = inputs_embeds.shape[0]
-            if position_ids.shape[0] > 1:
-                position_embeds = []
-                for b_idx in range(batch_size):
-                    position_embed = self.get_pos_embedding()(position_ids[b_idx])
-                    position_embeds.append(position_embed)
-
-                position_embeds = torch.cat(position_embeds, dim=0).unsqueeze(1)
-            else:
-                position_embeds = self.get_pos_embedding()(position_ids)
-            hidden_states = hidden_states + position_embeds
-            cos, sin = None, None
-
-        # Get sequence positions for flash attention
-        if self.attn_impl == "flash_attn":
-            seq_positions = cache_position[:, 0]
-            seq_positions = self.convert_sequence_positions_for_flash_attn(
-                seq_positions=seq_positions, max_seq_len=self.max_seq_len
-            )
-        else:
-            seq_positions = cache_position[:, :1]
-
-        # Get local cache positions for sliding window layers
-        if len(self.sliding_window_layers) > 0:
-            sliding_cache_pos = self.get_local_cache_positions(position_ids, query_position)
-
-        for layer_idx, layer in enumerate(self.model.layers):
-            is_sliding = True if layer_idx in self.sliding_window_layers else False
-            hidden_states = layer(
-                hidden_states=hidden_states,
-                attention_mask=attention_mask,
-                seq_positions=sliding_cache_pos if is_sliding else seq_positions,
-                past_key_values=past_key_values,
-                cos=cos,
-                sin=sin,
-                block_tables=local_block_tables if is_sliding else global_block_tables,
-            )
-
-        hidden_states = self.get_last_layernorm()(hidden_states)
-        return hidden_states
-
-
-def slice_and_unsqueeze_cos_sin(cos, sin, cache_position, unsqueeze_dim=1):
-    """Slice cos[cache_position], sin[cache_position] vector for the query."""
-    if cache_position.shape[0] > 1:
-        cos_all = []
-        sin_all = []
-        for i in range(cache_position.shape[0]):
-            cos_all.append(cos[cache_position[i : i + 1]].unsqueeze(unsqueeze_dim))
-            sin_all.append(sin[cache_position[i : i + 1]].unsqueeze(unsqueeze_dim))
-        cos = torch.cat(cos_all, dim=0)
-        sin = torch.cat(sin_all, dim=0)
-    else:
-        cos = cos[cache_position].unsqueeze(unsqueeze_dim)
-        sin = sin[cache_position].unsqueeze(unsqueeze_dim)
-
-    return cos, sin

optimum/rbln/transformers/models/seq2seq/configuration_seq2seq.py

@@ -12,9 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Any, Dict, Optional
-
-import rebel
+from typing import Any, Optional
 
 from ....configuration_utils import RBLNModelConfig
 from ....utils.logging import get_logger
@@ -24,6 +22,8 @@ logger = get_logger()
 
 
 class RBLNModelForSeq2SeqLMConfig(RBLNModelConfig):
+    support_paged_attention = None
+
     def __init__(
         self,
         batch_size: Optional[int] = None,
@@ -31,7 +31,9 @@ class RBLNModelForSeq2SeqLMConfig(RBLNModelConfig):
         dec_max_seq_len: Optional[int] = None,
         use_attention_mask: Optional[bool] = None,
         pad_token_id: Optional[int] = None,
-        **kwargs: Dict[str, Any],
+        kvcache_num_blocks: Optional[int] = None,
+        kvcache_block_size: Optional[int] = None,
+        **kwargs: Any,
     ):
         """
         Args:
@@ -39,9 +41,12 @@ class RBLNModelForSeq2SeqLMConfig(RBLNModelConfig):
             enc_max_seq_len (Optional[int]): Maximum sequence length for the encoder.
             dec_max_seq_len (Optional[int]): Maximum sequence length for the decoder.
             use_attention_mask (Optional[bool]): Whether to use attention masks during inference.
-                This is automatically set to True for RBLN-CA02 devices.
             pad_token_id (Optional[int]): The ID of the padding token in the vocabulary.
-            **kwargs: Additional arguments passed to the parent RBLNModelConfig.
+            kvcache_num_blocks (Optional[int]): The total number of blocks to allocate for the
+                PagedAttention KV cache for the SelfAttention. Defaults to batch_size.
+            kvcache_block_size (Optional[int]): Sets the size (in number of tokens) of each block
+                in the PagedAttention KV cache for the SelfAttention. Defaults to dec_max_seq_len.
+            kwargs: Additional arguments passed to the parent RBLNModelConfig.
 
         Raises:
             ValueError: If batch_size is not a positive integer.
@@ -55,12 +60,14 @@ class RBLNModelForSeq2SeqLMConfig(RBLNModelConfig):
         self.dec_max_seq_len = dec_max_seq_len
 
         self.use_attention_mask = use_attention_mask
-        npu = self.npu or rebel.get_npu_name()
-        if npu == "RBLN-CA02":
-            if self.use_attention_mask is False:
-                logger.warning("Attention mask should be used with RBLN-CA02. Setting use_attention_mask to True.")
-            self.use_attention_mask = True
-        else:
-            self.use_attention_mask = self.use_attention_mask or False
 
         self.pad_token_id = pad_token_id
+
+        if self.support_paged_attention:
+            self.kvcache_num_blocks = kvcache_num_blocks
+            self.kvcache_block_size = kvcache_block_size
+        else:
+            if kvcache_num_blocks is not None or kvcache_block_size is not None:
+                raise ValueError(
+                    "You cannot set kvcache_num_blocks or kvcache_block_size as paged attention is not supported for the model."
+                )

optimum/rbln/transformers/models/seq2seq/modeling_seq2seq.py

@@ -20,6 +20,7 @@ import rebel
 import torch
 from rebel.compile_context import CompileContext
 from transformers import AutoModelForSeq2SeqLM, PretrainedConfig, PreTrainedModel
+from transformers.generation.utils import GenerationMixin
 from transformers.modeling_outputs import BaseModelOutput, Seq2SeqLMOutput
 
 from ....configuration_utils import RBLNCompileConfig
@@ -38,7 +39,7 @@ if TYPE_CHECKING:
 class RBLNRuntimeEncoder(RBLNPytorchRuntime):
     mandatory_members = ["main_input_name"]
 
-    def forward(self, *args: List[torch.Tensor], **kwargs: Dict[str, torch.Tensor]):
+    def forward(self, *args: List[torch.Tensor], **kwargs: torch.Tensor):
         output = super().forward(*args, **kwargs)
         return BaseModelOutput(last_hidden_state=output)
 
@@ -83,7 +84,7 @@ class RBLNRuntimeDecoder(RBLNPytorchRuntime):
                 decoding_step = cache_position[b_idx].item()
                 if not (0 <= decoding_step < self.dec_max_seq_len):
                     raise ValueError(
-                        f"Decoding step {decoding_step} out of bounds for attention mask with shape {self.dec_attn_mask.shape}."
+                        f"Decoding step {decoding_step} out of bounds for decoder_max_seq_len ({self.dec_max_seq_len})."
                     )
                 decoder_attention_mask[b_idx, : decoding_step + 1] = 1
 
@@ -101,7 +102,7 @@ class RBLNRuntimeDecoder(RBLNPytorchRuntime):
         return Seq2SeqLMOutput(logits=lm_logits)
 
 
-class RBLNModelForSeq2SeqLM(RBLNModel, ABC):
+class RBLNModelForSeq2SeqLM(RBLNModel, GenerationMixin, ABC):
     """
     This is a generic model class that will be instantiated as one of the model classes of the library (with a sequence-to-sequence language modeling head) when created with the from_pretrained() class method.
     This model inherits from [`RBLNModel`]. Check the superclass documentation for the generic methods the library implements for all its models.
@@ -117,6 +118,7 @@ class RBLNModelForSeq2SeqLM(RBLNModel, ABC):
     main_input_name = "input_ids"
     auto_model_class = AutoModelForSeq2SeqLM
     support_causal_attn = None
+    _is_stateful = False
 
     def __post_init__(self, **kwargs):
         batch_size = self.rbln_config.batch_size
@@ -181,6 +183,21 @@ class RBLNModelForSeq2SeqLM(RBLNModel, ABC):
 
         return {"encoder": compiled_encoder, "decoder": compiled_decoder}
 
+    @classmethod
+    def _update_paged_attention_config(cls, model_config: PretrainedConfig, rbln_config: RBLNModelForSeq2SeqLMConfig):
+        rbln_config.kvcache_num_blocks = rbln_config.kvcache_num_blocks or rbln_config.batch_size
+        rbln_config.kvcache_block_size = rbln_config.kvcache_block_size or rbln_config.dec_max_seq_len
+
+        if rbln_config.kvcache_num_blocks != rbln_config.batch_size:
+            raise NotImplementedError(
+                f"kvcache_num_blocks ({rbln_config.kvcache_num_blocks}) must be equal to batch_size ({rbln_config.batch_size}) as flash attention is not supported yet."
+            )
+
+        if rbln_config.kvcache_block_size != rbln_config.dec_max_seq_len:
+            raise NotImplementedError(
+                f"kvcache_block_size ({rbln_config.kvcache_block_size}) must be equal to dec_max_seq_len ({rbln_config.dec_max_seq_len}) as flash attention is not supported yet."
+            )
+
     @classmethod
     def _update_rbln_config(
         cls,
@@ -238,6 +255,9 @@ class RBLNModelForSeq2SeqLM(RBLNModel, ABC):
         if max_position_embeddings is not None and rbln_config.dec_max_seq_len > max_position_embeddings:
             raise ValueError("`dec_max_seq_len` should be less or equal than max_position_embeddings!")
 
+        if rbln_config.support_paged_attention:
+            cls._update_paged_attention_config(model_config, rbln_config)
+
         # model input info
         enc_input_info = [
             ("input_ids", [1, rbln_config.enc_max_seq_len], "int64"),
@@ -310,6 +330,7 @@ class RBLNModelForSeq2SeqLM(RBLNModel, ABC):
         dec_compile_config = RBLNCompileConfig(compiled_model_name="decoder", input_info=dec_input_info)
 
         rbln_config.set_compile_cfgs([enc_compile_config, dec_compile_config])
+
         return rbln_config
 
     @classmethod

optimum/rbln/transformers/models/seq2seq/seq2seq_architecture.py

@@ -31,7 +31,7 @@ class Seq2SeqWrapper:
     Args:
         model (nn.Module): The Seq2Seq model to wrap.
         enc_max_seq_len (int): Maximum sequence length for the encoder's position embeddings and cache sizes.
-        **kwargs: Additional arguments to pass to the decoder wrapper.
+        kwargs: Additional arguments to pass to the decoder wrapper.
     """
 
     def __init__(self, model: nn.Module, enc_max_seq_len: int, **kwargs):
@@ -125,7 +125,7 @@ class Seq2SeqDecoderWrapper(nn.Module):
 
     Args:
         model (nn.Module): The Seq2Seq model containing the decoder.
-        **kwargs: Additional arguments for decoder configuration.
+        kwargs: Additional arguments for decoder configuration.
     """
 
     def __init__(self, model: nn.Module, use_attention_mask: bool = True, **kwargs):

optimum/rbln/transformers/models/siglip/__init__.py

@@ -12,9 +12,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .configuration_siglip import (
-    RBLNSiglipVisionModelConfig,
-)
-from .modeling_siglip import (
-    RBLNSiglipVisionModel,
-)
+from .configuration_siglip import RBLNSiglipVisionModelConfig
+from .modeling_siglip import RBLNSiglipVisionModel

optimum/rbln/transformers/models/siglip/configuration_siglip.py

@@ -42,7 +42,7 @@ class RBLNSiglipVisionModelConfig(RBLNModelConfig):
             interpolate_pos_encoding (Optional[bool]): Whether to interpolate the position encoding.
             output_hidden_states: (Optional[bool]): Whether to return hidden states.
             output_attentions: (Optional[bool]): Whether to return attentions.
-            **kwargs: Additional arguments passed to the parent RBLNModelConfig.
+            kwargs: Additional arguments passed to the parent RBLNModelConfig.
 
         Raises:
             ValueError: If batch_size is not a positive integer.

optimum/rbln/transformers/models/siglip/modeling_siglip.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union
+from typing import TYPE_CHECKING, Any, Optional, Tuple, Union
 
 import torch
 from transformers import SiglipVisionConfig, SiglipVisionModel
@@ -29,8 +29,6 @@ logger = get_logger(__name__)
 if TYPE_CHECKING:
     from transformers import AutoFeatureExtractor, AutoProcessor, AutoTokenizer, PreTrainedModel
 
-    from ....diffusers.modeling_diffusers import RBLNDiffusionMixin, RBLNDiffusionMixinConfig
-
 
 class _SiglipVisionModel(torch.nn.Module):
     def __init__(
@@ -65,6 +63,8 @@ class RBLNSiglipVisionModel(RBLNModel):
     on RBLN devices, supporting image encoding for multimodal vision-language tasks.
     """
 
+    _tp_support = False
+
     @classmethod
     def wrap_model_if_needed(cls, model: torch.nn.Module, rbln_config: RBLNSiglipVisionModelConfig) -> torch.nn.Module:
         wrapper_cfg = {
@@ -74,12 +74,6 @@ class RBLNSiglipVisionModel(RBLNModel):
         }
         return _SiglipVisionModel(model, **wrapper_cfg).eval()
 
-    @classmethod
-    def update_rbln_config_using_pipe(
-        cls, pipe: "RBLNDiffusionMixin", rbln_config: "RBLNDiffusionMixinConfig", submodule_name: str
-    ) -> "RBLNDiffusionMixinConfig":
-        return rbln_config
-
     @classmethod
     def _update_rbln_config(
         cls,
@@ -126,13 +120,8 @@ class RBLNSiglipVisionModel(RBLNModel):
         output_attentions: bool = None,
         output_hidden_states: bool = None,
         interpolate_pos_encoding: bool = False,
-        **kwargs: Dict[str, Any],
+        **kwargs: Any,
     ) -> Union[Tuple, BaseModelOutputWithPooling]:
-        if len(kwargs) > 0 and any(value is not None for value in kwargs.values()):
-            logger.warning(
-                f"Currently, optimum-rbln does not support kwargs {kwargs.keys()} for {self.__class__.__name__}."
-            )
-
         output_attentions = output_attentions if output_attentions is not None else self.rbln_config.output_attentions
         output_hidden_states = (
             output_hidden_states if output_hidden_states is not None else self.rbln_config.output_hidden_states
@@ -156,7 +145,7 @@ class RBLNSiglipVisionModel(RBLNModel):
                 f"Please compile again with the correct argument."
             )
 
-        output = super().forward(pixel_values, return_dict=return_dict)
+        output = super().forward(pixel_values, return_dict=return_dict, **kwargs)
         return output
 
     def _prepare_output(self, output, return_dict):

optimum/rbln/transformers/models/swin/__init__.py

@@ -0,0 +1,16 @@
+# Copyright 2025 Rebellions Inc. All rights reserved.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .configuration_swin import RBLNSwinBackboneConfig
+from .modeling_swin import RBLNSwinBackbone

optimum/rbln/transformers/models/swin/configuration_swin.py

@@ -0,0 +1,42 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Optional, Tuple, Union
+
+from ...configuration_generic import RBLNModelForImageClassificationConfig
+
+
+class RBLNSwinBackboneConfig(RBLNModelForImageClassificationConfig):
+    def __init__(
+        self,
+        image_size: Optional[Union[int, Tuple[int, int]]] = None,
+        batch_size: Optional[int] = None,
+        output_hidden_states: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        **kwargs: Any,
+    ):
+        """
+        Args:
+            batch_size (Optional[int]): The batch size for text processing. Defaults to 1.
+            kwargs: Additional arguments passed to the parent RBLNModelConfig.
+
+        Raises:
+            ValueError: If batch_size is not a positive integer.
+        """
+        super().__init__(**kwargs)
+        self.batch_size = batch_size or 1
+        if not isinstance(self.batch_size, int) or self.batch_size < 0:
+            raise ValueError(f"batch_size must be a positive integer, got {self.batch_size}")
+
+        self.image_size = image_size
+        self.output_hidden_states = output_hidden_states
+        self.output_attentions = output_attentions