optimum-rbln 0.1.12__py3-none-any.whl → 0.1.15__py3-none-any.whl

optimum/rbln/transformers/generation/utils.py

@@ -1,397 +0,0 @@
-import traceback
-import warnings
-from typing import List, Optional, Union
-
-import torch
-from transformers.generation import GenerationConfig
-from transformers.generation.logits_process import LogitsProcessorList
-from transformers.generation.stopping_criteria import (
-    StoppingCriteriaList,
-    validate_stopping_criteria,
-)
-from transformers.generation.streamers import BaseStreamer
-from transformers.generation.utils import SampleDecoderOnlyOutput
-
-
-class RBLNGenerationMixin:
-    # call 'greedy_search` directly is deprecated and removed in v4.41.
-    def greedy_search(self, *args, **kwargs):
-        return self._greedy_search(*args, **kwargs)
-
-    def _greedy_search(
-        self,
-        input_ids: torch.LongTensor,
-        logits_processor: Optional[LogitsProcessorList] = None,
-        stopping_criteria: Optional[StoppingCriteriaList] = None,
-        max_length: Optional[int] = None,
-        pad_token_id: Optional[int] = None,
-        eos_token_id: Optional[Union[int, List[int]]] = None,
-        output_logits: Optional[bool] = None,
-        return_dict_in_generate: Optional[bool] = None,
-        streamer: Optional["BaseStreamer"] = None,
-        generation_config: Optional[GenerationConfig] = None,  # thkim change for 4.41.0
-        **model_kwargs,
-    ) -> Union[SampleDecoderOnlyOutput, torch.LongTensor]:
-        ###################### thkim change for 4.41.0 ############################
-        if generation_config is not None:
-            pad_token_id = generation_config.pad_token_id
-            output_logits = generation_config.output_logits
-            return_dict_in_generate = generation_config.return_dict_in_generate
-        ##########################################################################
-        # init values
-        logits_processor = logits_processor if logits_processor is not None else LogitsProcessorList()
-        stopping_criteria = stopping_criteria if stopping_criteria is not None else StoppingCriteriaList()
-
-        if max_length is not None:
-            warnings.warn(
-                "`max_length` is deprecated in this function, use"
-                " `stopping_criteria=StoppingCriteriaList([MaxLengthCriteria(max_length=max_length)])` instead.",
-                UserWarning,
-            )
-            stopping_criteria = validate_stopping_criteria(stopping_criteria, max_length)
-
-        pad_token_id = pad_token_id if pad_token_id is not None else self.generation_config.pad_token_id
-        eos_token_id = eos_token_id if eos_token_id is not None else self.generation_config.eos_token_id
-        if isinstance(eos_token_id, int):
-            eos_token_id = [eos_token_id]
-        eos_token_id_tensor = torch.tensor(eos_token_id).to(input_ids.device) if eos_token_id is not None else None
-
-        return_dict_in_generate = (
-            return_dict_in_generate
-            if return_dict_in_generate is not None
-            else self.generation_config.return_dict_in_generate
-        )
-
-        # init attention / hidden states / scores tuples
-        raw_logits = () if (return_dict_in_generate and output_logits) else None
-
-        # keep track of which sequences are already finished
-        unfinished_sequences = torch.ones(input_ids.shape[0], dtype=torch.long, device=input_ids.device)
-
-        this_peer_finished = False  # used by synced_gpus only
-
-        while True:
-            # prepare model inputs
-            model_inputs = self.prepare_inputs_for_generation(input_ids, **model_kwargs)
-            # forward pass to get next token
-            try:
-                outputs = self(
-                    **model_inputs,
-                    return_dict=True,
-                )
-                next_token_logits = outputs.logits[:, -1, :]
-            except Exception:
-                traceback.print_exc()
-                break
-
-            # pre-process distribution
-            next_tokens_scores = logits_processor(input_ids, next_token_logits)
-
-            # Store scores, attentions and hidden_states when required
-            if return_dict_in_generate:
-                if output_logits:
-                    raw_logits += (next_token_logits,)
-
-            # argmax
-            next_tokens = torch.argmax(next_tokens_scores, dim=-1)
-
-            # finished sentences should have their next token be a padding token
-            if eos_token_id is not None:
-                if pad_token_id is None:
-                    raise ValueError("If `eos_token_id` is defined, make sure that `pad_token_id` is defined.")
-                next_tokens = next_tokens * unfinished_sequences + pad_token_id * (1 - unfinished_sequences)
-
-            ########################################################################################################
-            # thkim change for right-padding batch
-            # if min_input_len <= update_idx < max_input_len
-            #   update validate input_ids[:,update_idx]
-            # TODO : raw_logits contains dummy next_token's logits
-            if hasattr(self, "rightpad_max_len"):
-                update_idx = model_inputs["cache_position"] + model_inputs["query_length"]
-                if update_idx < self.rightpad_max_len:
-                    # update exist input_ids rather than concat
-                    valid_indices = model_kwargs["attention_mask"][:, update_idx] == 0
-                    dummy_indices = model_kwargs["attention_mask"][:, update_idx] == 1
-
-                    input_ids[valid_indices, update_idx] = next_tokens[valid_indices]
-                    model_kwargs["attention_mask"][valid_indices, update_idx] = 1
-                    model_kwargs["past_key_values"] = outputs["past_key_values"]
-
-                    # dummy next_token -> pad_token_id for streamer
-                    # in order to skip by 'skip_special_tokens = True"
-                    if streamer is not None:
-                        next_tokens[dummy_indices] = pad_token_id
-                else:
-                    input_ids = torch.cat([input_ids, next_tokens[:, None]], dim=-1)
-                    model_kwargs = self._update_model_kwargs_for_generation(
-                        outputs,
-                        model_kwargs,
-                        is_encoder_decoder=self.config.is_encoder_decoder,
-                    )
-            else:
-                ############################################END#########################################################
-                # update generated ids, model inputs, and length for next step
-                input_ids = torch.cat([input_ids, next_tokens[:, None]], dim=-1)
-                model_kwargs = self._update_model_kwargs_for_generation(
-                    outputs,
-                    model_kwargs,
-                    is_encoder_decoder=self.config.is_encoder_decoder,
-                )
-
-            if streamer is not None:
-                streamer.put(next_tokens.cpu())
-                if streamer.is_blocked():
-                    this_peer_finished = True
-
-            # if eos_token was found in one sentence, set sentence to finished
-            if eos_token_id_tensor is not None:
-                ####################################################################
-                # thkim : to do not finish sequence of dummy_decoder of right_padding
-                if hasattr(self, "rightpad_max_len"):
-                    update_idx = model_inputs["cache_position"] + model_inputs["query_length"]
-                    if update_idx < self.rightpad_max_len:
-                        next_tokens += (
-                            model_kwargs["attention_mask"][:, update_idx] * self.generation_config.eos_token_id
-                        )
-                ######################################################################
-                unfinished_sequences = unfinished_sequences.mul(
-                    next_tokens.tile(eos_token_id_tensor.shape[0], 1).ne(eos_token_id_tensor.unsqueeze(1)).prod(dim=0)
-                )
-
-                # stop when each sentence is finished
-                if unfinished_sequences.max() == 0:
-                    this_peer_finished = True
-
-            # stop if we exceed the maximum length
-            # thkim : backward compatibility bool vs torch.BoolTensor
-            is_stop = stopping_criteria(input_ids, None)
-            if isinstance(is_stop, torch.BoolTensor):
-                is_stop = torch.all(is_stop)
-            if is_stop:
-                this_peer_finished = True
-
-            if this_peer_finished:
-                break
-
-        if streamer is not None:
-            streamer.end()
-
-        if return_dict_in_generate:
-            ############## thkim : roate raw_logits when right_padding#####################
-            if hasattr(self, "rightpad_max_len"):
-                raw_logits = torch.stack(raw_logits).transpose(0, 1)
-                for i in range(input_ids.shape[0]):
-                    raw_logits[i] = torch.cat((raw_logits[i][self.dummy_len[i] :], raw_logits[i][: self.dummy_len[i]]))
-                raw_logits = raw_logits.transpose(1, 0)
-            ##################################################################################
-            return SampleDecoderOnlyOutput(
-                sequences=input_ids,
-                logits=raw_logits,
-            )
-        else:
-            return input_ids
-
-    # call 'sample` directly is deprecated and removed in v4.41.
-    def sample(self, *args, **kwargs):
-        return self._sample(*args, **kwargs)
-
-    def _sample(
-        self,
-        input_ids: torch.LongTensor,
-        logits_processor: Optional[LogitsProcessorList] = None,
-        stopping_criteria: Optional[StoppingCriteriaList] = None,
-        logits_warper: Optional[LogitsProcessorList] = None,
-        max_length: Optional[int] = None,
-        pad_token_id: Optional[int] = None,
-        eos_token_id: Optional[Union[int, List[int]]] = None,
-        output_attentions: Optional[bool] = None,
-        output_hidden_states: Optional[bool] = None,
-        output_scores: Optional[bool] = None,
-        output_logits: Optional[bool] = None,
-        return_dict_in_generate: Optional[bool] = None,
-        synced_gpus: bool = False,
-        streamer: Optional["BaseStreamer"] = None,
-        generation_config: Optional[GenerationConfig] = None,
-        do_sample: Optional[bool] = True,
-        **model_kwargs,
-    ) -> Union[SampleDecoderOnlyOutput, torch.LongTensor]:
-        ###################### thkim change for 4.41.0 ############################
-        if generation_config is not None:
-            pad_token_id = generation_config.pad_token_id
-            output_logits = generation_config.output_logits
-            return_dict_in_generate = generation_config.return_dict_in_generate
-            do_sample = generation_config.do_sample
-        ###########################################################################
-
-        # init values
-        logits_processor = logits_processor if logits_processor is not None else LogitsProcessorList()
-        stopping_criteria = stopping_criteria if stopping_criteria is not None else StoppingCriteriaList()
-
-        if max_length is not None:
-            warnings.warn(
-                "`max_length` is deprecated in this function, use"
-                " `stopping_criteria=StoppingCriteriaList([MaxLengthCriteria(max_length=max_length)])` instead.",
-                UserWarning,
-            )
-            stopping_criteria = validate_stopping_criteria(stopping_criteria, max_length)
-
-        logits_warper = logits_warper if logits_warper is not None else LogitsProcessorList()
-        pad_token_id = pad_token_id if pad_token_id is not None else self.generation_config.pad_token_id
-        eos_token_id = eos_token_id if eos_token_id is not None else self.generation_config.eos_token_id
-
-        if isinstance(eos_token_id, int):
-            eos_token_id = [eos_token_id]
-        eos_token_id_tensor = torch.tensor(eos_token_id).to(input_ids.device) if eos_token_id is not None else None
-
-        output_scores = output_scores if output_scores is not None else self.generation_config.output_scores
-        output_logits = output_logits if output_logits is not None else False
-
-        # init attention / hidden states / scores tuples
-        scores = () if (return_dict_in_generate and output_scores) else None
-        raw_logits = () if (return_dict_in_generate and output_logits) else None
-
-        # keep track of which sequences are already finished
-        batch_size, cur_len = input_ids.shape
-        unfinished_sequences = torch.ones(batch_size, dtype=torch.long, device=input_ids.device)
-        this_peer_finished = False
-
-        # model_kwargs["cache_position"] = torch.arange(cur_len, device=input_ids.device)
-
-        while True:
-            # prepare model inputs
-            model_inputs = self.prepare_inputs_for_generation(input_ids, **model_kwargs)
-
-            # forward pass to get next token
-            try:
-                outputs = self(
-                    **model_inputs,
-                    return_dict=True,
-                    output_attentions=output_attentions,
-                    output_hidden_states=output_hidden_states,
-                )
-                next_token_logits = outputs.logits[:, -1, :]
-            except Exception:
-                traceback.print_exc()
-                break
-
-            # pre-process distribution
-            next_token_scores = logits_processor(input_ids, next_token_logits)
-
-            ###################### thkim change for 4.41.0 ############################
-            if do_sample:
-                next_token_scores = logits_warper(input_ids, next_token_scores)
-            ###########################################################################
-
-            # Store scores, attentions and hidden_states when required
-            if return_dict_in_generate:
-                if output_scores:
-                    scores += (next_token_scores,)
-                if output_logits:
-                    raw_logits += (next_token_logits,)
-
-            # sample
-            ###################### thkim change for 4.41.0 ############################
-            if do_sample:
-                probs = torch.nn.functional.softmax(next_token_scores, dim=-1)
-                next_tokens = torch.multinomial(probs, num_samples=1).squeeze(1)
-            else:
-                next_tokens = torch.argmax(next_token_scores, dim=-1)
-            ###########################################################################
-
-            # finished sentences should have their next token be a padding token
-            if eos_token_id is not None:
-                if pad_token_id is None:
-                    raise ValueError("If `eos_token_id` is defined, make sure that `pad_token_id` is defined.")
-                next_tokens = next_tokens * unfinished_sequences + pad_token_id * (1 - unfinished_sequences)
-
-            ###############################thkim change for right-padding batch#################################
-            # if min_input_len <= update_idx < max_input_len
-            #   update validate input_ids[:,update_idx]
-            # TODO : raw_logits contains dummy next_token's logits
-
-            if hasattr(self, "rightpad_max_len"):
-                update_idx = model_inputs["cache_position"] + model_inputs["query_length"]
-                if update_idx < self.rightpad_max_len:
-                    # update exist input_ids rather than concat
-                    valid_indices = model_kwargs["attention_mask"][:, update_idx] == 0
-                    dummy_indices = model_kwargs["attention_mask"][:, update_idx] == 1
-
-                    input_ids[valid_indices, update_idx] = next_tokens[valid_indices]
-                    model_kwargs["attention_mask"][valid_indices, update_idx] = 1
-                    model_kwargs["past_key_values"] = outputs["past_key_values"]
-                    # dummy next_token -> pad_token_id for streamer
-                    # in order to skip by 'skip_special_tokens = True"
-                    if streamer is not None:
-                        next_tokens[dummy_indices] = pad_token_id
-                else:
-                    input_ids = torch.cat([input_ids, next_tokens[:, None]], dim=-1)
-                    model_kwargs = self._update_model_kwargs_for_generation(
-                        outputs,
-                        model_kwargs,
-                        is_encoder_decoder=self.config.is_encoder_decoder,
-                    )
-            else:
-                ############################################END#########################################################
-                # update generated ids, model inputs, and length for next step
-                input_ids = torch.cat([input_ids, next_tokens[:, None]], dim=-1)
-
-                model_kwargs = self._update_model_kwargs_for_generation(
-                    outputs,
-                    model_kwargs,
-                    is_encoder_decoder=self.config.is_encoder_decoder,
-                )
-
-            if streamer is not None:
-                streamer.put(next_tokens.cpu())
-                if streamer.is_blocked():
-                    this_peer_finished = True
-
-            # if eos_token was found in one sentence, set sentence to finished
-            if eos_token_id_tensor is not None:
-                ####################################################################
-                # thkim : to do not finish sequence of dummy_decoder of right_padding
-                if hasattr(self, "rightpad_max_len"):
-                    update_idx = model_inputs["cache_position"] + model_inputs["query_length"]
-                    if update_idx < self.rightpad_max_len:
-                        next_tokens += (
-                            model_kwargs["attention_mask"][:, update_idx] * self.generation_config.eos_token_id
-                        )
-
-                ######################################################################
-                unfinished_sequences = unfinished_sequences.mul(
-                    next_tokens.tile(eos_token_id_tensor.shape[0], 1).ne(eos_token_id_tensor.unsqueeze(1)).prod(dim=0)
-                )
-
-                # stop when each sentence is finished
-                if unfinished_sequences.max() == 0:
-                    this_peer_finished = True
-
-            # stop if we exceed the maximum length
-            # thkim : backward compatibility bool vs list[bool]
-            is_stop = stopping_criteria(input_ids, None)
-            if isinstance(is_stop, torch.BoolTensor):
-                is_stop = torch.all(is_stop)
-            if is_stop:
-                this_peer_finished = True
-
-            if this_peer_finished:
-                break
-
-        if streamer is not None:
-            streamer.end()
-
-        if return_dict_in_generate:
-            ############## thkim : roate raw_logits when right_padding#####################
-            if hasattr(self, "rightpad_max_len"):
-                raw_logits = torch.stack(raw_logits).transpose(0, 1)
-                for i in range(input_ids.shape[0]):
-                    raw_logits[i] = torch.cat((raw_logits[i][self.dummy_len[i] :], raw_logits[i][: self.dummy_len[i]]))
-                raw_logits = raw_logits.transpose(1, 0)
-            ##################################################################################
-            return SampleDecoderOnlyOutput(
-                sequences=input_ids,
-                scores=scores,
-                logits=raw_logits,
-            )
-        else:
-            return input_ids

optimum/rbln/transformers/models/exaone/hf_hub_cached/configuration_exaone.py

@@ -1,181 +0,0 @@
-# coding=utf-8
-# Copyright 2021 The LG AI Research EXAONE Lab. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""EXAONE model configuration"""
-
-from transformers.configuration_utils import PretrainedConfig
-from transformers.utils import logging
-
-
-logger = logging.get_logger(__name__)
-
-EXAONE_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
-
-
-class ExaoneConfig(PretrainedConfig):
-    r"""
-    This is the configuration class to store the configuration of a :class:`~transformers.ExaoneModel`. It is used to
-    instantiate a EXAONE model according to the specified arguments, defining the model architecture. Instantiating a
-    configuration with the defaults will yield a similar configuration to that of the Exaone
-    Configuration objects inherit from :class:`~transformers.PretrainedConfig` and can be used to control the model
-    outputs. Read the documentation from :class:`~transformers.PretrainedConfig` for more information.
-    Args:
-        vocab_size (:obj:`int`, `optional`, defaults to 102400):
-            Vocabulary size of the EXAONE model. Defines the number of different tokens that can be represented by the
-            :obj:`inputs_ids` passed when calling :class:`~transformers.ExaoneModel`. Vocabulary size of the model.
-            Defines the different tokens that can be represented by the `inputs_ids` passed to the forward method of
-            :class:`~transformers.EXAONEModel`.
-        max_position_embeddings (:obj:`int`, `optional`, defaults to 2048):
-            The maximum sequence length that this model might ever be used with. Typically set this to something large
-            just in case (e.g., 512 or 1024 or 2048).
-        hidden_size (:obj:`int`, `optional`, defaults to 2048):
-            Dimensionality of the encoder layers and the pooler layer.
-        num_layers (:obj:`int`, `optional`, defaults to 32):
-            Number of hidden layers in the Transformer encoder.
-        num_attention_heads (:obj:`int`, `optional`, defaults to 32):
-            Number of attention heads for each attention layer in the Transformer decoder.
-        num_key_value_heads (:obj:`int`, `optional`):
-            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
-            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
-            `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
-            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
-            by meanpooling all the original heads within that group. For more details checkout [this
-            paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
-            `num_attention_heads`.
-        intermediate_size (:obj:`int`, `optional`, defaults to `hidden_size * 4`):
-            Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
-        activation_function (:obj:`str` or :obj:`function`, `optional`, defaults to :obj:`"silu"`):
-            The non-linear activation function (function or string) in the decoder.
-        rope_theta (:obj:`float`, `optional`, defaults to 10000.0):
-            The base period of the RoPE embeddings.
-        rope_scaling (:obj:`Dict`, `optional`):
-            Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
-            and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
-            accordingly.
-            Expected contents:
-                `rope_type` (:obj:`str`):
-                    The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',
-                    'llama3'], with 'default' being the original RoPE implementation.
-                `factor` (:obj:`float`, `optional`):
-                    Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In
-                    most scaling types, a `factor` of x will enable the model to handle sequences of length x *
-                    original maximum pre-trained length.
-                `original_max_position_embeddings` (:obj:`int`, `optional`):
-                    Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during
-                    pretraining.
-                `attention_factor` (:obj:`float`, `optional`):
-                    Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention
-                    computation. If unspecified, it defaults to value recommended by the implementation, using the
-                    `factor` field to infer the suggested value.
-                `beta_fast` (:obj:`float`, `optional`):
-                    Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear
-                    ramp function. If unspecified, it defaults to 32.
-                `beta_slow` (:obj:`float`, `optional`):
-                    Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear
-                    ramp function. If unspecified, it defaults to 1.
-                `short_factor` (:obj:`List[float]`, `optional`):
-                    Only used with 'longrope'. The scaling factor to be applied to short contexts (<
-                    `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
-                    size divided by the number of attention heads divided by 2
-                `long_factor` (:obj:`List[float]`, `optional`):
-                    Only used with 'longrope'. The scaling factor to be applied to long contexts (<
-                    `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
-                    size divided by the number of attention heads divided by 2
-                `low_freq_factor` (:obj:`float`, `optional`):
-                    Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE
-                `high_freq_factor` (:obj:`float`, `optional`):
-                    Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE
-        embed_dropout (:obj:`float`, `optional`, defaults to 0.0):
-            The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.
-        attention_dropout (:obj:`float`, `optional`, defaults to 0.0):
-            The dropout ratio for the attention probabilities.
-        layer_norm_epsilon (:obj:`float`, `optional`, defaults to 1e-5):
-            The epsilon used by the layer normalization layers.
-        initializer_range (:obj:`float`, `optional`, defaults to 0.02):
-            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
-        use_cache (:obj:`bool`, `optional`, defaults to :obj:`True`):
-            Whether or not the model should return the last key/values attentions (not used by all models). Only
-            relevant if ``config.is_decoder=True``.
-        bos_token_id (:obj:`int`, `optional`, defaults to 0):
-            Beginning of stream token id.
-        eos_token_id (:obj:`int`, `optional`, defaults to 2):
-            End of stream token id.
-        tie_word_embeddings (:obj:`bool`, `optional`, defaults to :obj:`True`):
-            Whether to tie weight embeddings
-        gradient_checkpointing (:obj:`bool`, `optional`, defaults to :obj:`False`):
-            If True, use gradient checkpointing to save memory at the expense of slower backward pass.
-        Example::
-            >>> from transformers import EXAONEModel, ExaoneConfig
-            >>> # Initializing a EXAONE configuration
-            >>> configuration = ExaoneConfig()
-            >>> # Initializing a model from configuration
-            >>> model = EXAONEModel(configuration)
-            >>> # Accessing the model configuration
-            >>> configuration = model.config
-    """
-
-    model_type = "exaone"
-    keys_to_ignore_at_inference = ["past_key_values"]
-    attribute_map = {"num_hidden_layers": "num_layers"}
-
-    def __init__(
-        self,
-        vocab_size=102400,
-        max_position_embeddings=2048,
-        hidden_size=2048,
-        num_layers=32,
-        num_attention_heads=32,
-        num_key_value_heads=None,
-        intermediate_size=None,
-        activation_function="silu",
-        rope_theta=10000.0,
-        rope_scaling=None,
-        embed_dropout=0.0,
-        attention_dropout=0.0,
-        layer_norm_epsilon=1e-5,
-        initializer_range=0.02,
-        use_cache=True,
-        bos_token_id=0,
-        eos_token_id=2,
-        tie_word_embeddings=True,
-        **kwargs,
-    ):
-        self.vocab_size = vocab_size
-        self.max_position_embeddings = max_position_embeddings
-        self.hidden_size = hidden_size
-        self.num_layers = num_layers
-        self.num_attention_heads = num_attention_heads
-        self.num_hidden_layers = num_layers
-        if num_key_value_heads is None:
-            num_key_value_heads = num_attention_heads
-        self.num_key_value_heads = num_key_value_heads
-        if intermediate_size:
-            self.intermediate_size = intermediate_size
-        else:
-            self.intermediate_size = hidden_size * 4
-        self.activation_function = activation_function
-        self.embed_dropout = embed_dropout
-        self.attention_dropout = attention_dropout
-        self.layer_norm_epsilon = layer_norm_epsilon
-        self.initializer_range = initializer_range
-        self.use_cache = use_cache
-        self.rope_theta = rope_theta
-        self.rope_scaling = rope_scaling
-
-        self.bos_token_id = bos_token_id
-        self.eos_token_id = eos_token_id
-
-        super().__init__(
-            bos_token_id=bos_token_id, eos_token_id=eos_token_id, tie_word_embeddings=tie_word_embeddings, **kwargs
-        )