transformers 5.0.0__py3-none-any.whl → 5.0.0rc0__py3-none-any.whl

transformers/modeling_rope_utils.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 
 import math
-import warnings
 from functools import wraps
 from typing import TYPE_CHECKING, Optional, TypedDict
 
@@ -46,19 +45,17 @@ def dynamic_rope_update(rope_forward):
     def longrope_frequency_update(self, position_ids, device, layer_type=None):
         """Longrope uses long factor if sequence is larger than original pretraining length, short otherwise."""
         seq_len = torch.max(position_ids) + 1
-
+        original_max_position_embeddings = getattr(
+            self.config, "original_max_position_embeddings", self.config.max_position_embeddings
+        )
         if layer_type is None:
             rope_type = self.rope_type
             original_inv_freq = self.original_inv_freq
             prefix = ""
-            original_max_position_embeddings = self.config.rope_parameters["original_max_position_embeddings"]
         else:
             rope_type = self.rope_type[layer_type]
             original_inv_freq = getattr(self, f"{layer_type}_original_inv_freq")
             prefix = f"{layer_type}_"
-            original_max_position_embeddings = self.config.rope_parameters[layer_type][
-                "original_max_position_embeddings"
-            ]
 
         if seq_len > original_max_position_embeddings:
             if not hasattr(self, f"{layer_type}_long_inv_freq"):
@@ -132,8 +129,8 @@ def dynamic_rope_update(rope_forward):
 def _compute_linear_scaling_rope_parameters(
     config: Optional["PreTrainedConfig"] = None,
     device: Optional["torch.device"] = None,
-    seq_len: int | None = None,
-    layer_type: str | None = None,
+    seq_len: Optional[int] = None,
+    layer_type: Optional[str] = None,
 ) -> tuple["torch.Tensor", float]:
     """
     Computes the inverse frequencies with linear scaling. Credits to the Reddit user /u/kaiokendev
@@ -186,8 +183,8 @@ def _compute_linear_scaling_rope_parameters(
 def _compute_dynamic_ntk_parameters(
     config: Optional["PreTrainedConfig"] = None,
     device: Optional["torch.device"] = None,
-    seq_len: int | None = None,
-    layer_type: str | None = None,
+    seq_len: Optional[int] = None,
+    layer_type: Optional[str] = None,
 ) -> tuple["torch.Tensor", float]:
     """
     Computes the inverse frequencies with NTK scaling. Credits to the Reddit users /u/bloc97 and /u/emozilla
@@ -225,6 +222,7 @@ def _compute_dynamic_ntk_parameters(
         Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
         post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
     """
+    # TODO (joao): use the new `original_max_position_embeddings` from rope_parameters
     # For backward compatibility standardize the `rope_parameters_dict` if it uses old format
     config.standardize_rope_params()
     rope_parameters_dict = config.rope_parameters[layer_type] if layer_type is not None else config.rope_parameters
@@ -233,31 +231,32 @@ def _compute_dynamic_ntk_parameters(
     partial_rotary_factor = rope_parameters_dict.get("partial_rotary_factor", 1.0)
     head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)
     dim = int(head_dim * partial_rotary_factor)
+    max_position_embeddings = config.max_position_embeddings
     factor = rope_parameters_dict["factor"]
     attention_factor = 1.0  # Unused in this type of RoPE
 
     # seq_len: default to max_position_embeddings, e.g. at init time
     if seq_len is None:
-        seq_len = config.max_position_embeddings
+        seq_len = max_position_embeddings
     elif isinstance(seq_len, torch.Tensor):
         seq_len = torch.maximum(
             seq_len,
-            torch.tensor(config.max_position_embeddings, dtype=seq_len.dtype, device=seq_len.device),
+            torch.tensor(max_position_embeddings, dtype=seq_len.dtype, device=seq_len.device),
         )
     else:
-        seq_len = max(seq_len, config.max_position_embeddings)
+        seq_len = max(seq_len, max_position_embeddings)
 
     # Compute the inverse frequencies
-    base = base * ((factor * seq_len / config.max_position_embeddings) - (factor - 1)) ** (dim / (dim - 2))
+    base = base * ((factor * seq_len / max_position_embeddings) - (factor - 1)) ** (dim / (dim - 2))
     inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.int64).to(device=device, dtype=torch.float) / dim))
     return inv_freq, attention_factor
 
 
 def _compute_yarn_parameters(
     config: "PreTrainedConfig",
-    device: Optional["torch.device"] = None,
-    seq_len: int | None = None,
-    layer_type: str | None = None,
+    device: "torch.device",
+    seq_len: Optional[int] = None,
+    layer_type: Optional[str] = None,
 ) -> tuple["torch.Tensor", float]:
     """
     Computes the inverse frequencies with NTK scaling. Please refer to the
@@ -292,7 +291,8 @@ def _compute_yarn_parameters(
                     `mscale_all_dim` are provided, `mscale_all_dim` acts scalar augmenting `log(factor)` when computing
                     the denominator for the inferred value of `attention_factor`. If not provided, `attention_factor`
                     will be calculated based on `factor` only.
-                *   `original_max_position_embeddings` (`int`): The original max position embeddings used during pretraining.
+                *   `original_max_position_embeddings` (`int`, *optional*): The original max position embeddings used
+                    during pretraining. If not provided, the function falls back to `max_position_embeddings`.
                 *   `truncate` (`bool`, *optional*): Whether to truncate the correction range.
 
             Additionally, this function will make use of the following properties if they are found in the config:
@@ -323,13 +323,15 @@ def _compute_yarn_parameters(
     attention_factor = rope_parameters_dict.get("attention_factor")
     mscale = rope_parameters_dict.get("mscale")
     mscale_all_dim = rope_parameters_dict.get("mscale_all_dim")
-    original_max_position_embeddings = rope_parameters_dict["original_max_position_embeddings"]
 
-    # NOTE: DeekSeek-V3 (and potentially other models) have `original_max_position_embeddings` field
-    # containing the pretrained value. They use the ratio between `max_position_embeddings` and this value
-    # to compute the default attention scaling factor, instead of using `factor`.
-    if factor is None:
+    # NOTE: DeekSeek-V3 (and potentially other models) modify `max_position_embeddings` and have a
+    # `original_max_position_embeddings` field containing the pretrained value. They use the ratio between these two
+    # values to compute the default attention scaling factor, instead of using `factor`.
+    if "original_max_position_embeddings" in rope_parameters_dict:
+        original_max_position_embeddings = rope_parameters_dict["original_max_position_embeddings"]
         factor = config.max_position_embeddings / original_max_position_embeddings
+    else:
+        original_max_position_embeddings = config.max_position_embeddings
 
     def get_mscale(scale, mscale=1):
         if scale <= 1:
@@ -390,9 +392,9 @@ def _compute_yarn_parameters(
 
 def _compute_longrope_parameters(
     config: "PreTrainedConfig",
-    device: Optional["torch.device"] = None,
-    seq_len: int | None = None,
-    layer_type: str | None = None,
+    device: "torch.device",
+    seq_len: Optional[int] = None,
+    layer_type: Optional[str] = None,
 ) -> tuple["torch.Tensor", float]:
     """
     Computes the inverse frequencies with LongRoPE scaling. Please refer to the
@@ -437,6 +439,7 @@ def _compute_longrope_parameters(
         Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
         post-processing scaling factor applied to the computed cos/sin.
     """
+    # TODO (joao): use the new `original_max_position_embeddings` from rope_parameters
     # For backward compatibility standardize the `rope_parameters_dict` if it uses old format
     config.standardize_rope_params()
     rope_parameters_dict = config.rope_parameters[layer_type] if layer_type is not None else config.rope_parameters
@@ -450,13 +453,14 @@ def _compute_longrope_parameters(
     short_factor = rope_parameters_dict["short_factor"]
     factor = rope_parameters_dict.get("factor")
     attention_factor = rope_parameters_dict.get("attention_factor")
-    original_max_position_embeddings = rope_parameters_dict["original_max_position_embeddings"]
 
     # NOTE: Phi3 (and potentially other models) modify `max_position_embeddings` and have a
     # `original_max_position_embeddings` field containing the pretrained value. They use the ratio between these two
     # values to compute the default attention scaling factor, instead of using `factor`.
-    if factor is None:
+    if original_max_position_embeddings := getattr(config, "original_max_position_embeddings", None):
         factor = config.max_position_embeddings / original_max_position_embeddings
+    else:
+        original_max_position_embeddings = config.max_position_embeddings
 
     # Sets the attention factor as suggested in the paper
     if attention_factor is None:
@@ -478,9 +482,9 @@ def _compute_longrope_parameters(
 
 def _compute_llama3_parameters(
     config: "PreTrainedConfig",
-    device: Optional["torch.device"] = None,
-    seq_len: int | None = None,
-    layer_type: str | None = None,
+    device: "torch.device",
+    seq_len: Optional[int] = None,
+    layer_type: Optional[str] = None,
 ) -> tuple["torch.Tensor", float]:
     """
     Computes the inverse frequencies for llama 3.1.
@@ -582,7 +586,7 @@ class RopeParameters(TypedDict, total=False):
             most scaling types, a `factor` of x will enable the model to handle sequences of length x *
             original maximum pre-trained length.
         original_max_position_embeddings (`int`, *optional*):
-            Used with 'yarn', 'longrope' and 'llama3'. The original max position embeddings used during
+            Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during
             pretraining.
         attention_factor (`float`, *optional*):
             Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention
@@ -609,17 +613,17 @@ class RopeParameters(TypedDict, total=False):
     """
 
     rope_theta: float
-    rope_type: str | None
-    partial_rotary_factor: float | None
-    factor: float | None
-    original_max_position_embeddings: int | None
-    attention_factor: float | None
-    beta_fast: float | None
-    beta_slow: float | None
-    short_factor: list[float] | None
-    long_factor: list[float] | None
-    low_freq_factor: float | None
-    high_freq_factor: float | None
+    rope_type: Optional[str]
+    partial_rotary_factor: Optional[float]
+    factor: Optional[float]
+    original_max_position_embeddings: Optional[int]
+    attention_factor: Optional[float]
+    beta_fast: Optional[float]
+    beta_slow: Optional[float]
+    short_factor: Optional[list[float]]
+    long_factor: Optional[list[float]]
+    low_freq_factor: Optional[float]
+    high_freq_factor: Optional[float]
 
 
 class RotaryEmbeddingConfigMixin:
@@ -629,14 +633,13 @@ class RotaryEmbeddingConfigMixin:
 
     default_theta = 10_000.0
 
-    def convert_rope_params_to_dict(self, ignore_keys_at_rope_validation: set | None = None, **kwargs):
+    def convert_rope_params_to_dict(self, ignore_keys_at_rope_validation: Optional[set] = None, **kwargs):
         rope_scaling = kwargs.pop("rope_scaling", None)
         self.rope_parameters = rope_scaling or self.rope_parameters
         self.rope_parameters = self.rope_parameters if self.rope_parameters is not None else {}
 
         # Standardize and validate the correctness of rotary position embeddings parameters
         self.rope_parameters.setdefault("rope_theta", kwargs.pop("rope_theta", self.default_theta))
-
         if "partial_rotary_factor" in kwargs:
             self.rope_parameters.setdefault("partial_rotary_factor", kwargs["partial_rotary_factor"])
             ignore_keys_at_rope_validation = {"partial_rotary_factor"}
@@ -650,50 +653,28 @@ class RotaryEmbeddingConfigMixin:
         Helper to standardize the config's rope params field by ensuring the params are defined for each
         later type. For old model the fn will duplicate a single rope param in each layer type (backward compatibility)
         """
-        # Move `rope_theta` and `partial_rotary_factor` to the `rope_parameters`, if not there yet
+        # Move `rope_theta` and `partial_rotary_factor` to the params dict, if not there yet
         rope_theta = getattr(self, "rope_theta", None)
         partial_rotary_factor = getattr(self, "partial_rotary_factor", None)
-        rope_parameters = getattr(self, "rope_parameters", None) or {}
-        layer_types = getattr(self, "layer_types", None)
+        rope_parameters = self.rope_parameters
 
-        # Case 0: no RoPE params defined
-        if not (rope_parameters or rope_theta):
-            # partial_rotary_factor without rope_theta is invalid, so we don't check for it here
-            logger.warning("`standardize_rope_params` was called but no RoPE parameters were found.")
-            return
         # Case 1: RoPE param keys do not intersect with possible `layer_types` -> one global dict
-        elif layer_types is None or rope_parameters == {} or not set(rope_parameters.keys()).issubset(layer_types):
+        if getattr(self, "layer_types", None) is None or not set(rope_parameters.keys()).issubset(self.layer_types):
             rope_parameters.setdefault("rope_type", rope_parameters.get("type", "default"))
             rope_parameters.setdefault("rope_theta", rope_theta)
             if partial_rotary_factor is not None:
                 rope_parameters["partial_rotary_factor"] = partial_rotary_factor
-
-            # Move pretraining-time maximum length to rope parameter dict for RoPE types with scaling
-            if rope_parameters["rope_type"] in ["llama3", "yarn", "longrope"]:
-                if hasattr(self, "original_max_position_embeddings"):
-                    # NOTE: Phi3 (and potentially other models) save `original_max_position_embeddings` field
-                    # containing the pretrained value outside rope parameters. This is an exception case where we
-                    # give priority to `self.original_max_position_embeddings
-                    self.rope_parameters["original_max_position_embeddings"] = self.original_max_position_embeddings
-                else:
-                    self.rope_parameters.setdefault("original_max_position_embeddings", self.max_position_embeddings)
-
         # Case 2: different RoPE for each layer -> several params as nested dict
         else:
-            for layer_type in set(layer_types):
+            for layer_type in self.layer_types:
                 rope_parameters[layer_type].setdefault("rope_type", rope_parameters[layer_type].get("type", "default"))
                 rope_parameters[layer_type].setdefault("rope_theta", rope_theta)
                 if partial_rotary_factor is not None:
                     rope_parameters[layer_type]["partial_rotary_factor"] = partial_rotary_factor
 
-                if rope_parameters[layer_type]["rope_type"] in ["llama3", "yarn", "longrope"]:
-                    self.rope_parameters[layer_type].setdefault(
-                        "original_max_position_embeddings", self.max_position_embeddings
-                    )
-
         self.rope_parameters = rope_parameters
 
-    def validate_rope(self: "PreTrainedConfig", ignore_keys: set | None = None):
+    def validate_rope(self: "PreTrainedConfig", ignore_keys: Optional[set] = None):
         """
         Validate the RoPE config arguments, given a `"PreTrainedConfig"` object
         """
@@ -710,23 +691,23 @@ class RotaryEmbeddingConfigMixin:
 
         for rope_parameters in rope_parameters_dict.values():
             rope_type = rope_parameters.get("rope_type", rope_parameters.get("type", "default"))
-            validation_fn = getattr(self, f"_validate_{rope_type}_rope_parameters", None)
+            validation_fn = getattr(self, f"_validate_{rope_type}_rope_parameters")
             rope_parameters["rope_type"] = rope_type
 
             if validation_fn is not None:
                 validation_fn(rope_parameters, ignore_keys=ignore_keys)
             else:
                 logger.warning(
-                    f"Missing validation function in 'RotaryEmbeddingConfigMixin' for 'rope_type'='{rope_type}'"
+                    f"Missing validation function mapping in `ROPE_VALIDATION_FUNCTIONS` for 'rope_type'='{rope_type}'"
                 )
 
-    def _validate_default_rope_parameters(self, rope_parameters: dict, ignore_keys: set | None = None):
+    def _validate_default_rope_parameters(self, rope_parameters: dict, ignore_keys: Optional[set] = None):
         required_keys = {"rope_type", "rope_theta"}
         received_keys = set(rope_parameters.keys())
         rope_type = rope_parameters["rope_type"]
         self._check_received_keys(rope_type, received_keys, required_keys, ignore_keys=ignore_keys)
 
-    def _validate_linear_rope_parameters(self, rope_parameters: dict, ignore_keys: set | None = None):
+    def _validate_linear_rope_parameters(self, rope_parameters: dict, ignore_keys: Optional[set] = None):
         required_keys = {"rope_type", "factor", "rope_theta"}
         received_keys = set(rope_parameters.keys())
         rope_type = rope_parameters["rope_type"]
@@ -736,25 +717,27 @@ class RotaryEmbeddingConfigMixin:
         if factor is None or not isinstance(factor, float) or factor < 1.0:
             logger.warning(f"`rope_parameters`'s factor field must be a float >= 1, got {factor}")
 
-    def _validate_dynamic_rope_parameters(self, rope_parameters: dict, ignore_keys: set | None = None):
+    def _validate_dynamic_rope_parameters(self, rope_parameters: dict, ignore_keys: Optional[set] = None):
+        # TODO (joao): update logic for the inclusion of `original_max_position_embeddings`
+        optional_keys = {"original_max_position_embeddings"}
         required_keys = {"rope_type", "factor"}
         received_keys = set(rope_parameters.keys())
         rope_type = rope_parameters["rope_type"]
-        self._check_received_keys(rope_type, received_keys, required_keys, ignore_keys=ignore_keys)
+        self._check_received_keys(rope_type, received_keys, required_keys, optional_keys, ignore_keys=ignore_keys)
 
         factor = rope_parameters["factor"]
         if factor is None or not isinstance(factor, float) or factor < 1.0:
             logger.warning(f"`rope_parameters`'s factor field must be a float >= 1, got {factor}")
 
-    def _validate_yarn_rope_parameters(self, rope_parameters: dict, ignore_keys: set | None = None):
-        required_keys = {"rope_type", "factor", "rope_theta", "original_max_position_embeddings"}
+    def _validate_yarn_rope_parameters(self, rope_parameters: dict, ignore_keys: Optional[set] = None):
+        required_keys = {"rope_type", "factor", "rope_theta"}
         optional_keys = {
             "attention_factor",
             "beta_fast",
             "beta_slow",
+            "original_max_position_embeddings",
             "mscale",
             "mscale_all_dim",
-            "truncate",
         }
         received_keys = set(rope_parameters.keys())
         rope_type = rope_parameters["rope_type"]
@@ -782,24 +765,37 @@ class RotaryEmbeddingConfigMixin:
                 f"(defaults to 32 if None) and beta_slow={beta_slow} (defaults to 1 if None)"
             )
 
-        # Double-check: `factor` should be the ratio between the pre-yarn and post-yarn context lengths.
-        # NOTE: we might get `implicit_factor == 1` if config's `original_max_position_embeddings` was
-        # inferred from `max_position_embeddings` during standardization
-        original_max_position_embeddings = self.rope_parameters["original_max_position_embeddings"]
-        implicit_factor = self.max_position_embeddings / original_max_position_embeddings
-        if implicit_factor != factor and implicit_factor != 1:
+        # Models should set `config.rope_parameters["original_max_position_embeddings"]` to their original (pre-yarn) context
+        # length, with `config.max_position_embeddings` corresponding to their post-yarn context length.
+        # However, for BC purposes, we allow the former to be unset.
+        original_max_position_embeddings = self.rope_parameters.get("original_max_position_embeddings")
+        if original_max_position_embeddings is not None:
+            # Double-check: `factor` should be the ratio between the pre-yarn and post-yarn context lengths.
+            implicit_factor = self.max_position_embeddings / original_max_position_embeddings
+            if implicit_factor != factor:
+                logger.warning_once(
+                    f"The explicitly set RoPE scaling factor (config.rope_parameters['factor'] = {factor}) does not match "
+                    "the ratio implicitly set by other parameters (implicit factor = "
+                    "post-yarn context length / pre-yarn context length = "
+                    "config.max_position_embeddings / config.rope_parameters['original_max_position_embeddings'] = "
+                    f"{implicit_factor}). Using the explicit factor ({factor}) in YaRN. This may cause unexpected "
+                    "behaviour in model usage, please correct the 'max_position_embeddings' fields in the model config."
+                )
+        # No `config.rope_parameters["original_max_position_embeddings"]`. Is `config.max_position_embeddings` the
+        # pre-yarn or the post-yarn context length?
+        # BC: we assume it is the pre-yarn context length.
+        else:
             logger.warning_once(
-                f"The explicitly set RoPE scaling factor (config.rope_parameters['factor'] = {factor}) does not match "
-                "the ratio implicitly set by other parameters (implicit factor = "
-                "post-yarn context length / pre-yarn context length = "
-                "config.max_position_embeddings / config.rope_parameters['original_max_position_embeddings'] = "
-                f"{implicit_factor}). Using the explicit factor ({factor}) in YaRN. This may cause unexpected "
-                "behaviour in model usage, please correct the 'original_max_position_embeddings' fields in the model config."
+                "config.rope_parameters['original_max_position_embeddings'], the pre-yarn context length, is unset. We will "
+                "**assume** config.max_position_embeddings holds the pre-yarn context length. Some use cases may expect "
+                "config.max_position_embeddings to hold the post-yarn context length (pre-yarn context length * "
+                "factor) -- we recommend updating both fields for optimal downstream model usage."
             )
 
-    def _validate_longrope_rope_parameters(self, rope_parameters: dict, ignore_keys: set | None = None):
-        required_keys = {"rope_type", "short_factor", "long_factor", "rope_theta", "original_max_position_embeddings"}
-        optional_keys = {"attention_factor", "factor"}
+    def _validate_longrope_rope_parameters(self, rope_parameters: dict, ignore_keys: Optional[set] = None):
+        required_keys = {"rope_type", "short_factor", "long_factor", "rope_theta"}
+        # TODO (joao): update logic for the inclusion of `original_max_position_embeddings`
+        optional_keys = {"attention_factor", "factor", "original_max_position_embeddings"}
         received_keys = set(rope_parameters.keys())
         rope_type = rope_parameters["rope_type"]
         self._check_received_keys(rope_type, received_keys, required_keys, optional_keys, ignore_keys=ignore_keys)
@@ -824,30 +820,31 @@ class RotaryEmbeddingConfigMixin:
                 f"`rope_parameters`'s long_factor field must have length {dim // 2}, got {len(long_factor)}"
             )
 
-        factor = rope_parameters.get("factor")
-        original_max_position_embeddings = rope_parameters["original_max_position_embeddings"]
-
-        # Handle Phi3 divergence: we prefer the use of `attention_factor` and/or `factor` over
-        # `original_max_position_embeddings` to compute internal variables. The latter is undesirable
-        if factor is None and original_max_position_embeddings is not None:
+        # Handle Phi3 divergence: prefer the use of `attention_factor` and/or `factor` over
+        # `original_max_position_embeddings` to compute internal variables. The latter lives outside `rope_parameters` and is
+        # unique to longrope (= undesirable)
+        if hasattr(self, "original_max_position_embeddings"):
             logger.warning_once(
-                "This model config has set a `rope_parameters['original_max_position_embeddings']` field, to be used together with "
+                "This model has set a `original_max_position_embeddings` field, to be used together with "
                 "`max_position_embeddings` to determine a scaling factor. Please set the `factor` field of `rope_parameters`"
                 "with this ratio instead -- we recommend the use of this field over `original_max_position_embeddings`, "
                 "as it is compatible with most model architectures."
             )
-        elif factor is None and original_max_position_embeddings is None:
-            logger.warning("Missing required keys in `rope_parameters`: 'factor'")
-        elif not isinstance(factor, float) or factor < 1.0:
-            logger.warning(f"`rope_parameters`'s factor field must be a float >= 1, got {factor}")
-
-        attention_factor = rope_parameters.get("attention_factor")
-        if attention_factor is not None and (not isinstance(attention_factor, float) or attention_factor < 0.0):
-            logger.warning(
-                f"`rope_parameters`'s attention_factor field must be a float greater than 0, got {attention_factor}"
-            )
+        else:
+            factor = rope_parameters.get("factor")
+            if factor is None:
+                logger.warning("Missing required keys in `rope_parameters`: 'factor'")
+            elif not isinstance(factor, float) or factor < 1.0:
+                logger.warning(f"`rope_parameters`'s factor field must be a float >= 1, got {factor}")
+
+            attention_factor = rope_parameters.get("attention_factor")
+            if attention_factor is not None:
+                if not isinstance(attention_factor, float) or attention_factor < 0.0:
+                    logger.warning(
+                        f"`rope_parameters`'s attention_factor field must be a float greater than 0, got {attention_factor}"
+                    )
 
-    def _validate_llama3_rope_parameters(self, rope_parameters: dict, ignore_keys: set | None = None):
+    def _validate_llama3_rope_parameters(self, rope_parameters: dict, ignore_keys: Optional[set] = None):
         required_keys = {
             "rope_type",
             "factor",
@@ -893,8 +890,8 @@ class RotaryEmbeddingConfigMixin:
         rope_type: str,
         received_keys: set,
         required_keys: set,
-        optional_keys: set | None = None,
-        ignore_keys: set | None = None,
+        optional_keys: Optional[set] = None,
+        ignore_keys: Optional[set] = None,
     ):
         """Compare the received keys in `config.rope_parameters` against the expected and optional keys"""
         # BC: "rope_type" was originally "type" -- let's check for "rope_type" when "type" is present
@@ -902,10 +899,6 @@ class RotaryEmbeddingConfigMixin:
             received_keys -= {"type"}
             required_keys.add("rope_type")
 
-        optional_keys = optional_keys or set()
-        if "partial_rotary_factor" not in optional_keys:
-            optional_keys.add("partial_rotary_factor")
-
         # Some models need to store model-specific keys, and we don't want to throw warning at them
         if ignore_keys is not None:
             received_keys -= ignore_keys
@@ -914,23 +907,9 @@ class RotaryEmbeddingConfigMixin:
         if missing_keys:
             raise KeyError(f"Missing required keys in `rope_parameters` for 'rope_type'='{rope_type}': {missing_keys}")
 
-        unused_keys = received_keys - required_keys - optional_keys
+        if optional_keys is not None:
+            unused_keys = received_keys - required_keys - optional_keys
+        else:
+            unused_keys = received_keys - required_keys
         if unused_keys:
             logger.warning(f"Unrecognized keys in `rope_parameters` for 'rope_type'='{rope_type}': {unused_keys}")
-
-
-def rope_config_validation(config: RotaryEmbeddingConfigMixin, ignore_keys: set | None = None):
-    """
-    This is a deprecated function.
-    It has been kept for backward compatibility with custom code models.
-    """
-    warnings.warn(
-        "`rope_config_validation` is deprecated and has been removed. "
-        "Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. "
-        "PreTrainedConfig inherits this class, so please call self.validate_rope() instead. "
-        "Also, make sure to use the new rope_parameters syntax. "
-        "You can call self.standardize_rope_params() in the meantime.",
-        FutureWarning,
-    )
-    config.standardize_rope_params()
-    config.validate_rope(ignore_keys=ignore_keys)