liger-kernel-nightly 0.6.3.dev20251028065948__tar.gz → 0.6.3.dev20251031170118__tar.gz

{liger_kernel_nightly-0.6.3.dev20251028065948 → liger_kernel_nightly-0.6.3.dev20251031170118}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: liger_kernel_nightly
-Version: 0.6.3.dev20251028065948
+Version: 0.6.3.dev20251031170118
 Summary: Efficient Triton kernels for LLM Training
 License: BSD 2-CLAUSE LICENSE
         Copyright 2024 LinkedIn Corporation 

{liger_kernel_nightly-0.6.3.dev20251028065948 → liger_kernel_nightly-0.6.3.dev20251031170118}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "liger_kernel_nightly"
-version = "0.6.3.dev20251028065948"
+version = "0.6.3.dev20251031170118"
 description = "Efficient Triton kernels for LLM Training"
 urls = { "Homepage" = "https://github.com/linkedin/Liger-Kernel" }
 readme = { file = "README.md", content-type = "text/markdown" }

{liger_kernel_nightly-0.6.3.dev20251028065948 → liger_kernel_nightly-0.6.3.dev20251031170118}/src/liger_kernel/transformers/monkey_patch.py

@@ -2038,6 +2038,7 @@ def apply_liger_kernel_to_internvl(
     cross_entropy: bool = False,
     fused_linear_cross_entropy: bool = True,
     rms_norm: bool = True,
+    layer_norm: bool = True,
     model: Optional[PreTrainedModel] = None,
     **kwargs,
 ) -> None:
@@ -2048,37 +2049,60 @@ def apply_liger_kernel_to_internvl(
     NOTE: InternVL is not available in transformers<4.52.1
 
     Args:
-        rope (bool): Whether to apply Liger's rotary position embedding. Default is True.
         cross_entropy (bool): Whether to apply Liger's cross entropy loss. Default is False.
         fused_linear_cross_entropy (bool):
             Whether to apply Liger's fused linear cross entropy loss. Default is True.
             `cross_entropy` and `fused_linear_cross_entropy` cannot both be True.
             If `fused_linear_cross_entropy` is True, the logits will not be materialized but more memory efficient.
         rms_norm (bool): Whether to apply Liger's RMSNorm. Default is True.
-        swiglu (bool): Whether to apply Liger's SwiGLU MLP. Default is True.
+        layer_norm (bool): Whether to apply Liger's LayerNorm. Default is True.
         model (PreTrainedModel): The model instance to apply Liger kernels to, if the model has already been
         loaded. Default is None.
     """
     assert not (cross_entropy and fused_linear_cross_entropy), (
         "cross_entropy and fused_linear_cross_entropy cannot both be True."
     )
+    import torch.nn as torch_nn
 
     from transformers.models.internvl import modeling_internvl
+    from transformers.models.internvl.modeling_internvl import InternVLForConditionalGeneration
+    from transformers.models.internvl.modeling_internvl import InternVLModel
+    from transformers.models.internvl.modeling_internvl import InternVLVisionLayer
+    from transformers.models.internvl.modeling_internvl import InternVLVisionModel
+    from transformers.models.internvl.modeling_internvl import InternVLVisionRMSNorm
 
+    from liger_kernel.transformers.layer_norm import LigerLayerNorm
     from liger_kernel.transformers.model.internvl import lce_forward as internvl_lce_forward
+    from liger_kernel.transformers.rms_norm import LigerRMSNorm
+
+    if layer_norm and model is None:
+        modeling_internvl.nn.LayerNorm = LigerLayerNorm
 
     if cross_entropy:
-        logger.warning(TRANSFORMER_DEPRECATION_WARNING)
-        modeling_internvl.nn.CrossEntropyLoss = LigerCrossEntropyLoss
+        logger.info("Apply liger cross entropy")
+
+        from transformers.loss.loss_utils import nn
+
+        nn.functional.cross_entropy = liger_cross_entropy
     if fused_linear_cross_entropy:
         modeling_internvl.InternVLForConditionalGeneration.forward = internvl_lce_forward
     if rms_norm:
         modeling_internvl.InternVLVisionRMSNorm = LigerRMSNorm
 
     if model is not None:
-        text_model_name, vision_model_name = model.config.text_config.model_type, model.config.vision_config.model_type
+        # The model instance already exists, so we need to additionally patch the
+        # instance variables that reference already-instantiated modules
+        if isinstance(model, (InternVLForConditionalGeneration, InternVLModel)):
+            # NOTE: language_model and visual properties can be accessed throught conditional class.
+            text_model = model.language_model
+            vision_model: InternVLVisionModel = model.vision_tower
+        else:
+            raise TypeError(
+                f"Unsupported internvl model type. `model` must be `InternVLForConditionalGeneration`, `InternVLModel`. Got: {type(model)}"
+            )
+
+        text_model_name = model.config.text_config.model_type
         text_liger_fn = MODEL_TYPE_TO_APPLY_LIGER_FN.get(text_model_name, None)
-        vision_liger_fn = MODEL_TYPE_TO_APPLY_LIGER_FN.get(vision_model_name, None)
 
         kwargs = {"cross_entropy": False, "fused_linear_cross_entropy": False, **kwargs} | {"rms_norm": rms_norm}
         if text_liger_fn:
@@ -2091,25 +2115,33 @@ def apply_liger_kernel_to_internvl(
                     f"These parameters are not supported by {text_model_name}. Enter the remaining {list(text_kwargs.keys())} except for {list(remain_params)}\n"
                     f"Parameters accepted by {text_model_name}: {list(accept_params.keys())}"
                 )
-            text_kwargs["model"] = model.language_model
+            text_kwargs["model"] = text_model
             text_liger_fn(**text_kwargs)
         elif text_model_name not in MODEL_TYPE_TO_APPLY_LIGER_FN:
             logger.warning(f"{text_model_name} is not supported by Liger kernel.")
 
-        if vision_liger_fn:
-            accept_params = inspect.signature(vision_liger_fn).parameters
-            remain_params = set(kwargs) - (set(accept_params) & set(kwargs))
-            vision_kwargs = {k: v for k, v in kwargs.items() if k not in remain_params}
+        # Patch vision model RMSNorm layers
+        if rms_norm:
+            for encoder_layer in vision_model.encoder.layer:
+                encoder_layer: InternVLVisionLayer
+                if isinstance(encoder_layer.attention.q_norm, InternVLVisionRMSNorm):
+                    _patch_rms_norm_module(encoder_layer.attention.q_norm)
+                if isinstance(encoder_layer.attention.k_norm, InternVLVisionRMSNorm):
+                    _patch_rms_norm_module(encoder_layer.attention.k_norm)
 
-            if remain_params:
-                logger.warning(
-                    f"These parameters are not supported by {vision_model_name}. Enter the remaining {list(vision_kwargs.keys())} except for {list(remain_params)}\n"
-                    f"Parameters accepted by {vision_model_name}: {list(accept_params.keys())}"
-                )
-            vision_kwargs["model"] = model.vision_tower
-            vision_liger_fn(**vision_kwargs)
-        elif vision_model_name not in MODEL_TYPE_TO_APPLY_LIGER_FN:
-            logger.warning(f"{vision_model_name} is not supported by Liger kernel.")
+        # Patch vision model LayerNorm layers
+        if layer_norm:
+            # Patch layernorm
+            if isinstance(vision_model.layernorm, torch_nn.LayerNorm):
+                _patch_layer_norm_module(vision_model.layernorm)
+
+            # Patch encoder layers
+            for encoder_layer in vision_model.encoder.layer:
+                encoder_layer: InternVLVisionLayer
+                if isinstance(encoder_layer.layernorm_before, torch_nn.LayerNorm):
+                    _patch_layer_norm_module(encoder_layer.layernorm_before)
+                if isinstance(encoder_layer.layernorm_after, torch_nn.LayerNorm):
+                    _patch_layer_norm_module(encoder_layer.layernorm_after)
 
 
 def apply_liger_kernel_to_smolvlm(

{liger_kernel_nightly-0.6.3.dev20251028065948 → liger_kernel_nightly-0.6.3.dev20251031170118}/src/liger_kernel_nightly.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: liger_kernel_nightly
-Version: 0.6.3.dev20251028065948
+Version: 0.6.3.dev20251031170118
 Summary: Efficient Triton kernels for LLM Training
 License: BSD 2-CLAUSE LICENSE
         Copyright 2024 LinkedIn Corporation 

{liger_kernel_nightly-0.6.3.dev20251028065948 → liger_kernel_nightly-0.6.3.dev20251031170118}/test/convergence/bf16/test_mini_models.py

@@ -1004,7 +1004,7 @@ if GLM4V_MOE_AVAILABLE:
                 "moe_intermediate_size": 1408,
                 "num_experts_per_tok": 2,
                 "n_shared_experts": 1,
-                "n_routed_experts": 128,
+                "n_routed_experts": 8,
                 "routed_scaling_factor": 1.0,
                 "n_group": 1,
                 "topk_group": 1,
@@ -1536,9 +1536,9 @@ def run_mini_model(
             1e-5,
             torch.bfloat16,
             1e-2,
-            4e-1,
+            4e-1,  # rms_norm patch needs higher tolerance in bf16
             1e-1,
-            5e-1,  # TODO: very high tolerance set for now, need to investigate
+            5e-1,  # rms_norm patch needs higher tolerance in bf16
             2e-1,
             1e-2,
             marks=[

{liger_kernel_nightly-0.6.3.dev20251028065948 → liger_kernel_nightly-0.6.3.dev20251031170118}/test/convergence/bf16/test_mini_models_with_logits.py

@@ -1003,7 +1003,7 @@ if GLM4V_MOE_AVAILABLE:
                 "moe_intermediate_size": 1408,
                 "num_experts_per_tok": 2,
                 "n_shared_experts": 1,
-                "n_routed_experts": 128,
+                "n_routed_experts": 8,
                 "routed_scaling_factor": 1.0,
                 "n_group": 1,
                 "topk_group": 1,
@@ -1557,9 +1557,9 @@ def run_mini_model(
             1e-5,
             torch.bfloat16,
             1e-2,
-            1e-2,
+            4e-1,  # rms_norm patch needs higher tolerance in bf16
             1e-1,
-            1e-2,
+            5e-1,  # rms_norm patch needs higher tolerance in bf16
             1e-2,
             1e-2,
             marks=[

{liger_kernel_nightly-0.6.3.dev20251028065948 → liger_kernel_nightly-0.6.3.dev20251031170118}/test/convergence/fp32/test_mini_models.py

@@ -942,7 +942,7 @@ if GLM4V_MOE_AVAILABLE:
                 "moe_intermediate_size": 1408,
                 "num_experts_per_tok": 2,
                 "n_shared_experts": 1,
-                "n_routed_experts": 128,
+                "n_routed_experts": 8,
                 "routed_scaling_factor": 1.0,
                 "n_group": 1,
                 "topk_group": 1,

{liger_kernel_nightly-0.6.3.dev20251028065948 → liger_kernel_nightly-0.6.3.dev20251031170118}/test/convergence/fp32/test_mini_models_with_logits.py

@@ -1000,7 +1000,7 @@ if GLM4V_MOE_AVAILABLE:
                 "moe_intermediate_size": 1408,
                 "num_experts_per_tok": 2,
                 "n_shared_experts": 1,
-                "n_routed_experts": 128,
+                "n_routed_experts": 8,
                 "routed_scaling_factor": 1.0,
                 "n_group": 1,
                 "topk_group": 1,

{liger_kernel_nightly-0.6.3.dev20251028065948 → liger_kernel_nightly-0.6.3.dev20251031170118}/test/transformers/test_monkey_patch.py

@@ -1637,21 +1637,30 @@ def test_apply_liger_kernel_to_instance_for_internvl():
 
         # Instantiate a dummy model
         config = transformers.models.internvl.configuration_internvl.InternVLConfig(
-            torch_dtype=torch.bfloat16,
-            rms_norm_eps=1e-5,
-            hidden_size=32,
-            intermediate_size=48,
-            hidden_act="silu",
-            num_hidden_layers=2,
-            num_attention_heads=2,
-            max_position_embeddings=128,
-            vocab_size=1000,
+            dtype=torch.bfloat16,
+            text_config={
+                "rms_norm_eps": 1e-5,
+                "hidden_size": 256,  # 1024
+                "intermediate_size": 1024,  # 4096
+                "hidden_act": "silu",
+                "num_hidden_layers": 4,  # 24
+                "num_attention_heads": 4,  # 16
+                "num_key_value_heads": 2,  # 16
+                "max_position_embeddings": 4096,  # 8192
+                "vocab_size": 32000,  # 151936
+                "bos_token_id": 1,
+                "eos_token_id": 2,
+                "pad_token_id": 2,
+                "tie_word_embeddings": False,
+            },
             vision_config={
-                "depth": 4,
-                "embed_dim": 128,
-                "num_heads": 8,
-                "hidden_size": 1024,
+                "hidden_size": 256,  # 1024
+                "intermediate_size": 1024,  # 4096
+                "num_hidden_layers": 4,  # 24
+                "num_attention_heads": 4,  # 16
             },
+            image_token_id=10,
+            attn_implementation="sdpa",  # default value, pytorch native attention
         )
         dummy_model_instance = InternVLForConditionalGeneration._from_config(config)
 
@@ -1692,7 +1701,7 @@ def test_apply_liger_kernel_to_instance_for_smolvlm2():
 
         # Instantiate a dummy model
         config = transformers.models.smolvlm.configuration_smolvlm.SmolVLMConfig(
-            torch_dtype=torch.bfloat16,
+            dtype=torch.bfloat16,
             text_config={
                 "rms_norm_eps": 1e-5,
                 "hidden_size": 576,

{liger_kernel_nightly-0.6.3.dev20251028065948 → liger_kernel_nightly-0.6.3.dev20251031170118}/test/utils.py

@@ -585,10 +585,12 @@ def revert_liger_kernel_to_internvl(model_config: MiniModelConfig):
     """
     Revert all Liger kernel patches applied to InternVL.
     """
+    import torch.nn as nn
 
     from transformers.models.internvl import modeling_internvl
     from transformers.models.qwen2 import modeling_qwen2
 
+    importlib.reload(nn)
     importlib.reload(modeling_internvl)
     importlib.reload(modeling_qwen2)