liger-kernel-nightly 0.6.3.dev20251101160510__tar.gz → 0.6.3.dev20251105012545__tar.gz

{liger_kernel_nightly-0.6.3.dev20251101160510 → liger_kernel_nightly-0.6.3.dev20251105012545}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: liger_kernel_nightly
-Version: 0.6.3.dev20251101160510
+Version: 0.6.3.dev20251105012545
 Summary: Efficient Triton kernels for LLM Training
 License: BSD 2-CLAUSE LICENSE
         Copyright 2024 LinkedIn Corporation 

{liger_kernel_nightly-0.6.3.dev20251101160510 → liger_kernel_nightly-0.6.3.dev20251105012545}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "liger_kernel_nightly"
-version = "0.6.3.dev20251101160510"
+version = "0.6.3.dev20251105012545"
 description = "Efficient Triton kernels for LLM Training"
 urls = { "Homepage" = "https://github.com/linkedin/Liger-Kernel" }
 readme = { file = "README.md", content-type = "text/markdown" }

{liger_kernel_nightly-0.6.3.dev20251101160510 → liger_kernel_nightly-0.6.3.dev20251105012545}/src/liger_kernel/transformers/model/qwen3_vl_moe.py

@@ -5,13 +5,11 @@ from typing import Union
 
 import torch
 
+from transformers.models.qwen3_vl_moe.modeling_qwen3_vl_moe import Qwen3VLMoeCausalLMOutputWithPast
+from transformers.models.qwen3_vl_moe.modeling_qwen3_vl_moe import load_balancing_loss_func
 from transformers.utils import can_return_tuple
 
 from liger_kernel.transformers.model.loss_utils import LigerForCausalLMLoss
-from transformers.models.qwen3_vl_moe.modeling_qwen3_vl_moe import (
-    Qwen3VLMoeCausalLMOutputWithPast,
-    load_balancing_loss_func,
-)
 
 
 @can_return_tuple

{liger_kernel_nightly-0.6.3.dev20251101160510 → liger_kernel_nightly-0.6.3.dev20251105012545}/src/liger_kernel/transformers/monkey_patch.py

@@ -6,7 +6,6 @@ from types import MethodType
 from typing import Callable
 from typing import Optional
 
-import torch
 import transformers
 
 from packaging import version
@@ -36,11 +35,9 @@ from liger_kernel.transformers.model.qwen3_vl_moe import lce_forward as qwen3_vl
 from liger_kernel.transformers.model.smollm3 import lce_forward as smollm3_lce_forward
 from liger_kernel.transformers.qwen2vl_mrope import liger_multimodal_rotary_pos_emb
 from liger_kernel.transformers.rms_norm import LigerRMSNorm
-from liger_kernel.transformers.rope import (
-    liger_rotary_pos_emb,
-    liger_rotary_pos_emb_with_cast,
-    liger_rotary_pos_emb_with_cast_and_leading_batch,
-)
+from liger_kernel.transformers.rope import liger_rotary_pos_emb
+from liger_kernel.transformers.rope import liger_rotary_pos_emb_with_cast
+from liger_kernel.transformers.rope import liger_rotary_pos_emb_with_cast_and_leading_batch
 from liger_kernel.transformers.swiglu import LigerBlockSparseTop2MLP
 from liger_kernel.transformers.swiglu import LigerPhi3SwiGLUMLP
 from liger_kernel.transformers.swiglu import LigerSwiGLUMLP
@@ -64,7 +61,6 @@ def _bind_method_to_module(module, method_name: str, new_method: Callable):
     module.__dict__[method_name] = new_method.__get__(module, module.__class__)
 
 
-
 def _patch_rms_norm_module(module, offset=0.0, eps=1e-6, casting_mode="llama", in_place=True, row_mode=None):
     # Check if the module is a PEFT ModulesToSaveWrapper
     # If it is, we need to patch the modules_to_save.default and original_modules
@@ -1651,7 +1647,6 @@ def apply_liger_kernel_to_qwen2_5_vl(
                     _patch_rms_norm_module(decoder_layer.post_attention_layernorm)
 
 
-
 def apply_liger_kernel_to_qwen3_vl(
     rope: bool = True,
     cross_entropy: bool = False,
@@ -1688,7 +1683,6 @@ def apply_liger_kernel_to_qwen3_vl(
         modeling_qwen3_vl.apply_rotary_pos_emb = liger_rotary_pos_emb_with_cast
         modeling_qwen3_vl.apply_rotary_pos_emb_vision = liger_rotary_pos_emb_with_cast_and_leading_batch
 
-
     if rms_norm:
         modeling_qwen3_vl.Qwen3VLTextRMSNorm = LigerRMSNorm
 

{liger_kernel_nightly-0.6.3.dev20251101160510 → liger_kernel_nightly-0.6.3.dev20251105012545}/src/liger_kernel/transformers/rope.py

@@ -1,4 +1,5 @@
-from typing import Optional, Tuple
+from typing import Optional
+from typing import Tuple
 
 import torch
 
@@ -32,7 +33,6 @@ def liger_rotary_pos_emb_with_cast(
     position_ids: Optional[torch.Tensor] = None,
     unsqueeze_dim: int = 1,
 ) -> Tuple[torch.Tensor, torch.Tensor]:
-
     orig_q_dtype, orig_k_dtype = q.dtype, k.dtype
 
     q32 = q.to(torch.float32)
@@ -52,8 +52,6 @@ def liger_rotary_pos_emb_with_cast_and_leading_batch(
     position_ids: Optional[torch.Tensor] = None,
     unsqueeze_dim: int = 1,
 ) -> Tuple[torch.Tensor, torch.Tensor]:
-
-
     orig_q_dtype, orig_k_dtype = q.dtype, k.dtype
 
     q32 = q.to(torch.float32).unsqueeze(0)
@@ -61,7 +59,5 @@ def liger_rotary_pos_emb_with_cast_and_leading_batch(
     cos32 = cos.to(torch.float32).unsqueeze(0)
     sin32 = sin.to(torch.float32).unsqueeze(0)
 
-    q_out, k_out = liger_rotary_pos_emb(
-        q32, k32, cos32, sin32, position_ids=position_ids, unsqueeze_dim=unsqueeze_dim
-    )
+    q_out, k_out = liger_rotary_pos_emb(q32, k32, cos32, sin32, position_ids=position_ids, unsqueeze_dim=unsqueeze_dim)
     return q_out.to(orig_q_dtype).squeeze(0), k_out.to(orig_k_dtype).squeeze(0)

{liger_kernel_nightly-0.6.3.dev20251101160510 → liger_kernel_nightly-0.6.3.dev20251105012545}/src/liger_kernel_nightly.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: liger_kernel_nightly
-Version: 0.6.3.dev20251101160510
+Version: 0.6.3.dev20251105012545
 Summary: Efficient Triton kernels for LLM Training
 License: BSD 2-CLAUSE LICENSE
         Copyright 2024 LinkedIn Corporation 

{liger_kernel_nightly-0.6.3.dev20251101160510 → liger_kernel_nightly-0.6.3.dev20251105012545}/test/convergence/bf16/test_mini_models_with_logits.py

@@ -1347,11 +1347,10 @@ def run_mini_model(
 
         if "llava" in model_name:
             apply_liger_kernel_to_llama(**kwargs)
-            
+
         kwargs["fused_linear_cross_entropy"] = False
         kwargs["cross_entropy"] = False
 
-            
         MINI_MODEL_SETUPS[model_name].liger_kernel_patch_func(**kwargs)
     else:
         MINI_MODEL_SETUPS[model_name].liger_kernel_patch_revert_func(**revert_kwargs)

{liger_kernel_nightly-0.6.3.dev20251101160510 → liger_kernel_nightly-0.6.3.dev20251105012545}/test/convergence/fp32/test_mini_models.py

@@ -42,13 +42,11 @@ from liger_kernel.transformers import apply_liger_kernel_to_phi3
 from liger_kernel.transformers import apply_liger_kernel_to_qwen2
 from liger_kernel.transformers import apply_liger_kernel_to_qwen2_5_vl
 from liger_kernel.transformers import apply_liger_kernel_to_qwen2_vl
-from liger_kernel.transformers import apply_liger_kernel_to_qwen3_vl
-from liger_kernel.transformers import apply_liger_kernel_to_qwen3_vl_moe
 from liger_kernel.transformers import apply_liger_kernel_to_qwen3
 from liger_kernel.transformers import apply_liger_kernel_to_qwen3_moe
+from liger_kernel.transformers import apply_liger_kernel_to_qwen3_next
 from liger_kernel.transformers import apply_liger_kernel_to_qwen3_vl
 from liger_kernel.transformers import apply_liger_kernel_to_qwen3_vl_moe
-from liger_kernel.transformers import apply_liger_kernel_to_qwen3_next
 from liger_kernel.transformers import apply_liger_kernel_to_smollm3
 from test.utils import DEFAULT_DATASET_PATH
 from test.utils import MiniModelConfig
@@ -76,13 +74,11 @@ from test.utils import revert_liger_kernel_to_phi3
 from test.utils import revert_liger_kernel_to_qwen2
 from test.utils import revert_liger_kernel_to_qwen2_5_vl
 from test.utils import revert_liger_kernel_to_qwen2_vl
-from test.utils import revert_liger_kernel_to_qwen3_vl
-from test.utils import revert_liger_kernel_to_qwen3_vl_moe
 from test.utils import revert_liger_kernel_to_qwen3
 from test.utils import revert_liger_kernel_to_qwen3_moe
+from test.utils import revert_liger_kernel_to_qwen3_next
 from test.utils import revert_liger_kernel_to_qwen3_vl
 from test.utils import revert_liger_kernel_to_qwen3_vl_moe
-from test.utils import revert_liger_kernel_to_qwen3_next
 from test.utils import revert_liger_kernel_to_smollm3
 from test.utils import set_seed
 from test.utils import simple_collate_fn

{liger_kernel_nightly-0.6.3.dev20251101160510 → liger_kernel_nightly-0.6.3.dev20251105012545}/test/convergence/fp32/test_mini_models_multimodal.py

@@ -1281,7 +1281,6 @@ def run_mini_model_multimodal(
         revert_kwargs["model_type"] = "conditional_generation"
 
     if with_liger is True:
-            
         kwargs = {
             "rope": True,
             "rms_norm": True,

{liger_kernel_nightly-0.6.3.dev20251101160510 → liger_kernel_nightly-0.6.3.dev20251105012545}/test/convergence/fp32/test_mini_models_with_logits.py

@@ -42,13 +42,11 @@ from liger_kernel.transformers import apply_liger_kernel_to_phi3
 from liger_kernel.transformers import apply_liger_kernel_to_qwen2
 from liger_kernel.transformers import apply_liger_kernel_to_qwen2_5_vl
 from liger_kernel.transformers import apply_liger_kernel_to_qwen2_vl
-from liger_kernel.transformers import apply_liger_kernel_to_qwen3_vl
-from liger_kernel.transformers import apply_liger_kernel_to_qwen3_vl_moe
 from liger_kernel.transformers import apply_liger_kernel_to_qwen3
 from liger_kernel.transformers import apply_liger_kernel_to_qwen3_moe
+from liger_kernel.transformers import apply_liger_kernel_to_qwen3_next
 from liger_kernel.transformers import apply_liger_kernel_to_qwen3_vl
 from liger_kernel.transformers import apply_liger_kernel_to_qwen3_vl_moe
-from liger_kernel.transformers import apply_liger_kernel_to_qwen3_next
 from liger_kernel.transformers import apply_liger_kernel_to_smollm3
 from test.utils import DEFAULT_DATASET_PATH
 from test.utils import MiniModelConfig
@@ -76,13 +74,11 @@ from test.utils import revert_liger_kernel_to_phi3
 from test.utils import revert_liger_kernel_to_qwen2
 from test.utils import revert_liger_kernel_to_qwen2_5_vl
 from test.utils import revert_liger_kernel_to_qwen2_vl
-from test.utils import revert_liger_kernel_to_qwen3_vl
-from test.utils import revert_liger_kernel_to_qwen3_vl_moe
 from test.utils import revert_liger_kernel_to_qwen3
 from test.utils import revert_liger_kernel_to_qwen3_moe
+from test.utils import revert_liger_kernel_to_qwen3_next
 from test.utils import revert_liger_kernel_to_qwen3_vl
 from test.utils import revert_liger_kernel_to_qwen3_vl_moe
-from test.utils import revert_liger_kernel_to_qwen3_next
 from test.utils import revert_liger_kernel_to_smollm3
 from test.utils import set_seed
 from test.utils import simple_collate_fn

{liger_kernel_nightly-0.6.3.dev20251101160510 → liger_kernel_nightly-0.6.3.dev20251105012545}/test/transformers/test_monkey_patch.py

@@ -498,9 +498,7 @@ def test_apply_liger_kernel_to_instance_for_qwen3_vl_for_conditional_generation(
             LigerRMSNorm.forward
         )
         for decoder_layer in dummy_model_instance.language_model.layers:
-            assert inspect.getsource(decoder_layer.input_layernorm.forward) != inspect.getsource(
-                LigerRMSNorm.forward
-            )
+            assert inspect.getsource(decoder_layer.input_layernorm.forward) != inspect.getsource(LigerRMSNorm.forward)
             assert inspect.getsource(decoder_layer.post_attention_layernorm.forward) != inspect.getsource(
                 LigerRMSNorm.forward
             )
@@ -520,9 +518,7 @@ def test_apply_liger_kernel_to_instance_for_qwen3_vl_for_conditional_generation(
             LigerRMSNorm.forward
         )
         for decoder_layer in dummy_model_instance.language_model.layers:
-            assert inspect.getsource(decoder_layer.input_layernorm.forward) == inspect.getsource(
-                LigerRMSNorm.forward
-            )
+            assert inspect.getsource(decoder_layer.input_layernorm.forward) == inspect.getsource(LigerRMSNorm.forward)
             assert inspect.getsource(decoder_layer.post_attention_layernorm.forward) == inspect.getsource(
                 LigerRMSNorm.forward
             )
@@ -603,9 +599,7 @@ def test_apply_liger_kernel_to_instance_for_qwen3_vl():
             LigerRMSNorm.forward
         )
         for decoder_layer in dummy_model_instance.language_model.layers:
-            assert inspect.getsource(decoder_layer.input_layernorm.forward) != inspect.getsource(
-                LigerRMSNorm.forward
-            )
+            assert inspect.getsource(decoder_layer.input_layernorm.forward) != inspect.getsource(LigerRMSNorm.forward)
             assert inspect.getsource(decoder_layer.post_attention_layernorm.forward) != inspect.getsource(
                 LigerRMSNorm.forward
             )
@@ -625,9 +619,7 @@ def test_apply_liger_kernel_to_instance_for_qwen3_vl():
             LigerRMSNorm.forward
         )
         for decoder_layer in dummy_model_instance.language_model.layers:
-            assert inspect.getsource(decoder_layer.input_layernorm.forward) == inspect.getsource(
-                LigerRMSNorm.forward
-            )
+            assert inspect.getsource(decoder_layer.input_layernorm.forward) == inspect.getsource(LigerRMSNorm.forward)
             assert inspect.getsource(decoder_layer.post_attention_layernorm.forward) == inspect.getsource(
                 LigerRMSNorm.forward
             )
@@ -681,9 +673,7 @@ def test_apply_liger_kernel_to_instance_for_qwen3_vl_text():
         # Note: Text models don't have forward method patching, so skip this check
         assert inspect.getsource(dummy_model_instance.norm.forward) != inspect.getsource(LigerRMSNorm.forward)
         for decoder_layer in dummy_model_instance.layers:
-            assert inspect.getsource(decoder_layer.input_layernorm.forward) != inspect.getsource(
-                LigerRMSNorm.forward
-            )
+            assert inspect.getsource(decoder_layer.input_layernorm.forward) != inspect.getsource(LigerRMSNorm.forward)
             assert inspect.getsource(decoder_layer.post_attention_layernorm.forward) != inspect.getsource(
                 LigerRMSNorm.forward
             )
@@ -701,9 +691,7 @@ def test_apply_liger_kernel_to_instance_for_qwen3_vl_text():
         # Note: Text models don't have forward method patching, so skip this check
         assert inspect.getsource(dummy_model_instance.norm.forward) == inspect.getsource(LigerRMSNorm.forward)
         for decoder_layer in dummy_model_instance.layers:
-            assert inspect.getsource(decoder_layer.input_layernorm.forward) == inspect.getsource(
-                LigerRMSNorm.forward
-            )
+            assert inspect.getsource(decoder_layer.input_layernorm.forward) == inspect.getsource(LigerRMSNorm.forward)
             assert inspect.getsource(decoder_layer.post_attention_layernorm.forward) == inspect.getsource(
                 LigerRMSNorm.forward
             )
@@ -789,9 +777,7 @@ def test_apply_liger_kernel_to_instance_for_qwen3_vl_moe_for_conditional_generat
             LigerRMSNorm.forward
         )
         for decoder_layer in dummy_model_instance.language_model.layers:
-            assert inspect.getsource(decoder_layer.input_layernorm.forward) != inspect.getsource(
-                LigerRMSNorm.forward
-            )
+            assert inspect.getsource(decoder_layer.input_layernorm.forward) != inspect.getsource(LigerRMSNorm.forward)
             assert inspect.getsource(decoder_layer.post_attention_layernorm.forward) != inspect.getsource(
                 LigerRMSNorm.forward
             )
@@ -811,9 +797,7 @@ def test_apply_liger_kernel_to_instance_for_qwen3_vl_moe_for_conditional_generat
             LigerRMSNorm.forward
         )
         for decoder_layer in dummy_model_instance.language_model.layers:
-            assert inspect.getsource(decoder_layer.input_layernorm.forward) == inspect.getsource(
-                LigerRMSNorm.forward
-            )
+            assert inspect.getsource(decoder_layer.input_layernorm.forward) == inspect.getsource(LigerRMSNorm.forward)
             assert inspect.getsource(decoder_layer.post_attention_layernorm.forward) == inspect.getsource(
                 LigerRMSNorm.forward
             )
@@ -899,9 +883,7 @@ def test_apply_liger_kernel_to_instance_for_qwen3_vl_moe():
             LigerRMSNorm.forward
         )
         for decoder_layer in dummy_model_instance.language_model.layers:
-            assert inspect.getsource(decoder_layer.input_layernorm.forward) != inspect.getsource(
-                LigerRMSNorm.forward
-            )
+            assert inspect.getsource(decoder_layer.input_layernorm.forward) != inspect.getsource(LigerRMSNorm.forward)
             assert inspect.getsource(decoder_layer.post_attention_layernorm.forward) != inspect.getsource(
                 LigerRMSNorm.forward
             )
@@ -921,9 +903,7 @@ def test_apply_liger_kernel_to_instance_for_qwen3_vl_moe():
             LigerRMSNorm.forward
         )
         for decoder_layer in dummy_model_instance.language_model.layers:
-            assert inspect.getsource(decoder_layer.input_layernorm.forward) == inspect.getsource(
-                LigerRMSNorm.forward
-            )
+            assert inspect.getsource(decoder_layer.input_layernorm.forward) == inspect.getsource(LigerRMSNorm.forward)
             assert inspect.getsource(decoder_layer.post_attention_layernorm.forward) == inspect.getsource(
                 LigerRMSNorm.forward
             )
@@ -982,9 +962,7 @@ def test_apply_liger_kernel_to_instance_for_qwen3_vl_moe_text():
         # Note: Text models don't have forward method patching, so skip this check
         assert inspect.getsource(dummy_model_instance.norm.forward) != inspect.getsource(LigerRMSNorm.forward)
         for decoder_layer in dummy_model_instance.layers:
-            assert inspect.getsource(decoder_layer.input_layernorm.forward) != inspect.getsource(
-                LigerRMSNorm.forward
-            )
+            assert inspect.getsource(decoder_layer.input_layernorm.forward) != inspect.getsource(LigerRMSNorm.forward)
             assert inspect.getsource(decoder_layer.post_attention_layernorm.forward) != inspect.getsource(
                 LigerRMSNorm.forward
             )
@@ -1002,9 +980,7 @@ def test_apply_liger_kernel_to_instance_for_qwen3_vl_moe_text():
         # Note: Text models don't have forward method patching, so skip this check
         assert inspect.getsource(dummy_model_instance.norm.forward) == inspect.getsource(LigerRMSNorm.forward)
         for decoder_layer in dummy_model_instance.layers:
-            assert inspect.getsource(decoder_layer.input_layernorm.forward) == inspect.getsource(
-                LigerRMSNorm.forward
-            )
+            assert inspect.getsource(decoder_layer.input_layernorm.forward) == inspect.getsource(LigerRMSNorm.forward)
             assert inspect.getsource(decoder_layer.post_attention_layernorm.forward) == inspect.getsource(
                 LigerRMSNorm.forward
             )
@@ -1025,10 +1001,8 @@ def test_apply_liger_kernel_to_instance_for_qwen3_vl_moe_text():
 def test_qwen3_vl_rope_hooks_applied():
     # Ensure any monkey patching is cleaned up for subsequent tests
     with patch("transformers.models.qwen3_vl.modeling_qwen3_vl") as modeling_mod:
-        from liger_kernel.transformers.monkey_patch import (
-            liger_rotary_pos_emb,
-            _liger_qwen3_vl_apply_rotary_pos_emb_vision,
-        )
+        from liger_kernel.transformers.monkey_patch import _liger_qwen3_vl_apply_rotary_pos_emb_vision
+        from liger_kernel.transformers.monkey_patch import liger_rotary_pos_emb
 
         # Before applying, make sure attributes exist but are not the liger implementations
         setattr(modeling_mod, "apply_rotary_pos_emb", object())
@@ -1044,10 +1018,8 @@ def test_qwen3_vl_rope_hooks_applied():
 def test_qwen3_vl_moe_rope_hooks_applied():
     # Ensure any monkey patching is cleaned up for subsequent tests
     with patch("transformers.models.qwen3_vl_moe.modeling_qwen3_vl_moe") as modeling_mod:
-        from liger_kernel.transformers.monkey_patch import (
-            liger_rotary_pos_emb,
-            _liger_qwen3_vl_apply_rotary_pos_emb_vision,
-        )
+        from liger_kernel.transformers.monkey_patch import _liger_qwen3_vl_apply_rotary_pos_emb_vision
+        from liger_kernel.transformers.monkey_patch import liger_rotary_pos_emb
 
         # Before applying, make sure attributes exist but are not the liger implementations
         setattr(modeling_mod, "apply_rotary_pos_emb", object())