liger-kernel-nightly 0.6.0.dev20250708194445__tar.gz → 0.6.0.dev20250709042125__tar.gz

{liger_kernel_nightly-0.6.0.dev20250708194445 → liger_kernel_nightly-0.6.0.dev20250709042125}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: liger_kernel_nightly
-Version: 0.6.0.dev20250708194445
+Version: 0.6.0.dev20250709042125
 Summary: Efficient Triton kernels for LLM Training
 License: BSD 2-CLAUSE LICENSE
         Copyright 2024 LinkedIn Corporation 
@@ -33,7 +33,7 @@ License-File: NOTICE
 Requires-Dist: torch>=2.1.2
 Requires-Dist: triton>=2.3.1
 Provides-Extra: dev
-Requires-Dist: transformers>=4.44.2; extra == "dev"
+Requires-Dist: transformers>=4.49.0; extra == "dev"
 Requires-Dist: matplotlib>=3.7.2; extra == "dev"
 Requires-Dist: flake8>=4.0.1.1; extra == "dev"
 Requires-Dist: black>=24.4.2; extra == "dev"

{liger_kernel_nightly-0.6.0.dev20250708194445 → liger_kernel_nightly-0.6.0.dev20250709042125}/dev/modal/tests.py

@@ -14,7 +14,7 @@ app = modal.App("liger_tests", image=image)
 repo = image.add_local_dir(ROOT_PATH, remote_path=REMOTE_ROOT_PATH)
 
 
-@app.function(gpu="A10G", image=repo, timeout=60 * 45)
+@app.function(gpu="A10G", image=repo, timeout=60 * 60)
 def liger_tests():
     import subprocess
 

{liger_kernel_nightly-0.6.0.dev20250708194445 → liger_kernel_nightly-0.6.0.dev20250709042125}/dev/modal/tests_bwd.py

@@ -14,7 +14,7 @@ app = modal.App("liger_tests_bwd", image=image)
 repo = image.add_local_dir(ROOT_PATH, remote_path=REMOTE_ROOT_PATH)
 
 
-@app.function(gpu="A10G", image=repo, timeout=60 * 30)
+@app.function(gpu="A10G", image=repo, timeout=60 * 60)
 def liger_bwd_tests():
     import subprocess
 
@@ -24,9 +24,9 @@ def liger_bwd_tests():
         shell=True,
         cwd=REMOTE_ROOT_PATH,
     )
-    # force install transformers==4.44.2
+    # force install transformers==4.49.0
     subprocess.run(
-        ["uv pip install transformers==4.44.2 --system"],
+        ["uv pip install transformers==4.49.0 --system"],
         check=True,
         shell=True,
         cwd=REMOTE_ROOT_PATH,

{liger_kernel_nightly-0.6.0.dev20250708194445 → liger_kernel_nightly-0.6.0.dev20250709042125}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "liger_kernel_nightly"
-version = "0.6.0.dev20250708194445"
+version = "0.6.0.dev20250709042125"
 description = "Efficient Triton kernels for LLM Training"
 urls = { "Homepage" = "https://github.com/linkedin/Liger-Kernel" }
 readme = { file = "README.md", content-type = "text/markdown" }

{liger_kernel_nightly-0.6.0.dev20250708194445 → liger_kernel_nightly-0.6.0.dev20250709042125}/setup.py

@@ -31,7 +31,7 @@ def get_optional_dependencies():
     """Get optional dependency groups."""
     return {
         "dev": [
-            "transformers>=4.44.2",
+            "transformers>=4.49.0",
             "matplotlib>=3.7.2",
             "flake8>=4.0.1.1",
             "black>=24.4.2",

{liger_kernel_nightly-0.6.0.dev20250708194445 → liger_kernel_nightly-0.6.0.dev20250709042125}/src/liger_kernel/transformers/monkey_patch.py

@@ -537,7 +537,10 @@ def apply_liger_kernel_to_mllama(
         if isinstance(model, MllamaForConditionalGeneration):
             language_model: MllamaForCausalLM = model.language_model
             vision_model: MllamaVisionModel = model.vision_model
-            text_model: MllamaTextModel = language_model
+            if isinstance(language_model, MllamaForCausalLM):
+                text_model: MllamaTextModel = language_model.model
+            else:
+                text_model = language_model
         elif isinstance(model, MllamaForCausalLM):
             text_model = model.model
             vision_model = None
@@ -1096,7 +1099,9 @@ def apply_liger_kernel_to_paligemma(
     # PaliGemma submodules are ['vision_tower', 'multi_modal_projector', 'language_model']
 
     from transformers.models.gemma.modeling_gemma import GemmaForCausalLM
+    from transformers.models.gemma.modeling_gemma import GemmaModel
     from transformers.models.gemma2.modeling_gemma2 import Gemma2ForCausalLM
+    from transformers.models.gemma2.modeling_gemma2 import Gemma2Model
     from transformers.models.paligemma import modeling_paligemma
     from transformers.models.paligemma.modeling_paligemma import PaliGemmaForConditionalGeneration
     from transformers.models.siglip import modeling_siglip
@@ -1155,7 +1160,7 @@ def apply_liger_kernel_to_paligemma(
 
         language_model = model.language_model
 
-        if isinstance(language_model, GemmaForCausalLM):
+        if isinstance(language_model, (GemmaForCausalLM, GemmaModel)):
             apply_liger_kernel_to_gemma(
                 rope=rope,
                 cross_entropy=False,
@@ -1165,7 +1170,7 @@ def apply_liger_kernel_to_paligemma(
                 model=language_model,
             )
 
-        elif isinstance(language_model, Gemma2ForCausalLM):
+        elif isinstance(language_model, (Gemma2ForCausalLM, Gemma2Model)):
             apply_liger_kernel_to_gemma2(
                 rope=rope,
                 cross_entropy=False,

{liger_kernel_nightly-0.6.0.dev20250708194445 → liger_kernel_nightly-0.6.0.dev20250709042125}/src/liger_kernel_nightly.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: liger_kernel_nightly
-Version: 0.6.0.dev20250708194445
+Version: 0.6.0.dev20250709042125
 Summary: Efficient Triton kernels for LLM Training
 License: BSD 2-CLAUSE LICENSE
         Copyright 2024 LinkedIn Corporation 
@@ -33,7 +33,7 @@ License-File: NOTICE
 Requires-Dist: torch>=2.1.2
 Requires-Dist: triton>=2.3.1
 Provides-Extra: dev
-Requires-Dist: transformers>=4.44.2; extra == "dev"
+Requires-Dist: transformers>=4.49.0; extra == "dev"
 Requires-Dist: matplotlib>=3.7.2; extra == "dev"
 Requires-Dist: flake8>=4.0.1.1; extra == "dev"
 Requires-Dist: black>=24.4.2; extra == "dev"

{liger_kernel_nightly-0.6.0.dev20250708194445 → liger_kernel_nightly-0.6.0.dev20250709042125}/src/liger_kernel_nightly.egg-info/requires.txt

@@ -2,7 +2,7 @@ torch>=2.1.2
 triton>=2.3.1
 
 [dev]
-transformers>=4.44.2
+transformers>=4.49.0
 matplotlib>=3.7.2
 flake8>=4.0.1.1
 black>=24.4.2

{liger_kernel_nightly-0.6.0.dev20250708194445 → liger_kernel_nightly-0.6.0.dev20250709042125}/test/convergence/bf16/test_mini_models.py

@@ -957,8 +957,8 @@ def run_mini_model(
                     reason="LLaVa not available in this version of transformers",
                 ),
                 pytest.mark.skipif(
-                    version.parse(transformers.__version__) < version.parse("4.49.0"),
-                    reason="Mistral not available in transformers<=4.49.0",
+                    version.parse(transformers.__version__) < version.parse("4.52.0"),
+                    reason="LLaVa doesn't materialize logits in transformers<=4.52.0 so we can't test it",
                 ),
             ],
         ),

{liger_kernel_nightly-0.6.0.dev20250708194445 → liger_kernel_nightly-0.6.0.dev20250709042125}/test/convergence/fp32/test_mini_models.py

@@ -938,8 +938,8 @@ def run_mini_model(
                     reason="LLaVa not available in this version of transformers",
                 ),
                 pytest.mark.skipif(
-                    version.parse(transformers.__version__) < version.parse("4.49.0"),
-                    reason="Mistral not available in transformers<=4.49.0",
+                    version.parse(transformers.__version__) < version.parse("4.52.0"),
+                    reason="LLaVa doesn't materialize logits in transformers<=4.52.0 so we can't test it",
                 ),
             ],
         ),
@@ -1103,7 +1103,7 @@ def run_mini_model(
             torch.float32,
             1e-8,
             1e-4,
-            5e-3,  # 4e-3
+            4e-2,  # 4e-3
             1e-5,  # 1e-5
             5e-3,
             1e-5,

{liger_kernel_nightly-0.6.0.dev20250708194445 → liger_kernel_nightly-0.6.0.dev20250709042125}/test/transformers/test_monkey_patch.py

@@ -38,6 +38,7 @@ if transformer_version >= version.parse(SUPPORTED_TRANSFORMER_VERSION):
     from liger_kernel.transformers.model.mistral import lce_forward as mistral_lce_forward
     from liger_kernel.transformers.model.mixtral import lce_forward as mixtral_lce_forward
     from liger_kernel.transformers.model.mllama import lce_forward as mllama_lce_forward
+    from liger_kernel.transformers.model.paligemma import lce_forward as paligemma_lce_forward
     from liger_kernel.transformers.model.phi3 import lce_forward as phi3_lce_forward
     from liger_kernel.transformers.model.qwen2 import lce_forward as qwen2_lce_forward
 else:
@@ -49,6 +50,7 @@ else:
     )
     from liger_kernel.transformers.model.mixtral import lce_forward_deprecated as mixtral_lce_forward
     from liger_kernel.transformers.model.mllama import lce_forward_deprecated as mllama_lce_forward
+    from liger_kernel.transformers.model.paligemma import lce_forward_deprecated as paligemma_lce_forward
     from liger_kernel.transformers.model.phi3 import lce_forward_deprecated as phi3_lce_forward
     from liger_kernel.transformers.model.qwen2 import lce_forward_deprecated as qwen2_lce_forward
 
@@ -72,24 +74,6 @@ def is_llama4_available():
         return False
 
 
-def is_qwen2_vl_available():
-    try:
-        import transformers.models.qwen2_vl  # noqa: F401
-
-        return True
-    except ImportError:
-        return False
-
-
-def is_qwen2_5_vl_available():
-    try:
-        import transformers.models.qwen2_5_vl  # noqa: F401
-
-        return True
-    except ImportError:
-        return False
-
-
 def is_qwen3_available():
     try:
         import transformers.models.qwen3  # noqa: F401
@@ -126,6 +110,15 @@ def is_gemma3_available():
         return False
 
 
+def is_paligemma_available():
+    try:
+        import transformers.models.paligemma  # noqa: F401
+
+        return True
+    except ImportError:
+        return False
+
+
 def test_import_from_root():
     try:
         from liger_kernel.transformers import AutoLigerKernelForCausalLM  # noqa: F401
@@ -354,6 +347,7 @@ def test_apply_liger_kernel_to_instance_for_mllama_for_conditional_generation():
     # Ensure any monkey patching is cleaned up for subsequent tests
     with patch("transformers.models.mllama.modeling_mllama"):
         from transformers.models.mllama.modeling_mllama import MllamaForConditionalGeneration
+        from transformers.models.mllama.modeling_mllama import MllamaTextModel
 
         # Instantiate a dummy model
         config = transformers.models.mllama.configuration_mllama.MllamaConfig(
@@ -387,10 +381,14 @@ def test_apply_liger_kernel_to_instance_for_mllama_for_conditional_generation():
 
         # Check that model instance variables are not yet patched with Liger modules
         assert inspect.getsource(dummy_model_instance.forward) != inspect.getsource(mllama_lce_forward)
-        assert inspect.getsource(dummy_model_instance.language_model.norm.forward) != inspect.getsource(
-            LigerRMSNorm.forward
-        )
-        for layer in dummy_model_instance.language_model.layers:
+
+        if isinstance(dummy_model_instance.language_model, MllamaTextModel):
+            language_model = dummy_model_instance.language_model
+        else:
+            language_model = dummy_model_instance.language_model.model
+
+        assert inspect.getsource(language_model.norm.forward) != inspect.getsource(LigerRMSNorm.forward)
+        for layer in language_model.layers:
             assert inspect.getsource(layer.mlp.forward) != inspect.getsource(LigerSwiGLUMLP.forward)
             assert inspect.getsource(layer.input_layernorm.forward) != inspect.getsource(LigerRMSNorm.forward)
             assert inspect.getsource(layer.post_attention_layernorm.forward) != inspect.getsource(LigerRMSNorm.forward)
@@ -417,10 +415,8 @@ def test_apply_liger_kernel_to_instance_for_mllama_for_conditional_generation():
 
         # Check that the model's instance variables were correctly patched with Liger modules
         assert inspect.getsource(dummy_model_instance.forward) == inspect.getsource(mllama_lce_forward)
-        assert inspect.getsource(dummy_model_instance.language_model.norm.forward) == inspect.getsource(
-            LigerRMSNorm.forward
-        )
-        for layer in dummy_model_instance.language_model.layers:
+        assert inspect.getsource(language_model.norm.forward) == inspect.getsource(LigerRMSNorm.forward)
+        for layer in language_model.layers:
             assert inspect.getsource(layer.mlp.forward) == inspect.getsource(LigerSwiGLUMLP.forward)
             assert inspect.getsource(layer.input_layernorm.forward) == inspect.getsource(LigerRMSNorm.forward)
             assert inspect.getsource(layer.post_attention_layernorm.forward) == inspect.getsource(LigerRMSNorm.forward)
@@ -441,7 +437,6 @@ def test_apply_liger_kernel_to_instance_for_mllama_for_conditional_generation():
             assert inspect.getsource(layer.post_attention_layernorm.forward) == inspect.getsource(
                 LigerLayerNorm.forward
             )
-
         try:
             print(dummy_model_instance)
         except Exception as e:
@@ -793,6 +788,62 @@ def test_apply_liger_kernel_to_instance_for_gemma2():
             pytest.fail(f"An exception occured in extra_expr: {type(e).__name__} - {e}")
 
 
+@pytest.mark.skipif(not is_paligemma_available(), reason="paligemma module not available")
+def test_apply_liger_kernel_to_instance_for_paligemma():
+    # Ensure any monkey patching is cleaned up for subsequent tests
+    with patch("transformers.models.paligemma.modeling_paligemma"):
+        from transformers.models.paligemma.modeling_paligemma import PaliGemmaForConditionalGeneration
+
+        # Instantiate a dummy model
+        config = transformers.models.paligemma.configuration_paligemma.PaliGemmaConfig(
+            torch_dtype=torch.bfloat16,
+            text_config={
+                "num_hidden_layers": 2,
+                "rms_norm_eps": 1e-5,
+                "hidden_size": 32,
+                "intermediate_size": 64,
+                "hidden_act": "silu",
+            },
+            vision_config={
+                "num_hidden_layers": 2,
+                "layer_norm_eps": 1e-5,
+                "hidden_size": 48,
+                "intermediate_size": 64,
+            },
+        )
+
+        dummy_model_instance = PaliGemmaForConditionalGeneration(config)
+        assert isinstance(dummy_model_instance, PaliGemmaForConditionalGeneration)
+
+        # Check that model instance variables are not yet patched with Liger modules
+        assert inspect.getsource(dummy_model_instance.forward) != inspect.getsource(paligemma_lce_forward)
+        assert inspect.getsource(
+            dummy_model_instance.vision_tower.vision_model.post_layernorm.forward
+        ) != inspect.getsource(LigerLayerNorm.forward)
+
+        for layer in dummy_model_instance.vision_tower.vision_model.encoder.layers:
+            assert inspect.getsource(layer.layer_norm1.forward) != inspect.getsource(LigerLayerNorm.forward)
+            assert inspect.getsource(layer.layer_norm2.forward) != inspect.getsource(LigerLayerNorm.forward)
+
+        # Test applying kernels to the model instance
+        _apply_liger_kernel_to_instance(model=dummy_model_instance)
+
+        # Check that the model's instance variables were correctly patched with Liger modules
+        assert inspect.getsource(dummy_model_instance.forward) == inspect.getsource(paligemma_lce_forward)
+        assert inspect.getsource(
+            dummy_model_instance.vision_tower.vision_model.post_layernorm.forward
+        ) == inspect.getsource(LigerLayerNorm.forward)
+
+        for layer in dummy_model_instance.vision_tower.vision_model.encoder.layers:
+            assert inspect.getsource(layer.layer_norm1.forward) == inspect.getsource(LigerLayerNorm.forward)
+            assert inspect.getsource(layer.layer_norm2.forward) == inspect.getsource(LigerLayerNorm.forward)
+
+        try:
+            print(dummy_model_instance)
+        except Exception as e:
+            pytest.fail(f"An exception occured in extra_expr: {type(e).__name__} - {e}")
+
+
 @pytest.mark.skipif(not is_gemma3_available(), reason="gemma3 module not available")
 def test_apply_liger_kernel_to_instance_for_gemma3_text():
     # Ensure any monkey patching is cleaned up for subsequent tests
@@ -1063,7 +1114,10 @@ def test_apply_liger_kernel_to_instance_for_qwen3_moe():
             pytest.fail(f"An exception occured in extra_expr: {type(e).__name__} - {e}")
 
 
-@pytest.mark.skipif(not is_qwen2_vl_available(), reason="qwen2_vl module not available")
+@pytest.mark.skipif(
+    transformer_version < version.parse("4.52.4"),
+    reason="Qwen2-VL support is only compatible with transformers >= 4.52.4",
+)
 def test_apply_liger_kernel_to_instance_for_qwen2_vl_for_conditional_generation():
     # Ensure any monkey patching is cleaned up for subsequent tests
     with patch("transformers.models.qwen2_vl.modeling_qwen2_vl"):
@@ -1129,7 +1183,10 @@ def test_apply_liger_kernel_to_instance_for_qwen2_vl_for_conditional_generation(
             pytest.fail(f"An exception occured in extra_expr: {type(e).__name__} - {e}")
 
 
-@pytest.mark.skipif(not is_qwen2_vl_available(), reason="qwen2_vl module not available")
+@pytest.mark.skipif(
+    transformer_version < version.parse("4.52.4"),
+    reason="Qwen2-VL support is only compatible with transformers >= 4.52.4",
+)
 def test_apply_liger_kernel_to_instance_for_qwen2_vl():
     # Ensure any monkey patching is cleaned up for subsequent tests
     with patch("transformers.models.qwen2_vl.modeling_qwen2_vl"):
@@ -1195,7 +1252,10 @@ def test_apply_liger_kernel_to_instance_for_qwen2_vl():
             pytest.fail(f"An exception occured in extra_expr: {type(e).__name__} - {e}")
 
 
-@pytest.mark.skipif(not is_qwen2_vl_available(), reason="qwen2_vl module not available")
+@pytest.mark.skipif(
+    transformer_version < version.parse("4.52.4"),
+    reason="Qwen2-VL support is only compatible with transformers >= 4.52.4",
+)
 def test_apply_liger_kernel_to_instance_for_qwen2_vl_text():
     # Ensure any monkey patching is cleaned up for subsequent tests
     with patch("transformers.models.qwen2_vl.modeling_qwen2_vl"):
@@ -1243,7 +1303,10 @@ def test_apply_liger_kernel_to_instance_for_qwen2_vl_text():
             pytest.fail(f"An exception occured in extra_expr: {type(e).__name__} - {e}")
 
 
-@pytest.mark.skipif(not is_qwen2_5_vl_available(), reason="qwen2_5_vl module not available")
+@pytest.mark.skipif(
+    transformer_version < version.parse("4.52.4"),
+    reason="Qwen2.5-VL support is only compatible with transformers >= 4.52.4",
+)
 def test_apply_liger_kernel_to_instance_for_qwen2_5_vl():
     # Ensure any monkey patching is cleaned up for subsequent tests
     with patch("transformers.models.qwen2_5_vl.modeling_qwen2_5_vl"):
@@ -1309,7 +1372,10 @@ def test_apply_liger_kernel_to_instance_for_qwen2_5_vl():
             pytest.fail(f"An exception occured in extra_expr: {type(e).__name__} - {e}")
 
 
-@pytest.mark.skipif(not is_qwen2_5_vl_available(), reason="qwen2_5_vl module not available")
+@pytest.mark.skipif(
+    transformer_version < version.parse("4.52.4"),
+    reason="Qwen2.5-VL support is only compatible with transformers >= 4.52.4",
+)
 def test_apply_liger_kernel_to_instance_for_qwen2_5_vl_for_conditional_generation():
     # Ensure any monkey patching is cleaned up for subsequent tests
     with patch("transformers.models.qwen2_5_vl.modeling_qwen2_5_vl"):
@@ -1375,7 +1441,10 @@ def test_apply_liger_kernel_to_instance_for_qwen2_5_vl_for_conditional_generatio
             pytest.fail(f"An exception occured in extra_expr: {type(e).__name__} - {e}")
 
 
-@pytest.mark.skipif(not is_qwen2_5_vl_available(), reason="qwen2_5_vl module not available")
+@pytest.mark.skipif(
+    transformer_version < version.parse("4.52.4"),
+    reason="Qwen2.5-VL support is only compatible with transformers >= 4.52.4",
+)
 def test_apply_liger_kernel_to_instance_for_qwen2_5_vl_text():
     # Ensure any monkey patching is cleaned up for subsequent tests
     with patch("transformers.models.qwen2_5_vl.modeling_qwen2_5_vl"):