liger-kernel-nightly 0.5.8.dev20250422210723__tar.gz → 0.5.8.dev20250428050809__tar.gz

{liger_kernel_nightly-0.5.8.dev20250422210723 → liger_kernel_nightly-0.5.8.dev20250428050809}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: liger_kernel_nightly
-Version: 0.5.8.dev20250422210723
+Version: 0.5.8.dev20250428050809
 Summary: Efficient Triton kernels for LLM Training
 License: BSD 2-CLAUSE LICENSE
         Copyright 2024 LinkedIn Corporation 

{liger_kernel_nightly-0.5.8.dev20250422210723 → liger_kernel_nightly-0.5.8.dev20250428050809}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "liger_kernel_nightly"
-version = "0.5.8.dev20250422210723"
+version = "0.5.8.dev20250428050809"
 description = "Efficient Triton kernels for LLM Training"
 urls = { "Homepage" = "https://github.com/linkedin/Liger-Kernel" }
 readme = { file = "README.md", content-type = "text/markdown" }

{liger_kernel_nightly-0.5.8.dev20250422210723 → liger_kernel_nightly-0.5.8.dev20250428050809}/src/liger_kernel/transformers/model/gemma.py

@@ -201,14 +201,16 @@ def lce_forward(
 
     hidden_states = outputs[0]
 
+    shift_labels = loss_kwargs.pop("shift_labels", None)
     logits = None
     loss = None
     # if in training mode, don't materialize logits
-    if self.training and (labels is not None):
+    if self.training and (labels is not None or shift_labels is not None):
         loss = LigerForCausalLMLoss(
             hidden_states=hidden_states,
             lm_head_weight=self.lm_head.weight,
             labels=labels,
+            shift_labels=shift_labels,
             hidden_size=self.config.hidden_size,
             **loss_kwargs,
         )

{liger_kernel_nightly-0.5.8.dev20250422210723 → liger_kernel_nightly-0.5.8.dev20250428050809}/src/liger_kernel/transformers/model/gemma2.py

@@ -213,14 +213,16 @@ def lce_forward(
 
     hidden_states = outputs[0]
 
+    shift_labels = loss_kwargs.pop("shift_labels", None)
     logits = None
     loss = None
     # if in training mode, don't materialize logits
-    if self.training and (labels is not None):
+    if self.training and (labels is not None or shift_labels is not None):
         loss = LigerForCausalLMLoss(
             hidden_states=hidden_states,
             lm_head_weight=self.lm_head.weight,
             labels=labels,
+            shift_labels=shift_labels,
             hidden_size=self.config.hidden_size,
             final_logit_softcapping=self.config.final_logit_softcapping,
             **loss_kwargs,

{liger_kernel_nightly-0.5.8.dev20250422210723 → liger_kernel_nightly-0.5.8.dev20250428050809}/src/liger_kernel/transformers/model/gemma3.py

@@ -104,13 +104,15 @@ def causal_forward(
     # Only compute necessary logits, and do not upcast them to float if we are not computing the loss
     slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
     kept_hidden_states = hidden_states[:, slice_indices, :]
+    shift_labels = loss_kwargs.pop("shift_labels", None)
     loss = None
     logits = None
-    if self.training and (labels is not None):
+    if self.training and (labels is not None or shift_labels is not None):
         loss = LigerForCausalLMLoss(
             hidden_states=kept_hidden_states,
             lm_head_weight=self.lm_head.weight,
             labels=labels,
+            shift_labels=shift_labels,
             hidden_size=self.config.hidden_size,
             final_logit_softcapping=self.config.final_logit_softcapping,
             **loss_kwargs,

{liger_kernel_nightly-0.5.8.dev20250422210723 → liger_kernel_nightly-0.5.8.dev20250428050809}/src/liger_kernel/transformers/model/llama.py

@@ -213,14 +213,16 @@ def lce_forward(
     if self.config.pretraining_tp > 1:
         raise Exception("Liger Kernel does not support pretraining_tp!!")
 
+    shift_labels = loss_kwargs.pop("shift_labels", None)
     logits = None
     loss = None
     # if in training mode, don't materialize logits
-    if self.training and (labels is not None):
+    if self.training and (labels is not None or shift_labels is not None):
         loss = LigerForCausalLMLoss(
             hidden_states=hidden_states,
             lm_head_weight=self.lm_head.weight,
             labels=labels,
+            shift_labels=shift_labels,
             hidden_size=self.config.hidden_size,
             **loss_kwargs,
         )

{liger_kernel_nightly-0.5.8.dev20250422210723 → liger_kernel_nightly-0.5.8.dev20250428050809}/src/liger_kernel/transformers/model/mistral.py

@@ -92,14 +92,16 @@ def lce_forward(
 
     hidden_states = outputs[0]
 
+    shift_labels = loss_kwargs.pop("shift_labels", None)
     loss = None
     logits = None
 
-    if self.training and (labels is not None):
+    if self.training and (labels is not None or shift_labels is not None):
         loss = LigerForCausalLMLoss(
             hidden_states=hidden_states,
             lm_head_weight=self.lm_head.weight,
             labels=labels,
+            shift_labels=shift_labels,
             hidden_size=self.config.hidden_size,
             **loss_kwargs,
         )

{liger_kernel_nightly-0.5.8.dev20250422210723 → liger_kernel_nightly-0.5.8.dev20250428050809}/src/liger_kernel/transformers/model/mixtral.py

@@ -226,14 +226,16 @@ def lce_forward(
 
     hidden_states = outputs[0]
 
+    shift_labels = loss_kwargs.pop("shift_labels", None)
     logits = None
     loss = None
     # if in training mode, don't materialize logits
-    if self.training and (labels is not None):
+    if self.training and (labels is not None or shift_labels is not None):
         loss = LigerForCausalLMLoss(
             hidden_states=hidden_states,
             lm_head_weight=self.lm_head.weight,
             labels=labels,
+            shift_labels=shift_labels,
             hidden_size=self.config.hidden_size,
             **loss_kwargs,
         )

{liger_kernel_nightly-0.5.8.dev20250422210723 → liger_kernel_nightly-0.5.8.dev20250428050809}/src/liger_kernel/transformers/model/mllama.py

@@ -216,14 +216,16 @@ def lce_forward(
 
     hidden_states = outputs[0]
 
+    shift_labels = loss_kwargs.pop("shift_labels", None)
     logits = None
     loss = None
     # if in training mode, don't materialize logits
-    if self.training and (labels is not None):
+    if self.training and (labels is not None or shift_labels is not None):
         loss = LigerForCausalLMLoss(
             hidden_states=hidden_states,
             lm_head_weight=self.lm_head.weight,
             labels=labels,
+            shift_labels=shift_labels,
             hidden_size=self.config.hidden_size,
             **loss_kwargs,
         )

{liger_kernel_nightly-0.5.8.dev20250422210723 → liger_kernel_nightly-0.5.8.dev20250428050809}/src/liger_kernel/transformers/model/olmo2.py

@@ -89,14 +89,16 @@ def lce_forward(
 
     hidden_states = outputs[0]
 
+    shift_labels = loss_kwargs.pop("shift_labels", None)
     logits = None
     loss = None
     # if in training mode, don't materialize logits
-    if self.training and (labels is not None):
+    if self.training and (labels is not None or shift_labels is not None):
         loss = LigerForCausalLMLoss(
             hidden_states=hidden_states,
             lm_head_weight=self.lm_head.weight,
             labels=labels,
+            shift_labels=shift_labels,
             hidden_size=self.config.hidden_size,
             **loss_kwargs,
         )

{liger_kernel_nightly-0.5.8.dev20250422210723 → liger_kernel_nightly-0.5.8.dev20250428050809}/src/liger_kernel/transformers/model/phi3.py

@@ -214,14 +214,16 @@ def lce_forward(
 
     hidden_states = outputs[0]
 
+    shift_labels = loss_kwargs.pop("shift_labels", None)
     logits = None
     loss = None
     # if in training mode, don't materialize logits
-    if self.training and (labels is not None):
+    if self.training and (labels is not None or shift_labels is not None):
         loss = LigerForCausalLMLoss(
             hidden_states=hidden_states,
             lm_head_weight=self.lm_head.weight,
             labels=labels,
+            shift_labels=shift_labels,
             hidden_size=self.config.hidden_size,
             **loss_kwargs,
         )

{liger_kernel_nightly-0.5.8.dev20250422210723 → liger_kernel_nightly-0.5.8.dev20250428050809}/src/liger_kernel/transformers/model/qwen2.py

@@ -200,14 +200,16 @@ def lce_forward(
 
     hidden_states = outputs[0]
 
+    shift_labels = loss_kwargs.pop("shift_labels", None)
     logits = None
     loss = None
     # if in training mode, don't materialize logits
-    if self.training and (labels is not None):
+    if self.training and (labels is not None or shift_labels is not None):
         loss = LigerForCausalLMLoss(
             hidden_states=hidden_states,
             lm_head_weight=self.lm_head.weight,
             labels=labels,
+            shift_labels=shift_labels,
             hidden_size=self.config.hidden_size,
             **loss_kwargs,
         )

{liger_kernel_nightly-0.5.8.dev20250422210723 → liger_kernel_nightly-0.5.8.dev20250428050809}/src/liger_kernel/transformers/model/qwen2_5_vl.py

@@ -163,14 +163,16 @@ def lce_forward(
 
     hidden_states = outputs[0]
 
+    shift_labels = loss_kwargs.pop("shift_labels", None)
     loss = None
     logits = None
 
-    if self.training and (labels is not None):
+    if self.training and (labels is not None or shift_labels is not None):
         loss = LigerForCausalLMLoss(
             hidden_states=hidden_states,
             lm_head_weight=self.lm_head.weight,
             labels=labels,
+            shift_labels=shift_labels,
             hidden_size=self.config.hidden_size,
             **loss_kwargs,
         )

{liger_kernel_nightly-0.5.8.dev20250422210723 → liger_kernel_nightly-0.5.8.dev20250428050809}/src/liger_kernel/transformers/model/qwen2_vl.py

@@ -167,14 +167,16 @@ def lce_forward(
 
     hidden_states = outputs[0]
 
+    shift_labels = loss_kwargs.pop("shift_labels", None)
     loss = None
     logits = None
 
-    if self.training and (labels is not None):
+    if self.training and (labels is not None or shift_labels is not None):
         loss = LigerForCausalLMLoss(
             hidden_states=hidden_states,
             lm_head_weight=self.lm_head.weight,
             labels=labels,
+            shift_labels=shift_labels,
             hidden_size=self.config.hidden_size,
             **loss_kwargs,
         )

{liger_kernel_nightly-0.5.8.dev20250422210723 → liger_kernel_nightly-0.5.8.dev20250428050809}/src/liger_kernel_nightly.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: liger_kernel_nightly
-Version: 0.5.8.dev20250422210723
+Version: 0.5.8.dev20250428050809
 Summary: Efficient Triton kernels for LLM Training
 License: BSD 2-CLAUSE LICENSE
         Copyright 2024 LinkedIn Corporation 

{liger_kernel_nightly-0.5.8.dev20250422210723 → liger_kernel_nightly-0.5.8.dev20250428050809}/test/convergence/bf16/test_mini_models_multimodal.py

@@ -770,7 +770,6 @@ def run_mini_model_multimodal(
                     not QWEN2_VL_AVAILABLE,
                     reason="Qwen2-VL not available in this version of transformers",
                 ),
-                pytest.mark.skipif(device == "xpu", reason="skip for XPU"),
             ],
         ),
         pytest.param(
@@ -809,7 +808,6 @@ def run_mini_model_multimodal(
                     not QWEN2_5_VL_AVAILABLE,
                     reason="Qwen2.5-VL not available in this version of transformers",
                 ),
-                pytest.mark.skipif(device == "xpu", reason="skip for XPU"),
             ],
         ),
         pytest.param(
@@ -876,7 +874,7 @@ def run_mini_model_multimodal(
             torch.bfloat16,
             1e-3,
             1e-2,
-            0.25,  # Increase the absolute tolerance for the logits of Gemma-3.
+            0.4,  # Increase the absolute tolerance for the logits of Gemma-3.
             1e-1,
             1e-2,
             1e-2,

{liger_kernel_nightly-0.5.8.dev20250422210723 → liger_kernel_nightly-0.5.8.dev20250428050809}/test/convergence/fp32/test_mini_models_multimodal.py

@@ -765,7 +765,6 @@ def run_mini_model_multimodal(
                     not QWEN2_VL_AVAILABLE,
                     reason="Qwen2-VL not available in this version of transformers",
                 ),
-                pytest.mark.skipif(device == "xpu", reason="skip for XPU"),
             ],
         ),
         pytest.param(
@@ -800,7 +799,6 @@ def run_mini_model_multimodal(
                     not QWEN2_5_VL_AVAILABLE,
                     reason="Qwen2.5-VL not available in this version of transformers",
                 ),
-                pytest.mark.skipif(device == "xpu", reason="skip for XPU"),
             ],
         ),
         pytest.param(

{liger_kernel_nightly-0.5.8.dev20250422210723 → liger_kernel_nightly-0.5.8.dev20250428050809}/test/transformers/test_fused_linear_cross_entropy.py

@@ -105,7 +105,7 @@ class LigerLMHeadCE(torch.nn.Module):
 @pytest.mark.parametrize(
     "B, T, H, V",
     [
-        pytest.param(8, 128, 1024, 4096, marks=pytest.mark.skipif(device="xpu", reason="skip for XPU")),
+        (8, 128, 1024, 4096),
         (4, 47, 31, 123),  # random shape
     ],
 )
@@ -287,7 +287,7 @@ def test_correctness_functional(B, T, H, V, scalar, dtype, bias, ce_weight, atol
 @pytest.mark.parametrize(
     "B, T, H, V",
     [
-        pytest.param(8, 128, 1024, 4096, marks=pytest.mark.skipif(device="xpu", reason="skip for XPU")),
+        (8, 128, 1024, 4096),
         (4, 47, 31, 123),  # random shape
     ],
 )

{liger_kernel_nightly-0.5.8.dev20250422210723 → liger_kernel_nightly-0.5.8.dev20250428050809}/test/transformers/test_rms_norm.py

@@ -103,7 +103,7 @@ class GemmaRMSNorm(nn.Module):
     [
         (LlamaRMSNorm, 0.0, "llama"),
         (GemmaRMSNorm, 1.0, "gemma"),
-        pytest.param(BaseRMSNorm, 0.0, "none", marks=pytest.mark.skipif(device="xpu", reason="skip for XPU")),
+        (BaseRMSNorm, 0.0, "none"),
     ],
 )
 @pytest.mark.parametrize(