liger-kernel-nightly 0.6.2.dev20251005233549__py3-none-any.whl → 0.6.2.dev20251008084122__py3-none-any.whl

liger_kernel/transformers/model/gemma.py

@@ -228,10 +228,11 @@ def lce_forward(
         )
     else:
         logits = self.lm_head(kept_hidden_states)
-        if labels is not None:
+        if labels is not None or shift_labels is not None:
             loss = self.loss_function(
                 logits=logits,
                 labels=labels,
+                shift_labels=shift_labels,
                 vocab_size=self.config.vocab_size,
                 **kwargs,
             )

liger_kernel/transformers/model/gemma2.py

@@ -252,8 +252,14 @@ def lce_forward(
             logits = logits * self.config.final_logit_softcapping
 
         loss = None
-        if labels is not None:
-            loss = self.loss_function(logits, labels, self.vocab_size, **kwargs)
+        if labels is not None or shift_labels is not None:
+            loss = self.loss_function(
+                logits=logits,
+                labels=labels,
+                shift_labels=shift_labels,
+                vocab_size=self.vocab_size,
+                **kwargs,
+            )
 
     if not return_dict:
         output = (logits,) + outputs[1:]

liger_kernel/transformers/model/gemma3.py

@@ -119,8 +119,14 @@ def causal_forward(
             logits = logits / self.config.final_logit_softcapping
             logits = torch.tanh(logits)
             logits = logits * self.config.final_logit_softcapping
-        if labels is not None:
-            loss = self.loss_function(logits, labels, self.vocab_size, **loss_kwargs)
+        if labels is not None or shift_labels is not None:
+            loss = self.loss_function(
+                logits=logits,
+                labels=labels,
+                shift_labels=shift_labels,
+                vocab_size=self.vocab_size,
+                **loss_kwargs,
+            )
 
     if not return_dict:
         output = (logits,) + outputs[1:]
@@ -275,6 +281,25 @@ def multimodal_forward(
             # Flatten the tokens
             loss_fct = nn.CrossEntropyLoss()
 
+            flat_logits = shift_logits.view(-1, self.config.text_config.vocab_size)
+            flat_labels = shift_labels.view(-1).to(shift_logits.device)
+            loss = loss_fct(flat_logits, flat_labels)
+        elif shift_labels is not None:
+            # Upcast to float if we need to compute the loss to avoid potential precision issues
+            logits = logits.float()
+            shift_logits = logits[..., :-1, :]
+            if attention_mask is not None:
+                # we use the input attention mask to shift the logits and labels, because it is 2D.
+                # we also crop attn mask in case it is longer, which happens in PrefixTuning with peft
+                shift_attention_mask = attention_mask[:, -shift_logits.shape[1] :].to(logits.device)
+                shift_logits = shift_logits[shift_attention_mask.to(logits.device) != 0].contiguous()
+                shift_labels = shift_labels[shift_attention_mask.to(shift_labels.device) != 0].contiguous()
+            else:
+                shift_logits = shift_logits.contiguous()
+                shift_labels = shift_labels.contiguous()
+            # Flatten the tokens
+            loss_fct = nn.CrossEntropyLoss()
+
             flat_logits = shift_logits.view(-1, self.config.text_config.vocab_size)
             flat_labels = shift_labels.view(-1).to(shift_logits.device)
             loss = loss_fct(flat_logits, flat_labels)

liger_kernel/transformers/model/glm4.py

@@ -111,10 +111,11 @@ def lce_forward(
 
     else:
         logits = self.lm_head(kept_hidden_states)
-        if labels is not None:
+        if labels is not None or shift_labels is not None:
             loss = self.loss_function(
                 logits=logits,
                 labels=labels,
+                shift_labels=shift_labels,
                 vocab_size=self.config.vocab_size,
                 **kwargs,
             )

liger_kernel/transformers/model/glm4v.py

@@ -133,10 +133,11 @@ def lce_forward(
 
     else:
         logits = self.lm_head(kept_hidden_states)
-        if labels is not None:
+        if labels is not None or shift_labels is not None:
             loss = self.loss_function(
                 logits=logits,
                 labels=labels,
+                shift_labels=shift_labels,
                 vocab_size=self.config.vocab_size,
                 **kwargs,
             )

liger_kernel/transformers/model/glm4v_moe.py

@@ -134,10 +134,11 @@ def lce_forward(
 
     else:
         logits = self.lm_head(kept_hidden_states)
-        if labels is not None:
+        if labels is not None or shift_labels is not None:
             loss = self.loss_function(
                 logits=logits,
                 labels=labels,
+                shift_labels=shift_labels,
                 vocab_size=self.config.vocab_size,
                 **kwargs,
             )

liger_kernel/transformers/model/llama.py

@@ -248,10 +248,11 @@ def lce_forward(
 
     else:
         logits = self.lm_head(kept_hidden_states)
-        if labels is not None:
+        if labels is not None or shift_labels is not None:
             loss = self.loss_function(
                 logits=logits,
                 labels=labels,
+                shift_labels=shift_labels,
                 vocab_size=self.config.vocab_size,
                 **kwargs,
             )

liger_kernel/transformers/model/llama4.py

@@ -91,10 +91,11 @@ def lce_forward(
 
     else:  # if in inference mode materialize logits
         logits = self.lm_head(kept_hidden_states)
-        if labels is not None:
+        if labels is not None or shift_labels is not None:
             loss = self.loss_function(
                 logits=logits,
                 labels=labels,
+                shift_labels=shift_labels,
                 vocab_size=self.config.vocab_size,
                 **kwargs,
             )

liger_kernel/transformers/model/llava.py

@@ -313,9 +313,13 @@ def lce_forward(
 
     else:
         logits = self.lm_head(kept_hidden_states)
-        if labels is not None:
+        if labels is not None or shift_labels is not None:
             loss = self.loss_function(
-                logits=logits, labels=labels, vocab_size=self.config.text_config.vocab_size, **lm_kwargs
+                logits=logits,
+                labels=labels,
+                shift_labels=shift_labels,
+                vocab_size=self.config.text_config.vocab_size,
+                **lm_kwargs,
             )
 
     if not return_dict:

liger_kernel/transformers/model/mistral.py

@@ -115,10 +115,11 @@ def lce_forward(
         logits = self.lm_head(kept_hidden_states)
 
         loss = None
-        if labels is not None:
+        if labels is not None or shift_labels is not None:
             loss = self.loss_function(
                 logits=logits,
                 labels=labels,
+                shift_labels=shift_labels,
                 vocab_size=self.config.vocab_size,
                 **kwargs,
             )

liger_kernel/transformers/model/mixtral.py

@@ -248,8 +248,14 @@ def lce_forward(
         logits = self.lm_head(kept_hidden_states)
 
         loss = None
-        if labels is not None:
-            loss = self.loss_function(logits, labels, self.vocab_size, **kwargs)
+        if labels is not None or shift_labels is not None:
+            loss = self.loss_function(
+                logits=logits,
+                labels=labels,
+                shift_labels=shift_labels,
+                vocab_size=self.vocab_size,
+                **kwargs,
+            )
     aux_loss = None
     if output_router_logits:
         aux_loss = load_balancing_loss_func(

liger_kernel/transformers/model/mllama.py

@@ -239,10 +239,11 @@ def lce_forward(
 
     else:
         logits = self.lm_head(kept_hidden_states)
-        if labels is not None:
+        if labels is not None or shift_labels is not None:
             loss = self.loss_function(
                 logits=logits,
                 labels=labels,
+                shift_labels=shift_labels,
                 vocab_size=self.config.vocab_size,
                 **kwargs,
             )

liger_kernel/transformers/model/olmo2.py

@@ -111,10 +111,11 @@ def lce_forward(
 
     else:
         logits = self.lm_head(kept_hidden_states)
-        if labels is not None:
+        if labels is not None or shift_labels is not None:
             loss = self.loss_function(
                 logits=logits,
                 labels=labels,
+                shift_labels=shift_labels,
                 vocab_size=self.config.vocab_size,
                 **kwargs,
             )

liger_kernel/transformers/model/paligemma.py

@@ -379,6 +379,25 @@ def lce_forward(
             # Flatten the tokens
             loss_fct = CrossEntropyLoss()
 
+            flat_logits = shift_logits.view(-1, self.config.text_config.vocab_size)
+            flat_labels = shift_labels.view(-1).to(shift_logits.device)
+            loss = loss_fct(flat_logits, flat_labels)
+        elif shift_labels is not None:
+            # Upcast to float if we need to compute the loss to avoid potential precision issues
+            logits = logits.float()
+            shift_logits = logits[..., :-1, :]
+            if attention_mask is not None:
+                # we use the input attention mask to shift the logits and labels, because it is 2D.
+                # we also crop attn mask in case it is longer, which happens in PrefixTuning with peft
+                shift_attention_mask = attention_mask[:, -shift_logits.shape[1] :].to(logits.device)
+                shift_logits = shift_logits[shift_attention_mask.to(logits.device) != 0].contiguous()
+                shift_labels = shift_labels[shift_attention_mask.to(shift_labels.device) != 0].contiguous()
+            else:
+                shift_logits = shift_logits.contiguous()
+                shift_labels = shift_labels.contiguous()
+            # Flatten the tokens
+            loss_fct = CrossEntropyLoss()
+
             flat_logits = shift_logits.view(-1, self.config.text_config.vocab_size)
             flat_labels = shift_labels.view(-1).to(shift_logits.device)
             loss = loss_fct(flat_logits, flat_labels)

liger_kernel/transformers/model/phi3.py

@@ -91,10 +91,11 @@ def lce_forward(
 
     else:
         logits = self.lm_head(kept_hidden_states)
-        if labels is not None:
+        if labels is not None or shift_labels is not None:
             loss = self.loss_function(
                 logits=logits,
                 labels=labels,
+                shift_labels=shift_labels,
                 vocab_size=self.config.vocab_size,
                 **kwargs,
             )

liger_kernel/transformers/model/qwen2.py

@@ -228,10 +228,11 @@ def lce_forward(
 
     else:
         logits = self.lm_head(kept_hidden_states)
-        if labels is not None:
+        if labels is not None or shift_labels is not None:
             loss = self.loss_function(
                 logits=logits,
                 labels=labels,
+                shift_labels=shift_labels,
                 vocab_size=self.config.vocab_size,
                 **kwargs,
             )

liger_kernel/transformers/model/qwen2_5_vl.py

@@ -133,8 +133,13 @@ def lce_forward(
         logits = self.lm_head(hidden_states)
 
         loss = None
-        if labels is not None:
-            loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size)
+        if labels is not None or shift_labels is not None:
+            loss = self.loss_function(
+                logits=logits,
+                labels=labels,
+                shift_labels=shift_labels,
+                vocab_size=self.config.vocab_size,
+            )
 
     if not return_dict:
         output = (logits,) + outputs[1:]

liger_kernel/transformers/model/qwen2_vl.py

@@ -129,8 +129,13 @@ def lce_forward(
         logits = self.lm_head(hidden_states)
 
         loss = None
-        if labels is not None:
-            loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size)
+        if labels is not None or shift_labels is not None:
+            loss = self.loss_function(
+                logits=logits,
+                labels=labels,
+                shift_labels=shift_labels,
+                vocab_size=self.config.vocab_size,
+            )
 
     return Qwen2VLCausalLMOutputWithPast(
         loss=loss,

liger_kernel/transformers/model/qwen3.py

@@ -103,10 +103,11 @@ def lce_forward(
 
     else:
         logits = self.lm_head(kept_hidden_states)
-        if labels is not None:
+        if labels is not None or shift_labels is not None:
             loss = self.loss_function(
                 logits=logits,
                 labels=labels,
+                shift_labels=shift_labels,
                 vocab_size=self.config.vocab_size,
                 **kwargs,
             )

liger_kernel/transformers/model/qwen3_moe.py

@@ -107,8 +107,14 @@ def lce_forward(
         )
     else:  # if in inference model materialize logits
         logits = self.lm_head(kept_hidden_states)
-        if labels is not None:
-            loss = self.loss_function(logits, labels, self.vocab_size, **kwargs)
+        if labels is not None or shift_labels is not None:
+            loss = self.loss_function(
+                logits=logits,
+                labels=labels,
+                shift_labels=shift_labels,
+                vocab_size=self.vocab_size,
+                **kwargs,
+            )
 
     aux_loss = None
     if output_router_logits:

liger_kernel/transformers/model/smollm3.py

@@ -121,10 +121,11 @@ def lce_forward(
 
     else:
         logits = self.lm_head(kept_hidden_states)
-        if labels is not None:
+        if labels is not None or shift_labels is not None:
             loss = self.loss_function(
                 logits=logits,
                 labels=labels,
+                shift_labels=shift_labels,
                 vocab_size=self.config.vocab_size,
                 **kwargs,
             )

{liger_kernel_nightly-0.6.2.dev20251005233549.dist-info → liger_kernel_nightly-0.6.2.dev20251008084122.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: liger_kernel_nightly
-Version: 0.6.2.dev20251005233549
+Version: 0.6.2.dev20251008084122
 Summary: Efficient Triton kernels for LLM Training
 License: BSD 2-CLAUSE LICENSE
         Copyright 2024 LinkedIn Corporation 

{liger_kernel_nightly-0.6.2.dev20251005233549.dist-info → liger_kernel_nightly-0.6.2.dev20251008084122.dist-info}/RECORD

@@ -71,36 +71,36 @@ liger_kernel/transformers/tvd.py,sha256=XrRfyJIqN6HFxXk8MYyFVZM1OLz3mtSbRZvWfZ_J
 liger_kernel/transformers/experimental/__init__.py,sha256=oQqk-f32JYgWEP9DJCj6ty6bbJSGrdXsFDQFwGeX6vI,127
 liger_kernel/transformers/experimental/embedding.py,sha256=2P0QYdlFyFrG5OqTzTa1wcRgDSyjBMv5i1a7BrDPDQw,881
 liger_kernel/transformers/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-liger_kernel/transformers/model/gemma.py,sha256=mNX-mIwV6jI4zfbrUHp0C468pOmjzsL7mjXipGt-eS0,10007
-liger_kernel/transformers/model/gemma2.py,sha256=R_JFPyWTk7RyA7D05ZiIaNO5pX8gWcvfWf-6rdCRMxs,11296
-liger_kernel/transformers/model/gemma3.py,sha256=FKO4j3t4W_5uECRA1lhVnXC-It2GhirHm4tpCf9ApAc,12785
-liger_kernel/transformers/model/glm4.py,sha256=GlnEhdGJuDIqp2R9qC54biY3HwV1tWmfpJm6ijoAsrM,5257
-liger_kernel/transformers/model/glm4v.py,sha256=dE9rRx1bOIr4T9xSXj58dtukaR80szeuclbgX0A0ovg,5951
-liger_kernel/transformers/model/glm4v_moe.py,sha256=Nij7tIYGkcBhJUMvQhYoFd1tVT4pbkxc2k4MSvN2pak,6166
+liger_kernel/transformers/model/gemma.py,sha256=WryzpVmCm2H_XgLKNu3jJ6gVawjQDjapTetg4WHlbR4,10078
+liger_kernel/transformers/model/gemma2.py,sha256=eOQEfJBKezJNNrirhkPSagGxr9qj_y4lENOZgjUZKpE,11471
+liger_kernel/transformers/model/gemma3.py,sha256=-tvZw88S-STqmvdim-xrZZRJ17KLWoge_73ilIvhpIU,14157
+liger_kernel/transformers/model/glm4.py,sha256=2TBM5-4URpj6uX96G1AZ_DrjAmQtgLwXGzBvaXtfwdk,5328
+liger_kernel/transformers/model/glm4v.py,sha256=nlgEMOBjFEOu7a-cwwp9mWhTFqIs3QrOvcxW-uaPq-s,6022
+liger_kernel/transformers/model/glm4v_moe.py,sha256=q3-R_FoQPayS85AriJWWebblXB6Ix9fvxhSrI3mHiz4,6237
 liger_kernel/transformers/model/internvl.py,sha256=Uv8KGXOz9NhiKVZDeRNzAJH5kRuMZikUbswWM9u5KM0,6069
-liger_kernel/transformers/model/llama.py,sha256=i8jJgyZsMKWQ-zKloETLugtwFpUOdaWxLDceciFXKd4,12832
-liger_kernel/transformers/model/llama4.py,sha256=IgbB8sTh3dlETQnaNNy1bZLuXy-Nt7qmeAjF27ydGpg,4210
-liger_kernel/transformers/model/llava.py,sha256=bLCioday_SOm69ogMDBhy_4UsVkH2-BSl93-EXY6-7I,15076
+liger_kernel/transformers/model/llama.py,sha256=L_VuaxxFJpzEmpLnaqwBbI5-Q14Qgfj-ufhLydCWgdk,12903
+liger_kernel/transformers/model/llama4.py,sha256=epEO_VD1gJCDovabSIQLxxncoh-TQTBfj-UgIlR5c7U,4281
+liger_kernel/transformers/model/llava.py,sha256=t6kMiyBkteVam-ltiod2f1mevj8l8ZHxYDvfu9C_lEk,15196
 liger_kernel/transformers/model/loss_utils.py,sha256=02RVkPI7Qs4ZP4yU_udCAvD_2hgIaHmxremRKe3N7EE,1885
-liger_kernel/transformers/model/mistral.py,sha256=syYNL8dLThX2-4uC13Lu0krEZ5zw3InviDUR3AJmc-I,5500
-liger_kernel/transformers/model/mixtral.py,sha256=VY-y73IyjcCyWyI7ahxXLw0fJrhgjYfr1xwRYtsHX0o,11396
-liger_kernel/transformers/model/mllama.py,sha256=NhJtlXiuszJHo5YSJOvSGYH47ly7Hse8r-5BKznBg9s,11522
-liger_kernel/transformers/model/olmo2.py,sha256=6L_bo-ZUgO1lYppdJneOtYxNIylQKS6BiGp13g7Uq9E,5259
-liger_kernel/transformers/model/paligemma.py,sha256=xuIx3oOwTgftU3jqLfWOxUxgCLBNJh0yNC21an9qDjo,18773
-liger_kernel/transformers/model/phi3.py,sha256=AwScxUe3LjmHHyQg4gW9bMoUI7uA6fUEMXJ3YhBiHtQ,4046
-liger_kernel/transformers/model/qwen2.py,sha256=3fpOTEOkniQmkCfN1KUa3KhseHJVzhj2Ht9FdYPUy-E,9962
-liger_kernel/transformers/model/qwen2_5_vl.py,sha256=zEVVwotCXnAm3RRc8-1Nc8uitSWrwW4B9dYY2uOZDwg,6331
-liger_kernel/transformers/model/qwen2_vl.py,sha256=5vK-vtCDpKZ2w33xYp2BS8kQYWUbKMqaiKvQcI27Mss,5884
-liger_kernel/transformers/model/qwen3.py,sha256=w2jBHuK9kK9EmOr5dnEIXNQXUgUSV_sJUkXSEwxLPHs,4885
-liger_kernel/transformers/model/qwen3_moe.py,sha256=BkpfFH3fOH0yRfA7LF-AoHTLut2GV0Y4MOlkiIYewfU,5511
-liger_kernel/transformers/model/smollm3.py,sha256=mqayvpwpMbp2yd_Ue7IPzy-dA4KHSDi_ROZW5vHCHfQ,7596
+liger_kernel/transformers/model/mistral.py,sha256=XmM4N21RIOkJ9PJ4PZ3DcRUhGUczn_lbx0plf1zeHb0,5571
+liger_kernel/transformers/model/mixtral.py,sha256=SLdLO81AZL7zror0LXLkn2PHqKzjwMMs4kALNqoaT00,11571
+liger_kernel/transformers/model/mllama.py,sha256=5q8q2BxQR_8hNZ83XrJIbndw-l6T7ZyFLM7OCv_uPK0,11593
+liger_kernel/transformers/model/olmo2.py,sha256=9O1Cze2B6ON-i1jgjQwjpS_WsDEK0PzL003s-MkevWA,5330
+liger_kernel/transformers/model/paligemma.py,sha256=mnTnSmEDla_bbVmPFmqhNVT__Cuf-TM-KLGFUa1sU-4,19967
+liger_kernel/transformers/model/phi3.py,sha256=L4gG8htOABmaxzcmHph0bBFCACRvL9r6wuDVFXi2o7Q,4117
+liger_kernel/transformers/model/qwen2.py,sha256=lgn0X6EzAZUhOv17ZDD9choIDdaPVIAsIrrdvwzWXqs,10033
+liger_kernel/transformers/model/qwen2_5_vl.py,sha256=Ea3zvL1FJfjlaerpeXCq-1zmorrajwNsR-XsgWr4fFQ,6465
+liger_kernel/transformers/model/qwen2_vl.py,sha256=ZeasFPGs-bxm2Y_E15mo0YNx5wwtKYDV-bjVKjkLPBk,6018
+liger_kernel/transformers/model/qwen3.py,sha256=Q2aOg5erPrgVgRcqJm8sefLSDtvU1AD5B7aJnP7mRMM,4956
+liger_kernel/transformers/model/qwen3_moe.py,sha256=1CwTMCNFDYsjGoa_aHFBagtC5HuJTV-s0__5UvcjD3A,5686
+liger_kernel/transformers/model/smollm3.py,sha256=0KWVkDtXbjsBKhJnaquV6vUUYyLtfmNwYH0sxJt-qTk,7667
 liger_kernel/transformers/trainer/__init__.py,sha256=p7yQfklV8-467qSz_ZMimkbDF7HHWHwku25A-GYL0WU,193
 liger_kernel/transformers/trainer/orpo_trainer.py,sha256=tX0h63aOFe3rNqTmk6JpMf75UPo981yzEa6TghnjS0Q,5370
 liger_kernel/triton/__init__.py,sha256=qCiCamzCRv6lpV8IqpAc9YMdNKC7GKurClWceQPnlis,92
 liger_kernel/triton/monkey_patch.py,sha256=Rd0hUHAzDkFfHvnX7-PBaNK5EKnZhtfM_h-fgQH9HPY,1568
-liger_kernel_nightly-0.6.2.dev20251005233549.dist-info/LICENSE,sha256=OhzLDHJ0to4a8sodVLELZiCFylZ1NAAYLs-HrjPy0ag,1312
-liger_kernel_nightly-0.6.2.dev20251005233549.dist-info/METADATA,sha256=a9HNSwKTdwQwsiF67WLIuJaD2Lt7R91GwAK1H9nRE-M,24777
-liger_kernel_nightly-0.6.2.dev20251005233549.dist-info/NOTICE,sha256=njwnoPZLh9AN8SJQzxvCGLHi-8X__AvWRze6joNXIY8,2066
-liger_kernel_nightly-0.6.2.dev20251005233549.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
-liger_kernel_nightly-0.6.2.dev20251005233549.dist-info/top_level.txt,sha256=2eghu4hA3LnkM7ElW92tQ8zegWKgSbeo-k-aGe1YnvY,13
-liger_kernel_nightly-0.6.2.dev20251005233549.dist-info/RECORD,,
+liger_kernel_nightly-0.6.2.dev20251008084122.dist-info/LICENSE,sha256=OhzLDHJ0to4a8sodVLELZiCFylZ1NAAYLs-HrjPy0ag,1312
+liger_kernel_nightly-0.6.2.dev20251008084122.dist-info/METADATA,sha256=Uk7nue5xFD5LoCoKeAD8zHpI2-c-taOdvk1KyBdBUXE,24777
+liger_kernel_nightly-0.6.2.dev20251008084122.dist-info/NOTICE,sha256=njwnoPZLh9AN8SJQzxvCGLHi-8X__AvWRze6joNXIY8,2066
+liger_kernel_nightly-0.6.2.dev20251008084122.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
+liger_kernel_nightly-0.6.2.dev20251008084122.dist-info/top_level.txt,sha256=2eghu4hA3LnkM7ElW92tQ8zegWKgSbeo-k-aGe1YnvY,13
+liger_kernel_nightly-0.6.2.dev20251008084122.dist-info/RECORD,,