liger-kernel-nightly 0.5.10.dev20250624183504__py3-none-any.whl → 0.6.3.dev20251121010306__py3-none-any.whl

liger_kernel/transformers/model/loss_utils.py

@@ -1,10 +1,28 @@
 from typing import Optional
+from typing import Tuple
 
 import torch
 import torch.nn as nn
 
 import liger_kernel.transformers.functional as F
 
+from liger_kernel.transformers.functional import CrossEntropyOutput
+
+
+def unpack_cross_entropy_result(
+    result,
+) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor]]:
+    if isinstance(result, CrossEntropyOutput):
+        return result.loss, result.z_loss, result.token_accuracy
+
+    if isinstance(result, tuple):
+        loss = result[0]
+        z_loss = result[1] if len(result) > 1 else None
+        token_accuracy = result[2] if len(result) > 2 else None
+        return loss, z_loss, token_accuracy
+
+    return result, None, None
+
 
 def fixed_fused_linear_cross_entropy(
     hidden_states: torch.Tensor,
@@ -13,20 +31,31 @@ def fixed_fused_linear_cross_entropy(
     num_items_in_batch: Optional[int] = None,
     ignore_index: int = -100,
     final_logit_softcapping: Optional[float] = None,
+    accum_dtype: Optional[torch.dtype] = None,
+    return_token_accuracy: bool = False,
     **kwargs,
 ):
     reduction = "sum" if num_items_in_batch is not None else "mean"
-    loss = F.liger_fused_linear_cross_entropy(
+    result = F.liger_fused_linear_cross_entropy(
         hidden_states,
         lm_head_weight,
         target,
         reduction=reduction,
         ignore_index=ignore_index,
         softcap=final_logit_softcapping,
+        accum_dtype=accum_dtype,
+        return_token_accuracy=return_token_accuracy,
+        **kwargs,
     )
+
+    loss, _, token_accuracy = unpack_cross_entropy_result(result)
+
     if reduction == "sum":
         loss = loss / num_items_in_batch
 
+    if return_token_accuracy:
+        return CrossEntropyOutput(loss=loss, token_accuracy=token_accuracy)
+
     return loss
 
 
@@ -39,6 +68,7 @@ def LigerForCausalLMLoss(
     ignore_index: int = -100,
     shift_labels: Optional[torch.Tensor] = None,
     final_logit_softcapping: Optional[float] = None,
+    return_token_accuracy: bool = False,
     **kwargs,
 ):
     # Skip upcast since intermediate values for the loss are all fp32 in kernel
@@ -52,13 +82,14 @@ def LigerForCausalLMLoss(
     shift_labels = shift_labels.view(-1)
     # Enable model parallelism
     shift_labels = shift_labels.to(hidden_states.device)
-    loss = fixed_fused_linear_cross_entropy(
+    result = fixed_fused_linear_cross_entropy(
         hidden_states,
         lm_head_weight,
         shift_labels,
         num_items_in_batch,
         ignore_index,
         final_logit_softcapping,
+        return_token_accuracy=return_token_accuracy,
         **kwargs,
     )
-    return loss
+    return result

liger_kernel/transformers/model/mistral.py

@@ -6,10 +6,11 @@ from typing import Union
 import torch
 
 from transformers.cache_utils import Cache
-from transformers.modeling_outputs import CausalLMOutputWithPast
 from transformers.utils.deprecation import deprecate_kwarg
 
 from liger_kernel.transformers.model.loss_utils import LigerForCausalLMLoss
+from liger_kernel.transformers.model.loss_utils import unpack_cross_entropy_result
+from liger_kernel.transformers.model.output_classes import LigerCausalLMOutputWithPast
 
 
 @deprecate_kwarg("num_logits_to_keep", version="4.50", new_name="logits_to_keep")
@@ -29,7 +30,7 @@ def lce_forward(
     logits_to_keep: Union[int, torch.Tensor] = 0,
     skip_logits: Optional[bool] = None,
     **kwargs,
-) -> Union[Tuple, CausalLMOutputWithPast]:
+) -> Union[Tuple, LigerCausalLMOutputWithPast]:
     r"""
     Copy paste Mistral's forward but replace torch cross entropy with liger fused linear cross entropy
 
@@ -94,6 +95,7 @@ def lce_forward(
     shift_labels = kwargs.pop("shift_labels", None)
     loss = None
     logits = None
+    token_accuracy = None
 
     if skip_logits and labels is None and shift_labels is None:
         raise ValueError("skip_logits is True, but labels and shift_labels are None")
@@ -101,8 +103,9 @@ def lce_forward(
     if skip_logits is None:
         skip_logits = self.training and (labels is not None or shift_labels is not None)
 
+    # Compute loss
     if skip_logits:
-        loss = LigerForCausalLMLoss(
+        result = LigerForCausalLMLoss(
             hidden_states=kept_hidden_states,
             lm_head_weight=self.lm_head.weight,
             labels=labels,
@@ -110,29 +113,33 @@ def lce_forward(
             hidden_size=self.config.hidden_size,
             **kwargs,
         )
+        loss, _, token_accuracy = unpack_cross_entropy_result(result)
 
     else:
         logits = self.lm_head(kept_hidden_states)
 
         loss = None
-        if labels is not None:
+        if labels is not None or shift_labels is not None:
             loss = self.loss_function(
                 logits=logits,
                 labels=labels,
+                shift_labels=shift_labels,
                 vocab_size=self.config.vocab_size,
                 **kwargs,
             )
+
     if not return_dict:
-        output = (logits,) + outputs[1:]
-        return (loss,) + output if loss is not None else output
+        output_tuple = (logits,) + outputs[1:]
+        output = (loss,) + output_tuple if loss is not None else output_tuple
+        output = output + (token_accuracy,) if token_accuracy is not None else output
+        return output
 
-    return CausalLMOutputWithPast(
+    # Return custom output class with token_accuracy field
+    return LigerCausalLMOutputWithPast(
         loss=loss,
         logits=logits,
         past_key_values=outputs.past_key_values,
         hidden_states=outputs.hidden_states,
         attentions=outputs.attentions,
+        token_accuracy=token_accuracy,
     )
-
-
-# Note: Grad Acc is not fixed in mistral at transformer 4.46.1

liger_kernel/transformers/model/mixtral.py

@@ -12,6 +12,8 @@ from transformers.utils.deprecation import deprecate_kwarg
 
 from liger_kernel.transformers.fused_linear_cross_entropy import LigerFusedLinearCrossEntropyLoss
 from liger_kernel.transformers.model.loss_utils import LigerForCausalLMLoss
+from liger_kernel.transformers.model.loss_utils import unpack_cross_entropy_result
+from liger_kernel.transformers.model.output_classes import LigerMoeCausalLMOutputWithPast
 
 
 def lce_forward_deprecated(
@@ -158,7 +160,7 @@ def lce_forward(
     logits_to_keep: Union[int, torch.Tensor] = 0,
     skip_logits: Optional[bool] = None,
     **kwargs,
-) -> Union[Tuple, MoeCausalLMOutputWithPast]:
+) -> Union[Tuple, LigerMoeCausalLMOutputWithPast]:
     r"""
     Args:
         labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -226,6 +228,7 @@ def lce_forward(
     shift_labels = kwargs.pop("shift_labels", None)
     logits = None
     loss = None
+    token_accuracy = None
 
     if skip_logits and labels is None and shift_labels is None:
         raise ValueError("skip_logits is True, but labels and shift_labels are None")
@@ -234,8 +237,9 @@ def lce_forward(
         # By default, if in training mode, don't materialize logits
         skip_logits = self.training and (labels is not None or shift_labels is not None)
 
+    # Compute loss
     if skip_logits:
-        loss = LigerForCausalLMLoss(
+        result = LigerForCausalLMLoss(
             hidden_states=kept_hidden_states,
             lm_head_weight=self.lm_head.weight,
             labels=labels,
@@ -243,13 +247,20 @@ def lce_forward(
             hidden_size=self.config.hidden_size,
             **kwargs,
         )
+        loss, _, token_accuracy = unpack_cross_entropy_result(result)
 
     else:
         logits = self.lm_head(kept_hidden_states)
 
         loss = None
-        if labels is not None:
-            loss = self.loss_function(logits, labels, self.vocab_size, **kwargs)
+        if labels is not None or shift_labels is not None:
+            loss = self.loss_function(
+                logits=logits,
+                labels=labels,
+                shift_labels=shift_labels,
+                vocab_size=self.vocab_size,
+                **kwargs,
+            )
     aux_loss = None
     if output_router_logits:
         aux_loss = load_balancing_loss_func(
@@ -262,17 +273,21 @@ def lce_forward(
             loss += self.router_aux_loss_coef * aux_loss.to(loss.device)  # make sure to reside in the same device
 
     if not return_dict:
-        output = (logits,) + outputs[1:]
+        output_tuple = (logits,) + outputs[1:]
         if output_router_logits:
-            output = (aux_loss,) + output
-        return (loss,) + output if loss is not None else output
+            output_tuple = (aux_loss,) + output_tuple
+        if token_accuracy is not None:
+            output_tuple = output_tuple + (token_accuracy,)
+        return (loss,) + output_tuple if loss is not None else output_tuple
 
-    return MoeCausalLMOutputWithPast(
+    # Return custom output class with token_accuracy field
+    return LigerMoeCausalLMOutputWithPast(
         loss=loss,
         aux_loss=aux_loss,
         logits=logits,
         past_key_values=outputs.past_key_values,
         hidden_states=outputs.hidden_states,
         attentions=outputs.attentions,
-        router_logits=outputs.router_logits,
+        router_logits=outputs.router_logits if return_dict else outputs[-1],
+        token_accuracy=token_accuracy,
     )

liger_kernel/transformers/model/mllama.py

@@ -12,6 +12,8 @@ from transformers.utils.deprecation import deprecate_kwarg
 
 from liger_kernel.transformers.fused_linear_cross_entropy import LigerFusedLinearCrossEntropyLoss
 from liger_kernel.transformers.model.loss_utils import LigerForCausalLMLoss
+from liger_kernel.transformers.model.loss_utils import unpack_cross_entropy_result
+from liger_kernel.transformers.model.output_classes import LigerCausalLMOutputWithPast
 
 
 def lce_forward_deprecated(
@@ -149,7 +151,7 @@ def lce_forward(
     logits_to_keep: Union[int, torch.Tensor] = 0,
     skip_logits: Optional[bool] = None,
     **kwargs,
-) -> Union[Tuple, CausalLMOutputWithPast]:
+) -> Union[Tuple, LigerCausalLMOutputWithPast]:
     r"""
     Args:
         labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -190,7 +192,9 @@ def lce_forward(
         output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
     )
     return_dict = return_dict if return_dict is not None else self.config.use_return_dict
-
+    # Filter out accum_dtype from kwargs for model call as MllamaTextModel doesn't accept it in transformers 4.49.0
+    # but preserve it for loss function calls
+    model_kwargs = {k: v for k, v in kwargs.items() if k != "accum_dtype"}
     # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
     outputs = self.model(
         input_ids=input_ids,
@@ -206,7 +210,7 @@ def lce_forward(
         output_hidden_states=output_hidden_states,
         return_dict=return_dict,
         cache_position=cache_position,
-        **kwargs,
+        **model_kwargs,
     )
 
     hidden_states = outputs[0]
@@ -217,6 +221,7 @@ def lce_forward(
     shift_labels = kwargs.pop("shift_labels", None)
     logits = None
     loss = None
+    token_accuracy = None
 
     if skip_logits and labels is None and shift_labels is None:
         raise ValueError("skip_logits is True, but labels and shift_labels are None")
@@ -226,7 +231,7 @@ def lce_forward(
         skip_logits = self.training and (labels is not None or shift_labels is not None)
 
     if skip_logits:
-        loss = LigerForCausalLMLoss(
+        result = LigerForCausalLMLoss(
             hidden_states=kept_hidden_states,
             lm_head_weight=self.lm_head.weight,
             labels=labels,
@@ -234,25 +239,31 @@ def lce_forward(
             hidden_size=self.config.hidden_size,
             **kwargs,
         )
+        loss, _, token_accuracy = unpack_cross_entropy_result(result)
 
     else:
         logits = self.lm_head(kept_hidden_states)
-        if labels is not None:
+        if labels is not None or shift_labels is not None:
             loss = self.loss_function(
                 logits=logits,
                 labels=labels,
+                shift_labels=shift_labels,
                 vocab_size=self.config.vocab_size,
                 **kwargs,
             )
 
     if not return_dict:
         output = (logits,) + outputs[1:]
-        return (loss,) + output if loss is not None else output
+        output = (loss,) + output if loss is not None else output
+        output = output + (token_accuracy,) if token_accuracy is not None else output
+        return output
 
-    return CausalLMOutputWithPast(
+    # Return custom output class with token_accuracy field
+    return LigerCausalLMOutputWithPast(
         loss=loss,
         logits=logits,
         past_key_values=outputs.past_key_values,
         hidden_states=outputs.hidden_states,
         attentions=outputs.attentions,
+        token_accuracy=token_accuracy,
     )

liger_kernel/transformers/model/olmo2.py

@@ -5,10 +5,11 @@ from typing import Union
 
 import torch
 
-from transformers.modeling_outputs import CausalLMOutputWithPast
 from transformers.utils.deprecation import deprecate_kwarg
 
 from liger_kernel.transformers.model.loss_utils import LigerForCausalLMLoss
+from liger_kernel.transformers.model.loss_utils import unpack_cross_entropy_result
+from liger_kernel.transformers.model.output_classes import LigerCausalLMOutputWithPast
 
 
 @deprecate_kwarg("num_logits_to_keep", version="4.50", new_name="logits_to_keep")
@@ -28,7 +29,7 @@ def lce_forward(
     logits_to_keep: Union[int, torch.Tensor] = 0,
     skip_logits: Optional[bool] = None,
     **kwargs,
-) -> Union[Tuple, CausalLMOutputWithPast]:
+) -> Union[Tuple, LigerCausalLMOutputWithPast]:
     r"""
     Args:
         labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -91,6 +92,7 @@ def lce_forward(
     shift_labels = kwargs.pop("shift_labels", None)
     logits = None
     loss = None
+    token_accuracy = None
 
     if skip_logits and labels is None and shift_labels is None:
         raise ValueError("skip_logits is True, but labels and shift_labels are None")
@@ -99,8 +101,9 @@ def lce_forward(
         # By default, if in training mode, don't materialize logits
         skip_logits = self.training and (labels is not None or shift_labels is not None)
 
+    # Compute loss
     if skip_logits:
-        loss = LigerForCausalLMLoss(
+        result = LigerForCausalLMLoss(
             hidden_states=kept_hidden_states,
             lm_head_weight=self.lm_head.weight,
             labels=labels,
@@ -108,21 +111,31 @@ def lce_forward(
             hidden_size=self.config.hidden_size,
             **kwargs,
         )
+        loss, _, token_accuracy = unpack_cross_entropy_result(result)
 
     else:
         logits = self.lm_head(kept_hidden_states)
-        if labels is not None:
+        if labels is not None or shift_labels is not None:
             loss = self.loss_function(
                 logits=logits,
                 labels=labels,
+                shift_labels=shift_labels,
                 vocab_size=self.config.vocab_size,
                 **kwargs,
             )
 
-    return CausalLMOutputWithPast(
+    if not return_dict:
+        output = (logits,) + outputs[1:]
+        output = ((loss,) + output) if loss is not None else output
+        output = output + (token_accuracy,) if token_accuracy is not None else output
+        return output
+
+    # Return custom output class with token_accuracy field
+    return LigerCausalLMOutputWithPast(
         loss=loss,
         logits=logits,
         past_key_values=outputs.past_key_values,
         hidden_states=outputs.hidden_states,
         attentions=outputs.attentions,
+        token_accuracy=token_accuracy,
     )

liger_kernel/transformers/model/output_classes.py

@@ -0,0 +1,147 @@
+"""
+Custom output classes for Liger-Kernel that extend transformers' ModelOutput classes
+with optional token accuracy field.
+"""
+
+from dataclasses import dataclass
+from typing import Optional
+
+import torch
+
+from transformers.modeling_outputs import CausalLMOutputWithPast
+from transformers.modeling_outputs import MoeCausalLMOutputWithPast
+
+# The following model-specific outputs are optional and depend on the installed
+# transformers version. Guard their imports so our module remains importable
+# even when those models are not available in the environment.
+try:
+    from transformers.models.gemma3.modeling_gemma3 import Gemma3CausalLMOutputWithPast as _Gemma3CausalLMOutputWithPast
+except Exception:
+    _Gemma3CausalLMOutputWithPast = None
+
+try:
+    from transformers.models.glm4v_moe.modeling_glm4v_moe import (
+        Glm4vMoeCausalLMOutputWithPast as _Glm4vMoeCausalLMOutputWithPast,
+    )
+except Exception:
+    _Glm4vMoeCausalLMOutputWithPast = None
+
+try:
+    from transformers.models.internvl.modeling_internvl import (
+        InternVLCausalLMOutputWithPast as _InternVLCausalLMOutputWithPast,
+    )
+except Exception:
+    _InternVLCausalLMOutputWithPast = None
+
+try:
+    from transformers.models.llava.modeling_llava import LlavaCausalLMOutputWithPast as _LlavaCausalLMOutputWithPast
+except Exception:
+    _LlavaCausalLMOutputWithPast = None
+
+try:
+    from transformers.models.paligemma.modeling_paligemma import (
+        PaliGemmaCausalLMOutputWithPast as _PaliGemmaCausalLMOutputWithPast,
+    )
+except Exception:
+    _PaliGemmaCausalLMOutputWithPast = None
+
+try:
+    from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import (
+        Qwen2_5_VLCausalLMOutputWithPast as _Qwen2_5_VLCausalLMOutputWithPast,
+    )
+except Exception:
+    _Qwen2_5_VLCausalLMOutputWithPast = None
+
+try:
+    from transformers.models.qwen2_vl.modeling_qwen2_vl import (
+        Qwen2VLCausalLMOutputWithPast as _Qwen2VLCausalLMOutputWithPast,
+    )
+except Exception:
+    _Qwen2VLCausalLMOutputWithPast = None
+
+try:
+    from transformers.models.qwen3_vl.modeling_qwen3_vl import (
+        Qwen3VLCausalLMOutputWithPast as _Qwen3VLCausalLMOutputWithPast,
+    )
+except Exception:
+    _Qwen3VLCausalLMOutputWithPast = None
+
+try:
+    from transformers.models.qwen3_vl_moe.modeling_qwen3_vl_moe import (
+        Qwen3VLMoeCausalLMOutputWithPast as _Qwen3VLMoeCausalLMOutputWithPast,
+    )
+except Exception:
+    _Qwen3VLMoeCausalLMOutputWithPast = None
+
+
+@dataclass
+class LigerCausalLMOutputWithPast(CausalLMOutputWithPast):
+    token_accuracy: Optional[torch.FloatTensor] = None
+
+
+@dataclass
+class LigerMoeCausalLMOutputWithPast(MoeCausalLMOutputWithPast):
+    token_accuracy: Optional[torch.FloatTensor] = None
+
+
+if _Gemma3CausalLMOutputWithPast is not None:
+
+    @dataclass
+    class LigerGemma3CausalLMOutputWithPast(_Gemma3CausalLMOutputWithPast):
+        token_accuracy: Optional[torch.FloatTensor] = None
+
+
+if _Glm4vMoeCausalLMOutputWithPast is not None:
+
+    @dataclass
+    class LigerGlm4vMoeCausalLMOutputWithPast(_Glm4vMoeCausalLMOutputWithPast):
+        token_accuracy: Optional[torch.FloatTensor] = None
+
+
+if _LlavaCausalLMOutputWithPast is not None:
+
+    @dataclass
+    class LigerLlavaCausalLMOutputWithPast(_LlavaCausalLMOutputWithPast):
+        token_accuracy: Optional[torch.FloatTensor] = None
+
+
+if _InternVLCausalLMOutputWithPast is not None:
+
+    @dataclass
+    class LigerInternVLCausalLMOutputWithPast(_InternVLCausalLMOutputWithPast):
+        token_accuracy: Optional[torch.FloatTensor] = None
+
+
+if _PaliGemmaCausalLMOutputWithPast is not None:
+
+    @dataclass
+    class LigerPaliGemmaCausalLMOutputWithPast(_PaliGemmaCausalLMOutputWithPast):
+        token_accuracy: Optional[torch.FloatTensor] = None
+
+
+if _Qwen2_5_VLCausalLMOutputWithPast is not None:
+
+    @dataclass
+    class LigerQwen2_5_VLCausalLMOutputWithPast(_Qwen2_5_VLCausalLMOutputWithPast):
+        token_accuracy: Optional[torch.FloatTensor] = None
+
+
+if _Qwen2VLCausalLMOutputWithPast is not None:
+
+    @dataclass
+    class LigerQwen2VLCausalLMOutputWithPast(_Qwen2VLCausalLMOutputWithPast):
+        token_accuracy: Optional[torch.FloatTensor] = None
+
+
+if _Qwen3VLCausalLMOutputWithPast is not None:
+
+    @dataclass
+    class LigerQwen3VLCausalLMOutputWithPast(_Qwen3VLCausalLMOutputWithPast):
+        token_accuracy: Optional[torch.FloatTensor] = None
+
+
+if _Qwen3VLMoeCausalLMOutputWithPast is not None:
+
+    @dataclass
+    class LigerQwen3VLMoeCausalLMOutputWithPast(_Qwen3VLMoeCausalLMOutputWithPast):
+        token_accuracy: Optional[torch.FloatTensor] = None

liger_kernel/transformers/model/paligemma.py

@@ -13,6 +13,9 @@ from transformers.utils import logging
 from transformers.utils.deprecation import deprecate_kwarg
 
 from liger_kernel.transformers.fused_linear_cross_entropy import LigerFusedLinearCrossEntropyLoss
+from liger_kernel.transformers.model.loss_utils import LigerForCausalLMLoss
+from liger_kernel.transformers.model.loss_utils import unpack_cross_entropy_result
+from liger_kernel.transformers.model.output_classes import LigerPaliGemmaCausalLMOutputWithPast
 
 logger = logging.get_logger(__name__)
 
@@ -218,7 +221,7 @@ def lce_forward(
     logits_to_keep: Union[int, torch.Tensor] = 0,
     skip_logits: Optional[bool] = None,
     **lm_kwargs,
-) -> Union[Tuple, PaliGemmaCausalLMOutputWithPast]:
+) -> Union[Tuple, LigerPaliGemmaCausalLMOutputWithPast]:
     r"""
     Args:
         labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -331,6 +334,7 @@ def lce_forward(
 
     loss = None
     logits = None
+    token_accuracy = None
 
     if skip_logits and labels is None:
         raise ValueError("skip_logits is True, but labels is None")
@@ -358,8 +362,16 @@ def lce_forward(
         shift_hidden_states = shift_hidden_states.view(-1, self.config.text_config.hidden_size)
         shift_labels = shift_labels.view(-1).to(hidden_device)
 
-        lce = LigerFusedLinearCrossEntropyLoss()
-        loss = lce(self.language_model.lm_head.weight, shift_hidden_states, shift_labels)
+        # Use LigerForCausalLMLoss with accuracy support and pass already shifted labels
+        result = LigerForCausalLMLoss(
+            hidden_states=shift_hidden_states,
+            lm_head_weight=self.language_model.lm_head.weight,
+            labels=None,
+            shift_labels=shift_labels,
+            hidden_size=self.config.text_config.hidden_size,
+            **lm_kwargs,
+        )
+        loss, _, token_accuracy = unpack_cross_entropy_result(result)
     else:
         logits = self.language_model.lm_head(hidden_states)
         if labels is not None:
@@ -382,15 +394,39 @@ def lce_forward(
             flat_logits = shift_logits.view(-1, self.config.text_config.vocab_size)
             flat_labels = shift_labels.view(-1).to(shift_logits.device)
             loss = loss_fct(flat_logits, flat_labels)
+        elif shift_labels is not None:
+            # Upcast to float if we need to compute the loss to avoid potential precision issues
+            logits = logits.float()
+            shift_logits = logits[..., :-1, :]
+            if attention_mask is not None:
+                # we use the input attention mask to shift the logits and labels, because it is 2D.
+                # we also crop attn mask in case it is longer, which happens in PrefixTuning with peft
+                shift_attention_mask = attention_mask[:, -shift_logits.shape[1] :].to(logits.device)
+                shift_logits = shift_logits[shift_attention_mask.to(logits.device) != 0].contiguous()
+                shift_labels = shift_labels[shift_attention_mask.to(shift_labels.device) != 0].contiguous()
+            else:
+                shift_logits = shift_logits.contiguous()
+                shift_labels = shift_labels.contiguous()
+            # Flatten the tokens
+            loss_fct = CrossEntropyLoss()
+
+            flat_logits = shift_logits.view(-1, self.config.text_config.vocab_size)
+            flat_labels = shift_labels.view(-1).to(shift_logits.device)
+            loss = loss_fct(flat_logits, flat_labels)
+
     if not return_dict:
         output = (logits,) + outputs[1:]
-        return (loss,) + output if loss is not None else output
+        output = (loss,) + output if loss is not None else output
+        output = output + (token_accuracy,) if token_accuracy is not None else output
+        return output
 
-    return PaliGemmaCausalLMOutputWithPast(
+    # Return PaliGemma output with token_accuracy field
+    return LigerPaliGemmaCausalLMOutputWithPast(
         loss=loss,
         logits=logits,
         past_key_values=outputs.past_key_values,
         hidden_states=outputs.hidden_states,
         attentions=outputs.attentions,
         image_hidden_states=image_features if pixel_values is not None else None,
+        token_accuracy=token_accuracy,
     )