liger-kernel-nightly 0.4.0.dev20241107052928__py3-none-any.whl → 0.6.3.dev20251121010306__py3-none-any.whl

liger_kernel/transformers/model/olmo2.py

@@ -0,0 +1,141 @@
+from typing import List
+from typing import Optional
+from typing import Tuple
+from typing import Union
+
+import torch
+
+from transformers.utils.deprecation import deprecate_kwarg
+
+from liger_kernel.transformers.model.loss_utils import LigerForCausalLMLoss
+from liger_kernel.transformers.model.loss_utils import unpack_cross_entropy_result
+from liger_kernel.transformers.model.output_classes import LigerCausalLMOutputWithPast
+
+
+@deprecate_kwarg("num_logits_to_keep", version="4.50", new_name="logits_to_keep")
+def lce_forward(
+    self,
+    input_ids: torch.LongTensor = None,
+    attention_mask: Optional[torch.Tensor] = None,
+    position_ids: Optional[torch.LongTensor] = None,
+    past_key_values: Optional[List[torch.FloatTensor]] = None,
+    inputs_embeds: Optional[torch.FloatTensor] = None,
+    labels: Optional[torch.LongTensor] = None,
+    use_cache: Optional[bool] = None,
+    output_attentions: Optional[bool] = None,
+    output_hidden_states: Optional[bool] = None,
+    return_dict: Optional[bool] = None,
+    cache_position: Optional[torch.LongTensor] = None,
+    logits_to_keep: Union[int, torch.Tensor] = 0,
+    skip_logits: Optional[bool] = None,
+    **kwargs,
+) -> Union[Tuple, LigerCausalLMOutputWithPast]:
+    r"""
+    Args:
+        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+
+        logits_to_keep (`int` or `torch.Tensor`, *optional*):
+            If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all
+            `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that
+            token can save memory, which becomes pretty significant for long sequences or large vocabulary size.
+            If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension.
+            This is useful when using packed tensor format (single dimension for batch and sequence length).
+
+    Returns:
+
+    Example:
+
+    ```python
+    >>> from transformers import AutoTokenizer, Olmo2ForCausalLM
+
+    >>> model = Olmo2ForCausalLM.from_pretrained("allenai/Olmo2-1B-hf")
+    >>> tokenizer = AutoTokenizer.from_pretrained("allenai/Olmo2-1B-hf")
+
+    >>> prompt = "Hey, are you conscious? Can you talk to me?"
+    >>> inputs = tokenizer(prompt, return_tensors="pt")
+
+    >>> # Generate
+    >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
+    >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
+    'Hey, are you conscious? Can you talk to me?\nI’m not sure if you’re conscious of this, but I’m'
+    ```
+    """
+    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+    output_hidden_states = (
+        output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+    )
+    return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+    # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
+    outputs = self.model(
+        input_ids=input_ids,
+        attention_mask=attention_mask,
+        position_ids=position_ids,
+        past_key_values=past_key_values,
+        inputs_embeds=inputs_embeds,
+        use_cache=use_cache,
+        output_attentions=output_attentions,
+        output_hidden_states=output_hidden_states,
+        return_dict=return_dict,
+        cache_position=cache_position,
+        **kwargs,
+    )
+
+    hidden_states = outputs[0]
+    # Only compute necessary logits, and do not upcast them to float if we are not computing the loss
+    slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
+    kept_hidden_states = hidden_states[:, slice_indices, :]
+
+    shift_labels = kwargs.pop("shift_labels", None)
+    logits = None
+    loss = None
+    token_accuracy = None
+
+    if skip_logits and labels is None and shift_labels is None:
+        raise ValueError("skip_logits is True, but labels and shift_labels are None")
+
+    if skip_logits is None:
+        # By default, if in training mode, don't materialize logits
+        skip_logits = self.training and (labels is not None or shift_labels is not None)
+
+    # Compute loss
+    if skip_logits:
+        result = LigerForCausalLMLoss(
+            hidden_states=kept_hidden_states,
+            lm_head_weight=self.lm_head.weight,
+            labels=labels,
+            shift_labels=shift_labels,
+            hidden_size=self.config.hidden_size,
+            **kwargs,
+        )
+        loss, _, token_accuracy = unpack_cross_entropy_result(result)
+
+    else:
+        logits = self.lm_head(kept_hidden_states)
+        if labels is not None or shift_labels is not None:
+            loss = self.loss_function(
+                logits=logits,
+                labels=labels,
+                shift_labels=shift_labels,
+                vocab_size=self.config.vocab_size,
+                **kwargs,
+            )
+
+    if not return_dict:
+        output = (logits,) + outputs[1:]
+        output = ((loss,) + output) if loss is not None else output
+        output = output + (token_accuracy,) if token_accuracy is not None else output
+        return output
+
+    # Return custom output class with token_accuracy field
+    return LigerCausalLMOutputWithPast(
+        loss=loss,
+        logits=logits,
+        past_key_values=outputs.past_key_values,
+        hidden_states=outputs.hidden_states,
+        attentions=outputs.attentions,
+        token_accuracy=token_accuracy,
+    )

liger_kernel/transformers/model/output_classes.py

@@ -0,0 +1,147 @@
+"""
+Custom output classes for Liger-Kernel that extend transformers' ModelOutput classes
+with optional token accuracy field.
+"""
+
+from dataclasses import dataclass
+from typing import Optional
+
+import torch
+
+from transformers.modeling_outputs import CausalLMOutputWithPast
+from transformers.modeling_outputs import MoeCausalLMOutputWithPast
+
+# The following model-specific outputs are optional and depend on the installed
+# transformers version. Guard their imports so our module remains importable
+# even when those models are not available in the environment.
+try:
+    from transformers.models.gemma3.modeling_gemma3 import Gemma3CausalLMOutputWithPast as _Gemma3CausalLMOutputWithPast
+except Exception:
+    _Gemma3CausalLMOutputWithPast = None
+
+try:
+    from transformers.models.glm4v_moe.modeling_glm4v_moe import (
+        Glm4vMoeCausalLMOutputWithPast as _Glm4vMoeCausalLMOutputWithPast,
+    )
+except Exception:
+    _Glm4vMoeCausalLMOutputWithPast = None
+
+try:
+    from transformers.models.internvl.modeling_internvl import (
+        InternVLCausalLMOutputWithPast as _InternVLCausalLMOutputWithPast,
+    )
+except Exception:
+    _InternVLCausalLMOutputWithPast = None
+
+try:
+    from transformers.models.llava.modeling_llava import LlavaCausalLMOutputWithPast as _LlavaCausalLMOutputWithPast
+except Exception:
+    _LlavaCausalLMOutputWithPast = None
+
+try:
+    from transformers.models.paligemma.modeling_paligemma import (
+        PaliGemmaCausalLMOutputWithPast as _PaliGemmaCausalLMOutputWithPast,
+    )
+except Exception:
+    _PaliGemmaCausalLMOutputWithPast = None
+
+try:
+    from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import (
+        Qwen2_5_VLCausalLMOutputWithPast as _Qwen2_5_VLCausalLMOutputWithPast,
+    )
+except Exception:
+    _Qwen2_5_VLCausalLMOutputWithPast = None
+
+try:
+    from transformers.models.qwen2_vl.modeling_qwen2_vl import (
+        Qwen2VLCausalLMOutputWithPast as _Qwen2VLCausalLMOutputWithPast,
+    )
+except Exception:
+    _Qwen2VLCausalLMOutputWithPast = None
+
+try:
+    from transformers.models.qwen3_vl.modeling_qwen3_vl import (
+        Qwen3VLCausalLMOutputWithPast as _Qwen3VLCausalLMOutputWithPast,
+    )
+except Exception:
+    _Qwen3VLCausalLMOutputWithPast = None
+
+try:
+    from transformers.models.qwen3_vl_moe.modeling_qwen3_vl_moe import (
+        Qwen3VLMoeCausalLMOutputWithPast as _Qwen3VLMoeCausalLMOutputWithPast,
+    )
+except Exception:
+    _Qwen3VLMoeCausalLMOutputWithPast = None
+
+
+@dataclass
+class LigerCausalLMOutputWithPast(CausalLMOutputWithPast):
+    token_accuracy: Optional[torch.FloatTensor] = None
+
+
+@dataclass
+class LigerMoeCausalLMOutputWithPast(MoeCausalLMOutputWithPast):
+    token_accuracy: Optional[torch.FloatTensor] = None
+
+
+if _Gemma3CausalLMOutputWithPast is not None:
+
+    @dataclass
+    class LigerGemma3CausalLMOutputWithPast(_Gemma3CausalLMOutputWithPast):
+        token_accuracy: Optional[torch.FloatTensor] = None
+
+
+if _Glm4vMoeCausalLMOutputWithPast is not None:
+
+    @dataclass
+    class LigerGlm4vMoeCausalLMOutputWithPast(_Glm4vMoeCausalLMOutputWithPast):
+        token_accuracy: Optional[torch.FloatTensor] = None
+
+
+if _LlavaCausalLMOutputWithPast is not None:
+
+    @dataclass
+    class LigerLlavaCausalLMOutputWithPast(_LlavaCausalLMOutputWithPast):
+        token_accuracy: Optional[torch.FloatTensor] = None
+
+
+if _InternVLCausalLMOutputWithPast is not None:
+
+    @dataclass
+    class LigerInternVLCausalLMOutputWithPast(_InternVLCausalLMOutputWithPast):
+        token_accuracy: Optional[torch.FloatTensor] = None
+
+
+if _PaliGemmaCausalLMOutputWithPast is not None:
+
+    @dataclass
+    class LigerPaliGemmaCausalLMOutputWithPast(_PaliGemmaCausalLMOutputWithPast):
+        token_accuracy: Optional[torch.FloatTensor] = None
+
+
+if _Qwen2_5_VLCausalLMOutputWithPast is not None:
+
+    @dataclass
+    class LigerQwen2_5_VLCausalLMOutputWithPast(_Qwen2_5_VLCausalLMOutputWithPast):
+        token_accuracy: Optional[torch.FloatTensor] = None
+
+
+if _Qwen2VLCausalLMOutputWithPast is not None:
+
+    @dataclass
+    class LigerQwen2VLCausalLMOutputWithPast(_Qwen2VLCausalLMOutputWithPast):
+        token_accuracy: Optional[torch.FloatTensor] = None
+
+
+if _Qwen3VLCausalLMOutputWithPast is not None:
+
+    @dataclass
+    class LigerQwen3VLCausalLMOutputWithPast(_Qwen3VLCausalLMOutputWithPast):
+        token_accuracy: Optional[torch.FloatTensor] = None
+
+
+if _Qwen3VLMoeCausalLMOutputWithPast is not None:
+
+    @dataclass
+    class LigerQwen3VLMoeCausalLMOutputWithPast(_Qwen3VLMoeCausalLMOutputWithPast):
+        token_accuracy: Optional[torch.FloatTensor] = None