liger-kernel-nightly 0.6.3.dev20251101160510__py3-none-any.whl → 0.6.3.dev20251105012545__py3-none-any.whl

liger_kernel/transformers/model/qwen3_vl_moe.py

@@ -5,13 +5,11 @@ from typing import Union
 
 import torch
 
+from transformers.models.qwen3_vl_moe.modeling_qwen3_vl_moe import Qwen3VLMoeCausalLMOutputWithPast
+from transformers.models.qwen3_vl_moe.modeling_qwen3_vl_moe import load_balancing_loss_func
 from transformers.utils import can_return_tuple
 
 from liger_kernel.transformers.model.loss_utils import LigerForCausalLMLoss
-from transformers.models.qwen3_vl_moe.modeling_qwen3_vl_moe import (
-    Qwen3VLMoeCausalLMOutputWithPast,
-    load_balancing_loss_func,
-)
 
 
 @can_return_tuple

liger_kernel/transformers/monkey_patch.py

@@ -6,7 +6,6 @@ from types import MethodType
 from typing import Callable
 from typing import Optional
 
-import torch
 import transformers
 
 from packaging import version
@@ -36,11 +35,9 @@ from liger_kernel.transformers.model.qwen3_vl_moe import lce_forward as qwen3_vl
 from liger_kernel.transformers.model.smollm3 import lce_forward as smollm3_lce_forward
 from liger_kernel.transformers.qwen2vl_mrope import liger_multimodal_rotary_pos_emb
 from liger_kernel.transformers.rms_norm import LigerRMSNorm
-from liger_kernel.transformers.rope import (
-    liger_rotary_pos_emb,
-    liger_rotary_pos_emb_with_cast,
-    liger_rotary_pos_emb_with_cast_and_leading_batch,
-)
+from liger_kernel.transformers.rope import liger_rotary_pos_emb
+from liger_kernel.transformers.rope import liger_rotary_pos_emb_with_cast
+from liger_kernel.transformers.rope import liger_rotary_pos_emb_with_cast_and_leading_batch
 from liger_kernel.transformers.swiglu import LigerBlockSparseTop2MLP
 from liger_kernel.transformers.swiglu import LigerPhi3SwiGLUMLP
 from liger_kernel.transformers.swiglu import LigerSwiGLUMLP
@@ -64,7 +61,6 @@ def _bind_method_to_module(module, method_name: str, new_method: Callable):
     module.__dict__[method_name] = new_method.__get__(module, module.__class__)
 
 
-
 def _patch_rms_norm_module(module, offset=0.0, eps=1e-6, casting_mode="llama", in_place=True, row_mode=None):
     # Check if the module is a PEFT ModulesToSaveWrapper
     # If it is, we need to patch the modules_to_save.default and original_modules
@@ -1651,7 +1647,6 @@ def apply_liger_kernel_to_qwen2_5_vl(
                     _patch_rms_norm_module(decoder_layer.post_attention_layernorm)
 
 
-
 def apply_liger_kernel_to_qwen3_vl(
     rope: bool = True,
     cross_entropy: bool = False,
@@ -1688,7 +1683,6 @@ def apply_liger_kernel_to_qwen3_vl(
         modeling_qwen3_vl.apply_rotary_pos_emb = liger_rotary_pos_emb_with_cast
         modeling_qwen3_vl.apply_rotary_pos_emb_vision = liger_rotary_pos_emb_with_cast_and_leading_batch
 
-
     if rms_norm:
         modeling_qwen3_vl.Qwen3VLTextRMSNorm = LigerRMSNorm
 

liger_kernel/transformers/rope.py

@@ -1,4 +1,5 @@
-from typing import Optional, Tuple
+from typing import Optional
+from typing import Tuple
 
 import torch
 
@@ -32,7 +33,6 @@ def liger_rotary_pos_emb_with_cast(
     position_ids: Optional[torch.Tensor] = None,
     unsqueeze_dim: int = 1,
 ) -> Tuple[torch.Tensor, torch.Tensor]:
-
     orig_q_dtype, orig_k_dtype = q.dtype, k.dtype
 
     q32 = q.to(torch.float32)
@@ -52,8 +52,6 @@ def liger_rotary_pos_emb_with_cast_and_leading_batch(
     position_ids: Optional[torch.Tensor] = None,
     unsqueeze_dim: int = 1,
 ) -> Tuple[torch.Tensor, torch.Tensor]:
-
-
     orig_q_dtype, orig_k_dtype = q.dtype, k.dtype
 
     q32 = q.to(torch.float32).unsqueeze(0)
@@ -61,7 +59,5 @@ def liger_rotary_pos_emb_with_cast_and_leading_batch(
     cos32 = cos.to(torch.float32).unsqueeze(0)
     sin32 = sin.to(torch.float32).unsqueeze(0)
 
-    q_out, k_out = liger_rotary_pos_emb(
-        q32, k32, cos32, sin32, position_ids=position_ids, unsqueeze_dim=unsqueeze_dim
-    )
+    q_out, k_out = liger_rotary_pos_emb(q32, k32, cos32, sin32, position_ids=position_ids, unsqueeze_dim=unsqueeze_dim)
     return q_out.to(orig_q_dtype).squeeze(0), k_out.to(orig_k_dtype).squeeze(0)

{liger_kernel_nightly-0.6.3.dev20251101160510.dist-info → liger_kernel_nightly-0.6.3.dev20251105012545.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: liger_kernel_nightly
-Version: 0.6.3.dev20251101160510
+Version: 0.6.3.dev20251105012545
 Summary: Efficient Triton kernels for LLM Training
 License: BSD 2-CLAUSE LICENSE
         Copyright 2024 LinkedIn Corporation 

{liger_kernel_nightly-0.6.3.dev20251101160510.dist-info → liger_kernel_nightly-0.6.3.dev20251105012545.dist-info}/RECORD

@@ -59,12 +59,12 @@ liger_kernel/transformers/jsd.py,sha256=DGqRnxIZxsvxo0_tbbxX3b-sDbDjC_yKufyRIHCc
 liger_kernel/transformers/kl_div.py,sha256=WLffFbh1EExD2Eb1F7lN11fo9JJC-0751WJjZAF1Fj8,409
 liger_kernel/transformers/layer_norm.py,sha256=c9pk3PEasOKYR0rhe5e5nNrnYKVCEW4VC8S6LpCq9EQ,906
 liger_kernel/transformers/llama4_rope.py,sha256=kS6PSHEwf3dS7hD7C7p8S0geugx2EMCiP0h0F7LsUoY,3639
-liger_kernel/transformers/monkey_patch.py,sha256=Qo5phPCiSF_w29R5AiDO382penkmzuEijv_iNenuuHc,124681
+liger_kernel/transformers/monkey_patch.py,sha256=O_kl0l56oHinVv-bwl1LU5nKPm6nA0YBjKTYmmwgRbk,124732
 liger_kernel/transformers/multi_token_attention.py,sha256=K3NIY9_5TPgZ4_Rahn0xnkMXxD_fmlJHK4CWGYvGQp0,1752
 liger_kernel/transformers/poly_norm.py,sha256=g5tC75i3qy1_N26ZUP-jfpct7ivQAEdJfIfx8IXzeyE,1377
 liger_kernel/transformers/qwen2vl_mrope.py,sha256=5EwSqrMdsL9MYspeBMXBsNJKvH0MOmRrtJXAJlnnlOI,1047
 liger_kernel/transformers/rms_norm.py,sha256=HwddVqrqS58jE-M2_4NkFGARtCDBhGnkKyjBN9b3FYI,3004
-liger_kernel/transformers/rope.py,sha256=SoOyYArsioIQzp6eZo6vnFumISf06Gl3O8WWkMmr-gQ,2360
+liger_kernel/transformers/rope.py,sha256=VMlDZI6zss9mLaLcN5XCE_ktmYRwAi_Eh4TIgO6NrIQ,2361
 liger_kernel/transformers/softmax.py,sha256=yadlAgE4V2JByMwrDDa2s5SUBp8Jgd57xwnVvAWoBaI,264
 liger_kernel/transformers/sparsemax.py,sha256=0lQA0UEOs4mu8CMruZ3VLhImxQVXJWhPsAKUsYA7vj8,403
 liger_kernel/transformers/swiglu.py,sha256=LZ8YeLIdv2k46JleZMjzubGk98smt6t780kSgcVLsQk,3454
@@ -98,16 +98,16 @@ liger_kernel/transformers/model/qwen3.py,sha256=Q2aOg5erPrgVgRcqJm8sefLSDtvU1AD5
 liger_kernel/transformers/model/qwen3_moe.py,sha256=1CwTMCNFDYsjGoa_aHFBagtC5HuJTV-s0__5UvcjD3A,5686
 liger_kernel/transformers/model/qwen3_next.py,sha256=7To7azriAogxeE7oEvByKztH9154dnDiDVNHHm7PZK4,5632
 liger_kernel/transformers/model/qwen3_vl.py,sha256=YU76HJ0A9kG5CUaZM4i9Bzci4eeXcNl_VSC2tsPWA3k,6301
-liger_kernel/transformers/model/qwen3_vl_moe.py,sha256=0WuGA-pg5hzKPKc_B3d32qyzXMlkVi3_wlNu9d0KLOg,4392
+liger_kernel/transformers/model/qwen3_vl_moe.py,sha256=ykNIvGBtmcTkn236lhmJHzU1IHVR1Kq1YYYlJ5ynhw4,4445
 liger_kernel/transformers/model/smollm3.py,sha256=0KWVkDtXbjsBKhJnaquV6vUUYyLtfmNwYH0sxJt-qTk,7667
 liger_kernel/transformers/model/smolvlm.py,sha256=yFpPKawLVo3zXzLjM7Y_T8FyRrPxVyp-YPFMM8m3k0c,6734
 liger_kernel/transformers/trainer/__init__.py,sha256=p7yQfklV8-467qSz_ZMimkbDF7HHWHwku25A-GYL0WU,193
 liger_kernel/transformers/trainer/orpo_trainer.py,sha256=tX0h63aOFe3rNqTmk6JpMf75UPo981yzEa6TghnjS0Q,5370
 liger_kernel/triton/__init__.py,sha256=qCiCamzCRv6lpV8IqpAc9YMdNKC7GKurClWceQPnlis,92
 liger_kernel/triton/monkey_patch.py,sha256=Rd0hUHAzDkFfHvnX7-PBaNK5EKnZhtfM_h-fgQH9HPY,1568
-liger_kernel_nightly-0.6.3.dev20251101160510.dist-info/LICENSE,sha256=OhzLDHJ0to4a8sodVLELZiCFylZ1NAAYLs-HrjPy0ag,1312
-liger_kernel_nightly-0.6.3.dev20251101160510.dist-info/METADATA,sha256=rsY01xVUY_8qxjoUXKklmwMso2nGFtFS5caQA2iDGlE,24777
-liger_kernel_nightly-0.6.3.dev20251101160510.dist-info/NOTICE,sha256=njwnoPZLh9AN8SJQzxvCGLHi-8X__AvWRze6joNXIY8,2066
-liger_kernel_nightly-0.6.3.dev20251101160510.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
-liger_kernel_nightly-0.6.3.dev20251101160510.dist-info/top_level.txt,sha256=2eghu4hA3LnkM7ElW92tQ8zegWKgSbeo-k-aGe1YnvY,13
-liger_kernel_nightly-0.6.3.dev20251101160510.dist-info/RECORD,,
+liger_kernel_nightly-0.6.3.dev20251105012545.dist-info/LICENSE,sha256=OhzLDHJ0to4a8sodVLELZiCFylZ1NAAYLs-HrjPy0ag,1312
+liger_kernel_nightly-0.6.3.dev20251105012545.dist-info/METADATA,sha256=MKC5NuGeIkIrDXRVDM3wv-p0cyVbwya5NujVcmSz-mQ,24777
+liger_kernel_nightly-0.6.3.dev20251105012545.dist-info/NOTICE,sha256=njwnoPZLh9AN8SJQzxvCGLHi-8X__AvWRze6joNXIY8,2066
+liger_kernel_nightly-0.6.3.dev20251105012545.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
+liger_kernel_nightly-0.6.3.dev20251105012545.dist-info/top_level.txt,sha256=2eghu4hA3LnkM7ElW92tQ8zegWKgSbeo-k-aGe1YnvY,13
+liger_kernel_nightly-0.6.3.dev20251105012545.dist-info/RECORD,,