liger-kernel-nightly 0.6.1.dev20250812205818__tar.gz → 0.6.1.dev20250819172918__tar.gz

{liger_kernel_nightly-0.6.1.dev20250812205818 → liger_kernel_nightly-0.6.1.dev20250819172918}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: liger_kernel_nightly
-Version: 0.6.1.dev20250812205818
+Version: 0.6.1.dev20250819172918
 Summary: Efficient Triton kernels for LLM Training
 License: BSD 2-CLAUSE LICENSE
         Copyright 2024 LinkedIn Corporation 

{liger_kernel_nightly-0.6.1.dev20250812205818 → liger_kernel_nightly-0.6.1.dev20250819172918}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "liger_kernel_nightly"
-version = "0.6.1.dev20250812205818"
+version = "0.6.1.dev20250819172918"
 description = "Efficient Triton kernels for LLM Training"
 urls = { "Homepage" = "https://github.com/linkedin/Liger-Kernel" }
 readme = { file = "README.md", content-type = "text/markdown" }

{liger_kernel_nightly-0.6.1.dev20250812205818 → liger_kernel_nightly-0.6.1.dev20250819172918}/src/liger_kernel/transformers/__init__.py

@@ -10,11 +10,15 @@ from liger_kernel.transformers.fused_linear_cross_entropy import LigerFusedLinea
 from liger_kernel.transformers.fused_linear_jsd import LigerFusedLinearJSD  # noqa: F401
 from liger_kernel.transformers.geglu import LigerGEGLUMLP  # noqa: F401
 from liger_kernel.transformers.jsd import LigerJSD  # noqa: F401
+from liger_kernel.transformers.kl_div import LigerKLDIVLoss  # noqa: F401
 from liger_kernel.transformers.layer_norm import LigerLayerNorm  # noqa: F401
 from liger_kernel.transformers.llama4_rope import liger_llama4_text_rotary_pos_emb  # noqa: F401
 from liger_kernel.transformers.llama4_rope import liger_llama4_vision_rotary_pos_emb  # noqa: F401
+from liger_kernel.transformers.multi_token_attention import LigerMultiTokenAttention  # noqa: F401
 from liger_kernel.transformers.rms_norm import LigerRMSNorm  # noqa: F401
 from liger_kernel.transformers.rope import liger_rotary_pos_emb  # noqa: F401
+from liger_kernel.transformers.softmax import LigerSoftmax  # noqa: F401
+from liger_kernel.transformers.sparsemax import LigerSparsemax  # noqa: F401
 from liger_kernel.transformers.swiglu import LigerBlockSparseTop2MLP  # noqa: F401
 from liger_kernel.transformers.swiglu import LigerPhi3SwiGLUMLP  # noqa: F401
 from liger_kernel.transformers.swiglu import LigerQwen3MoeSwiGLUMLP  # noqa: F401
@@ -134,6 +138,10 @@ __all__ = [
     "LigerQwen3MoeSwiGLUMLP",
     "LigerSwiGLUMLP",
     "LigerTVDLoss",
+    "LigerKLDIVLoss",
+    "LigerMultiTokenAttention",
+    "LigerSoftmax",
+    "LigerSparsemax",
 ]
 
 # Add transformer-dependent symbols only if available

liger_kernel_nightly-0.6.1.dev20250819172918/src/liger_kernel/transformers/experimental/__init__.py

@@ -0,0 +1,5 @@
+from liger_kernel.transformers.experimental.embedding import LigerEmbedding  # noqa: F401
+
+__all__ = [
+    "LigerEmbedding",
+]

{liger_kernel_nightly-0.6.1.dev20250812205818 → liger_kernel_nightly-0.6.1.dev20250819172918}/src/liger_kernel/transformers/model/mllama.py

@@ -190,7 +190,9 @@ def lce_forward(
         output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
     )
     return_dict = return_dict if return_dict is not None else self.config.use_return_dict
-
+    # Filter out accum_dtype from kwargs for model call as MllamaTextModel doesn't accept it in transformers 4.49.0
+    # but preserve it for loss function calls
+    model_kwargs = {k: v for k, v in kwargs.items() if k != "accum_dtype"}
     # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
     outputs = self.model(
         input_ids=input_ids,
@@ -206,7 +208,7 @@ def lce_forward(
         output_hidden_states=output_hidden_states,
         return_dict=return_dict,
         cache_position=cache_position,
-        **kwargs,
+        **model_kwargs,
     )
 
     hidden_states = outputs[0]

{liger_kernel_nightly-0.6.1.dev20250812205818 → liger_kernel_nightly-0.6.1.dev20250819172918}/src/liger_kernel_nightly.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: liger_kernel_nightly
-Version: 0.6.1.dev20250812205818
+Version: 0.6.1.dev20250819172918
 Summary: Efficient Triton kernels for LLM Training
 License: BSD 2-CLAUSE LICENSE
         Copyright 2024 LinkedIn Corporation 

{liger_kernel_nightly-0.6.1.dev20250812205818 → liger_kernel_nightly-0.6.1.dev20250819172918}/src/liger_kernel_nightly.egg-info/SOURCES.txt

@@ -183,6 +183,7 @@ src/liger_kernel/transformers/sparsemax.py
 src/liger_kernel/transformers/swiglu.py
 src/liger_kernel/transformers/trainer_integration.py
 src/liger_kernel/transformers/tvd.py
+src/liger_kernel/transformers/experimental/__init__.py
 src/liger_kernel/transformers/experimental/embedding.py
 src/liger_kernel/transformers/model/__init__.py
 src/liger_kernel/transformers/model/gemma.py

{liger_kernel_nightly-0.6.1.dev20250812205818 → liger_kernel_nightly-0.6.1.dev20250819172918}/test/convergence/bf16/test_mini_models_multimodal.py

@@ -557,6 +557,7 @@ if QWEN2_5_VL_AVAILABLE:
                 "hidden_size": 128,  # 1280
                 "num_heads": 16,
                 "in_chans": 3,
+                "out_hidden_size": 1024,
             },
             attn_implementation="sdpa",
         ),
@@ -860,7 +861,17 @@ def run_mini_model_multimodal(
     for i in range(num_steps):
         batch = next(loader_iter).to(model.device)
         optimizer.zero_grad()
-        output = model(**batch, accum_dtype=torch.float32)
+        supports_accum = getattr(model, "_supports_accum_dtype", None)
+        if supports_accum is None:
+            import inspect
+
+            params = inspect.signature(model.forward).parameters
+            supports_accum = ("accum_dtype" in params) or any(
+                p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values()
+            )
+            setattr(model, "_supports_accum_dtype", supports_accum)
+
+        output = model(**batch, accum_dtype=torch.float32) if supports_accum else model(**batch)
         output.loss.backward()
         optimizer.step()
 

{liger_kernel_nightly-0.6.1.dev20250812205818 → liger_kernel_nightly-0.6.1.dev20250819172918}/test/convergence/fp32/test_mini_models_multimodal.py

@@ -555,6 +555,7 @@ if QWEN2_5_VL_AVAILABLE:
                 "hidden_size": 128,  # 1280
                 "num_heads": 16,
                 "in_chans": 3,
+                "out_hidden_size": 1024,
             },
             attn_implementation="sdpa",
         ),