liger-kernel-nightly 0.5.3.dev20250221011057__tar.gz → 0.5.3.dev20250221011147__tar.gz

{liger_kernel_nightly-0.5.3.dev20250221011057 → liger_kernel_nightly-0.5.3.dev20250221011147}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: liger_kernel_nightly
-Version: 0.5.3.dev20250221011057
+Version: 0.5.3.dev20250221011147
 Summary: Efficient Triton kernels for LLM Training
 License: BSD 2-CLAUSE LICENSE
         Copyright 2024 LinkedIn Corporation 

{liger_kernel_nightly-0.5.3.dev20250221011057 → liger_kernel_nightly-0.5.3.dev20250221011147}/examples/huggingface/training.py

@@ -15,7 +15,6 @@ from liger_kernel.transformers import AutoLigerKernelForCausalLM
 class CustomArguments:
     model_name: str = "meta-llama/Meta-Llama-3-8B"
     dataset: str = "tatsu-lab/alpaca"
-    max_seq_length: int = 512
     use_liger: bool = False
 
 
@@ -66,7 +65,6 @@ def train():
         model=model,
         args=training_args,
         data_collator=collator,
-        max_seq_length=custom_args.max_seq_length,
         train_dataset=train_dataset,
         eval_dataset=eval_dataset,
         formatting_func=formatting_prompts_func,

{liger_kernel_nightly-0.5.3.dev20250221011057 → liger_kernel_nightly-0.5.3.dev20250221011147}/examples/huggingface/training_multimodal.py

@@ -1,15 +1,11 @@
 import os
-
-from dataclasses import dataclass
-
-import datasets
 import torch
 import transformers
-
-from callback import EfficiencyCallback
+import datasets
+from dataclasses import dataclass
+from trl import SFTTrainer, SFTConfig
+from trl.trainer import ConstantLengthDataset
 from datasets import Image as ImageFeature
-from trl import SFTTrainer
-
 from liger_kernel.transformers import monkey_patch
 
 
@@ -19,8 +15,6 @@ class CustomArguments:
     dataset: str = "HuggingFaceM4/the_cauldron"
     dataset_subset: str = "ai2d"
     dataset_split: str = "train"
-    max_seq_length: int = 512
-    dataset_text_field: str = "texts"
     use_liger: bool = False
 
 
@@ -95,75 +89,37 @@ def _format_for_convo(example, tokenizer):
 def train():
     parser = transformers.HfArgumentParser((transformers.TrainingArguments, CustomArguments))
     training_args, custom_args = parser.parse_args_into_dataclasses()
-    training_args.remove_unused_columns = False  # required to not drop the image column
-    training_args.dataset_kwargs = {"skip_prepare_dataset": True}
-
-    model, processor, image_token_id = construct_model_and_processor(custom_args.model_name, custom_args.use_liger)
 
-    dataset = (
-        datasets.load_dataset(
-            custom_args.dataset,
-            custom_args.dataset_subset,
-            split=custom_args.dataset_split,
-        )
-        .map(
-            _validate_and_extract_the_cauldron,
-            batched=True,
-            num_proc=min(os.cpu_count(), 16),
-            desc="Extracting text and images",
-        )
-        .map(
-            _format_for_convo,
-            fn_kwargs={"tokenizer": processor.tokenizer},
-            desc="Formatting for convo",
-        )
-        .cast_column("images", ImageFeature())
-        .train_test_split(test_size=0.1)
+    model, processor, image_token_id = construct_model_and_processor(
+        custom_args.model_name, custom_args.use_liger
     )
 
-    train_dataset = dataset["train"]
-    eval_dataset = dataset["test"]
-
-    def collate_fn(examples):
-        """
-        Taken directly from the TRL documentation with minor modifications:
-        https://huggingface.co/docs/trl/en/sft_trainer#a-custom-collator-for-processing-multi-modal-data
-
-        Modifications:
-        1. `apply_chat_template` is used to preprocess the texts before training begins (see above)
-        2. `example["messages"]` -> `example["texts"]` to conform with the_cauldron dataset schema
-        3. Ignoring image tokens in the loss computation
-        """
-        # Get the texts and images
-        texts = [example["texts"] for example in examples]
-        images = [example["images"] for example in examples]
-
-        # Tokenize the texts and process the images
-        batch = processor(text=texts, images=images, return_tensors="pt", padding=True)
-
-        # The labels are the input_ids, and we mask the padding tokens in the loss computation
-        labels = batch["input_ids"].clone()
-        labels[labels == processor.tokenizer.pad_token_id] = -100
+    dataset = datasets.load_dataset(
+        custom_args.dataset, 
+        custom_args.dataset_subset, 
+        split=custom_args.dataset_split
+    )
 
-        # Ignore the image token index in the loss computation
-        labels[labels == image_token_id] = -100
-        batch["labels"] = labels
+    train_dataset, eval_dataset = prepare_dataset(dataset, processor, image_token_id)
 
-        return batch
+    sft_config = SFTConfig(
+        output_dir=training_args.output_dir,
+        per_device_train_batch_size=training_args.per_device_train_batch_size,
+        per_device_eval_batch_size=training_args.per_device_eval_batch_size,
+        learning_rate=training_args.learning_rate,
+        num_train_epochs=training_args.num_train_epochs,
+        gradient_accumulation_steps=training_args.gradient_accumulation_steps,
+    )
 
     trainer = SFTTrainer(
         model=model,
-        args=training_args,
-        data_collator=collate_fn,
-        max_seq_length=custom_args.max_seq_length,
-        dataset_text_field=custom_args.dataset_text_field,
+        args=sft_config,
         train_dataset=train_dataset,
         eval_dataset=eval_dataset,
-        tokenizer=processor.tokenizer,
-        callbacks=[EfficiencyCallback()],
+        processing_class=processor,
     )
     trainer.train()
 
 
 if __name__ == "__main__":
-    train()
+    train()

{liger_kernel_nightly-0.5.3.dev20250221011057 → liger_kernel_nightly-0.5.3.dev20250221011147}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "liger_kernel_nightly"
-version = "0.5.3.dev20250221011057"
+version = "0.5.3.dev20250221011147"
 description = "Efficient Triton kernels for LLM Training"
 urls = { "Homepage" = "https://github.com/linkedin/Liger-Kernel" }
 readme = { file = "README.md", content-type = "text/markdown" }

{liger_kernel_nightly-0.5.3.dev20250221011057 → liger_kernel_nightly-0.5.3.dev20250221011147}/src/liger_kernel_nightly.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: liger_kernel_nightly
-Version: 0.5.3.dev20250221011057
+Version: 0.5.3.dev20250221011147
 Summary: Efficient Triton kernels for LLM Training
 License: BSD 2-CLAUSE LICENSE
         Copyright 2024 LinkedIn Corporation