liger-kernel-nightly 0.4.1.dev20241112233904__py3-none-any.whl → 0.4.1.dev20241114041219__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of liger-kernel-nightly might be problematic. Click here for more details.
- liger_kernel/chunked_loss/__init__.py +0 -0
- liger_kernel/chunked_loss/fused_linear_preference.py +107 -0
- liger_kernel/chunked_loss/orpo_loss.py +117 -0
- {liger_kernel_nightly-0.4.1.dev20241112233904.dist-info → liger_kernel_nightly-0.4.1.dev20241114041219.dist-info}/METADATA +14 -80
- {liger_kernel_nightly-0.4.1.dev20241112233904.dist-info → liger_kernel_nightly-0.4.1.dev20241114041219.dist-info}/RECORD +9 -6
- {liger_kernel_nightly-0.4.1.dev20241112233904.dist-info → liger_kernel_nightly-0.4.1.dev20241114041219.dist-info}/LICENSE +0 -0
- {liger_kernel_nightly-0.4.1.dev20241112233904.dist-info → liger_kernel_nightly-0.4.1.dev20241114041219.dist-info}/NOTICE +0 -0
- {liger_kernel_nightly-0.4.1.dev20241112233904.dist-info → liger_kernel_nightly-0.4.1.dev20241114041219.dist-info}/WHEEL +0 -0
- {liger_kernel_nightly-0.4.1.dev20241112233904.dist-info → liger_kernel_nightly-0.4.1.dev20241114041219.dist-info}/top_level.txt +0 -0
|
File without changes
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import torch
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class LigerFusedLinearPreferenceBase(torch.autograd.Function):
|
|
5
|
+
@staticmethod
|
|
6
|
+
def forward(
|
|
7
|
+
ctx,
|
|
8
|
+
_input,
|
|
9
|
+
weight,
|
|
10
|
+
target,
|
|
11
|
+
bias=None,
|
|
12
|
+
loss_fn=None,
|
|
13
|
+
chunk_size=1,
|
|
14
|
+
compiled=True,
|
|
15
|
+
):
|
|
16
|
+
"""
|
|
17
|
+
Base class for fused linear layer with preference loss.
|
|
18
|
+
Expects _input to be stacked with chosen and rejected inputs on the batch dimension.
|
|
19
|
+
|
|
20
|
+
Args:
|
|
21
|
+
_input (torch.Tensor): Input tensor. Shape: (batch_size, seq_len, hidden_size).
|
|
22
|
+
weight (torch.Tensor): Weight tensor. Shape: (vocab_size, hidden_size).
|
|
23
|
+
target (torch.Tensor): Target tensor. Shape: (batch_size, seq_len).
|
|
24
|
+
bias (torch.Tensor, optional): Bias tensor. Shape: (vocab_size,).
|
|
25
|
+
loss_fn (callable): Loss function to compute the loss on a chunk of input/target.
|
|
26
|
+
chunk_size (int): Size of a chunk (# of batches of stacked chosen and rejected inputs).
|
|
27
|
+
compiled (bool): Whether to use torch compile for chunk accumulation.
|
|
28
|
+
"""
|
|
29
|
+
# TODO: Tune CHUNK_SIZE to fully utilize the GPU
|
|
30
|
+
CHUNK_SIZE = chunk_size
|
|
31
|
+
|
|
32
|
+
grad_weight = torch.zeros_like(weight)
|
|
33
|
+
grad_chosen_inputs = []
|
|
34
|
+
grad_rejected_inputs = []
|
|
35
|
+
grad_bias = torch.zeros_like(bias) if bias is not None else None
|
|
36
|
+
loss_acc = torch.zeros((), device=_input.device)
|
|
37
|
+
|
|
38
|
+
chunks = max(1, _input.shape[0] // (2 * CHUNK_SIZE))
|
|
39
|
+
|
|
40
|
+
def accumulate_chunk(input_chunk, target_chunk):
|
|
41
|
+
if bias is not None:
|
|
42
|
+
(chunk_grad_input, chunk_grad_weight, chunk_grad_bias), (
|
|
43
|
+
chunk_loss,
|
|
44
|
+
(chunk_or_loss, chunk_chosen_logps, chunk_rejected_logps),
|
|
45
|
+
) = torch.func.grad_and_value(loss_fn, argnums=(0, 1, 3), has_aux=True)(
|
|
46
|
+
input_chunk, weight, target_chunk, bias
|
|
47
|
+
)
|
|
48
|
+
grad_bias.add_(chunk_grad_bias)
|
|
49
|
+
else:
|
|
50
|
+
(chunk_grad_input, chunk_grad_weight), (
|
|
51
|
+
chunk_loss,
|
|
52
|
+
(chunk_or_loss, chunk_chosen_logps, chunk_rejected_logps),
|
|
53
|
+
) = torch.func.grad_and_value(loss_fn, argnums=(0, 1), has_aux=True)(
|
|
54
|
+
input_chunk, weight, target_chunk
|
|
55
|
+
)
|
|
56
|
+
grad_weight.add_(chunk_grad_weight)
|
|
57
|
+
loss_acc.add_(chunk_loss)
|
|
58
|
+
return chunk_grad_input
|
|
59
|
+
|
|
60
|
+
len_chosen = target.shape[0] // 2
|
|
61
|
+
_chosen_input_chunks = torch.chunk(_input[:len_chosen], chunks=chunks, dim=0)
|
|
62
|
+
_chosen_target_chunks = torch.chunk(target[:len_chosen], chunks=chunks, dim=0)
|
|
63
|
+
_rejected_input_chunks = torch.chunk(_input[len_chosen:], chunks=chunks, dim=0)
|
|
64
|
+
_rejected_target_chunks = torch.chunk(target[len_chosen:], chunks=chunks, dim=0)
|
|
65
|
+
|
|
66
|
+
for (
|
|
67
|
+
chosen_input_chunk,
|
|
68
|
+
rejected_input_chunk,
|
|
69
|
+
chosen_target_chunk,
|
|
70
|
+
rejected_target_chunk,
|
|
71
|
+
) in zip(
|
|
72
|
+
_chosen_input_chunks,
|
|
73
|
+
_rejected_input_chunks,
|
|
74
|
+
_chosen_target_chunks,
|
|
75
|
+
_rejected_target_chunks,
|
|
76
|
+
):
|
|
77
|
+
input_chunk = torch.cat([chosen_input_chunk, rejected_input_chunk], dim=0)
|
|
78
|
+
target_chunk = torch.cat(
|
|
79
|
+
[chosen_target_chunk, rejected_target_chunk], dim=0
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
if compiled:
|
|
83
|
+
accumulate_chunk = torch.compile(accumulate_chunk)
|
|
84
|
+
grad_input = accumulate_chunk(input_chunk, target_chunk)
|
|
85
|
+
|
|
86
|
+
grad_chosen_inputs.append(grad_input[: chosen_target_chunk.shape[0]])
|
|
87
|
+
grad_rejected_inputs.append(grad_input[chosen_target_chunk.shape[0] :])
|
|
88
|
+
|
|
89
|
+
# combine grad_chosen_inputs and grad_rejected_inputs
|
|
90
|
+
grad_inputs = grad_chosen_inputs + grad_rejected_inputs
|
|
91
|
+
|
|
92
|
+
ctx.save_for_backward(
|
|
93
|
+
torch.cat(grad_inputs, dim=0),
|
|
94
|
+
grad_weight,
|
|
95
|
+
grad_bias,
|
|
96
|
+
)
|
|
97
|
+
return loss_acc
|
|
98
|
+
|
|
99
|
+
@staticmethod
|
|
100
|
+
def backward(ctx, grad_output):
|
|
101
|
+
grad_input, grad_weight, grad_bias = ctx.saved_tensors
|
|
102
|
+
if torch.ne(grad_output, torch.tensor(1.0, device=grad_output.device)):
|
|
103
|
+
grad_input = grad_input * grad_output
|
|
104
|
+
grad_weight = grad_weight * grad_output
|
|
105
|
+
grad_bias = grad_bias * grad_output if grad_bias is not None else None
|
|
106
|
+
|
|
107
|
+
return grad_input, grad_weight, None, grad_bias, None, None, None
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
from functools import partial
|
|
2
|
+
|
|
3
|
+
import torch
|
|
4
|
+
import torch.nn.functional as F
|
|
5
|
+
|
|
6
|
+
from liger_kernel.chunked_loss.fused_linear_preference import (
|
|
7
|
+
LigerFusedLinearPreferenceBase,
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def odds_ratio_loss(chosen_logps, rejected_logps, beta=0.1):
|
|
12
|
+
"""
|
|
13
|
+
Compute odds-ratio loss.
|
|
14
|
+
Args:
|
|
15
|
+
chosen_logps (torch.Tensor): Avg log probabilities of chosen tokens. Shape: (batch_size,).
|
|
16
|
+
rejected_logps (torch.Tensor): Avg log probabilities of rejected tokens. Shape: (batch_size,).
|
|
17
|
+
beta (float): Weight for the odds ratio loss.
|
|
18
|
+
"""
|
|
19
|
+
log_odds = (chosen_logps - rejected_logps) - (
|
|
20
|
+
torch.log1p(-torch.exp(chosen_logps)) - torch.log1p(-torch.exp(rejected_logps))
|
|
21
|
+
)
|
|
22
|
+
ratio = F.logsigmoid(log_odds)
|
|
23
|
+
return beta * ratio.sum()
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _compute_orpo_loss(
|
|
27
|
+
input_chunk,
|
|
28
|
+
weight,
|
|
29
|
+
target_chunk,
|
|
30
|
+
bias=None,
|
|
31
|
+
full_target=None,
|
|
32
|
+
ignore_index=-100,
|
|
33
|
+
beta=0.1,
|
|
34
|
+
compute_nll_loss=True,
|
|
35
|
+
):
|
|
36
|
+
"""
|
|
37
|
+
Compute ORPO loss for a chunk of input and target.
|
|
38
|
+
Args:
|
|
39
|
+
input_chunk (torch.Tensor): Chunk of input tensor. Shape: (2 * chunk_size, sequence_length, hidden_size).
|
|
40
|
+
weight (torch.Tensor): Weight tensor. Shape: (vocab_size, hidden_size).
|
|
41
|
+
target_chunk (torch.Tensor): Chunk of target tensor. Shape: (2 * chunk_size, sequence_length).
|
|
42
|
+
bias (torch.Tensor, optional): Bias tensor. Shape: (vocab_size,).
|
|
43
|
+
full_target (torch.Tensor): Full target tensor. Shape: (batch_size, sequence_length).
|
|
44
|
+
ignore_index (int): Index to ignore for loss computation.
|
|
45
|
+
beta (float): Weight for the odds ratio loss.
|
|
46
|
+
"""
|
|
47
|
+
len_chosen_chunk = target_chunk.shape[0] // 2
|
|
48
|
+
|
|
49
|
+
logits_chunk = input_chunk @ weight.t() # chunk_size x V
|
|
50
|
+
if bias is not None:
|
|
51
|
+
logits_chunk = logits_chunk + bias
|
|
52
|
+
log_probs_chunk = F.log_softmax(logits_chunk.float(), dim=-1)
|
|
53
|
+
|
|
54
|
+
chosen_nll_loss = 0.0
|
|
55
|
+
if compute_nll_loss:
|
|
56
|
+
chosen_nll_loss = F.nll_loss(
|
|
57
|
+
log_probs_chunk[:len_chosen_chunk].view(-1, log_probs_chunk.shape[-1]),
|
|
58
|
+
target_chunk[:len_chosen_chunk].view(-1),
|
|
59
|
+
reduction="sum",
|
|
60
|
+
ignore_index=ignore_index,
|
|
61
|
+
)
|
|
62
|
+
chosen_nll_loss = (
|
|
63
|
+
chosen_nll_loss
|
|
64
|
+
/ (full_target[: full_target.shape[0] // 2] != ignore_index).sum()
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
loss_mask = target_chunk != ignore_index
|
|
68
|
+
label_chunk = torch.where(loss_mask, target_chunk, 0)
|
|
69
|
+
|
|
70
|
+
per_token_logps = log_probs_chunk.gather(-1, label_chunk.unsqueeze(-1)).squeeze(-1)
|
|
71
|
+
average_log_prob = (per_token_logps * loss_mask).sum(-1) / loss_mask.sum(-1)
|
|
72
|
+
|
|
73
|
+
chosen_logps = average_log_prob[:len_chosen_chunk]
|
|
74
|
+
rejected_logps = average_log_prob[len_chosen_chunk:]
|
|
75
|
+
|
|
76
|
+
or_loss = odds_ratio_loss(chosen_logps, rejected_logps, beta=beta)
|
|
77
|
+
or_loss = or_loss / (full_target.shape[0] // 2)
|
|
78
|
+
|
|
79
|
+
loss = chosen_nll_loss - or_loss
|
|
80
|
+
return loss, (or_loss, chosen_logps, rejected_logps)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class LigerFusedLinearORPOFunction(LigerFusedLinearPreferenceBase):
|
|
84
|
+
@staticmethod
|
|
85
|
+
def forward(
|
|
86
|
+
ctx,
|
|
87
|
+
_input,
|
|
88
|
+
weight,
|
|
89
|
+
target,
|
|
90
|
+
bias=None,
|
|
91
|
+
ignore_index=-100,
|
|
92
|
+
beta=0.1,
|
|
93
|
+
compute_nll_loss=True,
|
|
94
|
+
compiled=True,
|
|
95
|
+
):
|
|
96
|
+
"""
|
|
97
|
+
Fused linear layer with ORPO (Odds-Ratio Preference Optimization) loss.
|
|
98
|
+
Handles both the forward and backward pass of the final linear layer with ORPO loss.
|
|
99
|
+
Inspired from LigerFusedLinearCrossEntropyFunction (https://arxiv.org/abs/2410.10989) which fuses final linear layer and CE loss.
|
|
100
|
+
"""
|
|
101
|
+
orpo_loss_fn = partial(
|
|
102
|
+
_compute_orpo_loss,
|
|
103
|
+
full_target=target,
|
|
104
|
+
ignore_index=ignore_index,
|
|
105
|
+
beta=beta,
|
|
106
|
+
compute_nll_loss=compute_nll_loss,
|
|
107
|
+
)
|
|
108
|
+
return LigerFusedLinearPreferenceBase.forward(
|
|
109
|
+
ctx, _input, weight, target, bias, loss_fn=orpo_loss_fn
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
@staticmethod
|
|
113
|
+
def backward(ctx, grad_output):
|
|
114
|
+
# Get gradients for _input, weight, bias, and target from the base class
|
|
115
|
+
grads = LigerFusedLinearPreferenceBase.backward(ctx, grad_output)[:4]
|
|
116
|
+
# Return these gradients, followed by None for the remaining inputs
|
|
117
|
+
return *grads, None, None, None, None
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: liger_kernel_nightly
|
|
3
|
-
Version: 0.4.1.
|
|
3
|
+
Version: 0.4.1.dev20241114041219
|
|
4
4
|
Summary: Efficient Triton kernels for LLM Training
|
|
5
5
|
License: BSD 2-CLAUSE LICENSE
|
|
6
6
|
Copyright 2024 LinkedIn Corporation
|
|
@@ -99,7 +99,8 @@ Requires-Dist: transformers~=4.0; extra == "transformers"
|
|
|
99
99
|
|
|
100
100
|
<details>
|
|
101
101
|
<summary>Latest News 🔥</summary>
|
|
102
|
-
|
|
102
|
+
|
|
103
|
+
- [2024/11/6] We release [v0.4.0](https://github.com/linkedin/Liger-Kernel/releases/tag/v0.4.0): Full AMD support, Tech Report, Modal CI, Llama-3.2-Vision!
|
|
103
104
|
- [2024/10/21] We have released the tech report of Liger Kernel on Arxiv: https://arxiv.org/pdf/2410.10989
|
|
104
105
|
- [2024/9/6] We release v0.2.1 ([X post](https://x.com/liger_kernel/status/1832168197002510649)). 2500+ Stars, 10+ New Contributors, 50+ PRs, 50k Downloads in two weeks!
|
|
105
106
|
- [2024/8/31] CUDA MODE talk, [Liger-Kernel: Real-world Triton kernel for LLM Training](https://youtu.be/gWble4FreV4?si=dxPeIchhkJ36Mbns), [Slides](https://github.com/cuda-mode/lectures?tab=readme-ov-file#lecture-28-liger-kernel)
|
|
@@ -127,18 +128,12 @@ With one line of code, Liger Kernel can increase throughput by more than 20% and
|
|
|
127
128
|
|
|
128
129
|
## Examples
|
|
129
130
|
|
|
130
|
-
### Basic
|
|
131
|
-
|
|
132
|
-
| **Example** | **Description** | **Lightning Studio** |
|
|
133
|
-
|------------------------------------------------|---------------------------------------------------------------------------------------------------|----------------------|
|
|
134
|
-
| [**Hugging Face Trainer**](https://github.com/linkedin/Liger-Kernel/tree/main/examples/huggingface) | Train LLaMA 3-8B ~20% faster with over 40% memory reduction on Alpaca dataset using 4 A100s with FSDP | TBA |
|
|
135
|
-
| [**Lightning Trainer**](https://github.com/linkedin/Liger-Kernel/tree/main/examples/lightning) | Increase 15% throughput and reduce memory usage by 40% with LLaMA3-8B on MMLU dataset using 8 A100s with DeepSpeed ZeRO3 | TBA |
|
|
136
131
|
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
| **
|
|
140
|
-
|
|
141
|
-
| [**Medusa Multi-head LLM (Retraining Phase)**](https://github.com/linkedin/Liger-Kernel/tree/main/examples/medusa) | Reduce memory usage by 80% with 5 LM heads and improve throughput by 40% using 8 A100s with FSDP |
|
|
132
|
+
| **Use Case** | **Description** |
|
|
133
|
+
|------------------------------------------------|---------------------------------------------------------------------------------------------------|
|
|
134
|
+
| [**Hugging Face Trainer**](https://github.com/linkedin/Liger-Kernel/tree/main/examples/huggingface) | Train LLaMA 3-8B ~20% faster with over 40% memory reduction on Alpaca dataset using 4 A100s with FSDP |
|
|
135
|
+
| [**Lightning Trainer**](https://github.com/linkedin/Liger-Kernel/tree/main/examples/lightning) | Increase 15% throughput and reduce memory usage by 40% with LLaMA3-8B on MMLU dataset using 8 A100s with DeepSpeed ZeRO3 |
|
|
136
|
+
| [**Medusa Multi-head LLM (Retraining Phase)**](https://github.com/linkedin/Liger-Kernel/tree/main/examples/medusa) | Reduce memory usage by 80% with 5 LM heads and improve throughput by 40% using 8 A100s with FSDP | |
|
|
142
137
|
|
|
143
138
|
## Key Features
|
|
144
139
|
|
|
@@ -149,13 +144,6 @@ With one line of code, Liger Kernel can increase throughput by more than 20% and
|
|
|
149
144
|
- **Multi-GPU supported:** Compatible with multi-GPU setups (PyTorch FSDP, DeepSpeed, DDP, etc.).
|
|
150
145
|
- **Trainer Framework Integration**: [Axolotl](https://github.com/axolotl-ai-cloud/axolotl), [LLaMa-Factory](https://github.com/hiyouga/LLaMA-Factory), [SFTTrainer](https://github.com/huggingface/trl/releases/tag/v0.10.1), [Hugging Face Trainer](https://github.com/huggingface/transformers/pull/32860), [SWIFT](https://github.com/modelscope/ms-swift)
|
|
151
146
|
|
|
152
|
-
## Target Audiences
|
|
153
|
-
|
|
154
|
-
- **Researchers**: Looking to compose models using efficient and reliable kernels for frontier experiments.
|
|
155
|
-
- **ML Practitioners**: Focused on maximizing GPU training efficiency with optimal, high-performance kernels.
|
|
156
|
-
- **Curious Novices**: Eager to learn how to write reliable Triton kernels to enhance training efficiency.
|
|
157
|
-
|
|
158
|
-
|
|
159
147
|
## Installation
|
|
160
148
|
|
|
161
149
|
### Dependencies
|
|
@@ -261,23 +249,6 @@ loss = loss_fn(model.weight, input, target)
|
|
|
261
249
|
loss.backward()
|
|
262
250
|
```
|
|
263
251
|
|
|
264
|
-
|
|
265
|
-
## Structure
|
|
266
|
-
|
|
267
|
-
### Source Code
|
|
268
|
-
|
|
269
|
-
- `ops/`: Core Triton operations.
|
|
270
|
-
- `transformers/`: PyTorch `nn.Module` implementations built on Triton operations, compliant with the `transformers` API.
|
|
271
|
-
|
|
272
|
-
### Tests
|
|
273
|
-
|
|
274
|
-
- `transformers/`: Correctness tests for the Triton-based layers.
|
|
275
|
-
- `convergence/`: Patches Hugging Face models with all kernels, runs multiple iterations, and compares weights, logits, and loss layer-by-layer.
|
|
276
|
-
|
|
277
|
-
### Benchmark
|
|
278
|
-
|
|
279
|
-
- `benchmark/`: Execution time and memory benchmarks compared to Hugging Face layers.
|
|
280
|
-
|
|
281
252
|
## APIs
|
|
282
253
|
|
|
283
254
|
### AutoModel
|
|
@@ -346,54 +317,17 @@ $$\text{GeGLU}(x)=\text{GELU}(xW+b)\otimes(xV+c)$$
|
|
|
346
317
|
- **Embedding**: [Embedding](https://pytorch.org/docs/stable/generated/torch.nn.Embedding.html) is implemented by fusing embedding lookup and output operations. It achieves a peak speedup of ~1.5x in the forward pass and an overall speedup of ~1.1x.
|
|
347
318
|
- **Matmul int2xint8**: is implemented by using the cache tiled matrix multiplication and by fusing the matmul with the unpacking process which achieves a considerable speed up and performs on par with @torch.compile
|
|
348
319
|
<!-- TODO: be more specific about batch size -->
|
|
349
|
-
> **Note:**
|
|
350
|
-
> Reported speedups and memory reductions are with respect to the LLaMA 3-8B Hugging Face layer implementations. All models use 4K hidden size and 4K sequence length and are evaluated based on memory usage and wall time for the forward+backward pass on a single NVIDIA A100 80G GPU using small batch sizes. Liger kernels exhibit more efficient scaling to larger batch sizes, detailed further in the [Benchmark](./benchmark) folder.
|
|
351
|
-
|
|
352
|
-
## Contributing
|
|
353
|
-
|
|
354
|
-
[CONTRIBUTING GUIDE](https://github.com/linkedin/Liger-Kernel/blob/main/CONTRIBUTING.md)
|
|
355
|
-
|
|
356
|
-
## Acknowledgement
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
### Design
|
|
360
|
-
|
|
361
|
-
- [@claire_yishan](https://twitter.com/claire_yishan) for the LOGO design
|
|
362
|
-
- [Wave Snippets](https://www.wavesnippets.com/) for generating the animated code snippets
|
|
363
|
-
|
|
364
|
-
### Code
|
|
365
|
-
|
|
366
|
-
We referenced or used the following projects:
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
| # | Project | Description | Location | License |
|
|
371
|
-
|---|----------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------|
|
|
372
|
-
| 1 | [Unsloth](https://github.com/unslothai/unsloth/blob/fd753fed99ed5f10ef8a9b7139588d9de9ddecfb/unsloth/kernels/utils.py#L43) | `calculate_settings` to determine block size and warp; We reuse it for Norm and MLP | [Liger Kernel Utils](https://github.com/linkedin/Liger-Kernel/blob/e249eee723978bf8610ff1ea2297d048a2417e20/src/liger_kernel/ops/utils.py#L23) | [Apache](https://github.com/unslothai/unsloth/blob/fd753fed99ed5f10ef8a9b7139588d9de9ddecfb/LICENSE) |
|
|
373
|
-
| 2 | [Unsloth](https://github.com/unslothai/unsloth/blob/976d11a10d54383aeb7a692c69e01151a20bfd72/unsloth/kernels/rms_layernorm.py#L48) | We modified and added dW calculation on top of Unsloth implementation | [Liger Kernel RMS Norm](https://github.com/linkedin/Liger-Kernel/blob/e249eee723978bf8610ff1ea2297d048a2417e20/src/liger_kernel/ops/rms_norm.py#L50) | [Apache](https://github.com/unslothai/unsloth/blob/fd753fed99ed5f10ef8a9b7139588d9de9ddecfb/LICENSE) |
|
|
374
|
-
| 3 | [Triton tutorial](https://triton-lang.org/main/index.html) | We modified on top of triton tutorials | [Liger Kernel RMS Norm](https://github.com/linkedin/Liger-Kernel/blob/e249eee723978bf8610ff1ea2297d048a2417e20/src/liger_kernel/ops/rms_norm.py#L50) | [MIT](https://github.com/triton-lang/triton/blob/main/LICENSE) |
|
|
375
|
-
| 4 | [tiny shakespeare dataset](https://huggingface.co/datasets/karpathy/tiny_shakespeare) | We use tiny shakespeare dataset to conduct convergence test on mini model | [Liger Kernel Convergence](https://github.com/linkedin/Liger-Kernel/tree/main/test/convergence) | N/A |
|
|
376
|
-
| 5 | [Efficient Cross Entropy](https://github.com/mgmalek/efficient_cross_entropy) | We use the idea of gradient-in-forward and chunking | [Liger Kernel Linear Cross Entropy](https://github.com/linkedin/Liger-Kernel/blob/main/src/liger_kernel/ops/fused_linear_cross_entropy.py) | [MIT](https://github.com/mgmalek/efficient_cross_entropy/blob/main/LICENSE) |
|
|
377
|
-
| 6 | [Flash attn](https://github.com/Dao-AILab/flash-attention) | We take many optimization ideas from the work, such as tiling and recomputation | | [BSD](https://github.com/Dao-AILab/flash-attention/blob/main/LICENSE) |
|
|
378
|
-
| 7 | [AutoAWQ](https://github.com/casper-hansen/AutoAWQ) | We reference the design of automodel | [Liger Kernel Auto Model](https://github.com/linkedin/Liger-Kernel/blob/main/src/liger_kernel/transformers/auto_model.py) | [MIT](https://github.com/casper-hansen/AutoAWQ/blob/main/LICENSE) |
|
|
379
|
-
| 8 | [llm.c](https://github.com/karpathy/llm.c) | We reference the design of end-to-end testing | [Liger Kernel Convergence Tests](https://github.com/linkedin/Liger-Kernel/tree/main/test/convergence) | [MIT](https://github.com/karpathy/llm.c/blob/master/LICENSE) |
|
|
380
|
-
|
|
381
|
-
Many thanks to the contributors to these projects for their invaluable work that helped make Liger possible.
|
|
382
|
-
|
|
383
|
-
## License
|
|
384
320
|
|
|
385
|
-
|
|
386
|
-
It also includes components from projects licensed under:
|
|
321
|
+
## Contributing, Acknowledgements, and License
|
|
387
322
|
|
|
388
|
-
-
|
|
389
|
-
-
|
|
390
|
-
-
|
|
391
|
-
- MIT License (see `LICENSE-MIT-llmc` for details).
|
|
392
|
-
- MIT License (see `LICENSE-MIT-triton` for details).
|
|
323
|
+
- [Contributing Guidelines](https://github.com/linkedin/Liger-Kernel/blob/main/docs/CONTRIBUTING.md)
|
|
324
|
+
- [Acknowledgements](https://github.com/linkedin/Liger-Kernel/blob/main/docs/Acknowledgement.md)
|
|
325
|
+
- [License Information](https://github.com/linkedin/Liger-Kernel/blob/main/docs/License.md)
|
|
393
326
|
|
|
394
327
|
## Contact
|
|
395
328
|
|
|
396
|
-
- For
|
|
329
|
+
- For issues, create a Github ticket in this repository
|
|
330
|
+
- For open discussion, join [our discord channel](https://discord.gg/gpumode)
|
|
397
331
|
- For formal collaboration, send an email to byhsu@linkedin.com
|
|
398
332
|
|
|
399
333
|
## Cite this work
|
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
liger_kernel/env_report.py,sha256=jye8RvUkmhqaIshdeIpoUABoAu7FPKJUib4FnAfvkpw,1132
|
|
2
|
+
liger_kernel/chunked_loss/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
+
liger_kernel/chunked_loss/fused_linear_preference.py,sha256=WYsIW3t_DEPeC2VkKACimMI0LhZwOciLluDhKnaMmWE,4416
|
|
4
|
+
liger_kernel/chunked_loss/orpo_loss.py,sha256=HgiWjS1ueVpi63ZGVveqwXkXY7EePfSzfbW-bbZlDxE,4220
|
|
2
5
|
liger_kernel/ops/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
6
|
liger_kernel/ops/cross_entropy.py,sha256=sfUb7-jIZp0EKXjg1DYy2Wdzw_Mg-mHmGoR5bpdm4tw,15526
|
|
4
7
|
liger_kernel/ops/fused_linear_cross_entropy.py,sha256=JPiQ0TgPjtQ-3F5ovC0b5ZnBk067XUmzyNuGO3KZv44,9963
|
|
@@ -43,9 +46,9 @@ liger_kernel/transformers/model/qwen2.py,sha256=EyhSSzQOskGjSnCsKMZpd1s5IAIlHd5P
|
|
|
43
46
|
liger_kernel/transformers/model/qwen2_vl.py,sha256=j6xAhp9AG195dsZK5f8dFYVM9uKtWApZrggT5Y08jn4,7055
|
|
44
47
|
liger_kernel/triton/__init__.py,sha256=yfRe0zMb47QnqjecZWG7LnanfCTzeku7SgWRAwNVmzU,101
|
|
45
48
|
liger_kernel/triton/monkey_patch.py,sha256=5BcGKTtdqeYchypBIBopGIWPx1-cFALz7sOKoEsqXJ0,1584
|
|
46
|
-
liger_kernel_nightly-0.4.1.
|
|
47
|
-
liger_kernel_nightly-0.4.1.
|
|
48
|
-
liger_kernel_nightly-0.4.1.
|
|
49
|
-
liger_kernel_nightly-0.4.1.
|
|
50
|
-
liger_kernel_nightly-0.4.1.
|
|
51
|
-
liger_kernel_nightly-0.4.1.
|
|
49
|
+
liger_kernel_nightly-0.4.1.dev20241114041219.dist-info/LICENSE,sha256=OhzLDHJ0to4a8sodVLELZiCFylZ1NAAYLs-HrjPy0ag,1312
|
|
50
|
+
liger_kernel_nightly-0.4.1.dev20241114041219.dist-info/METADATA,sha256=o4-CpcoMQuNzIpWwfEGDLui8edp-sWcFlXXZFDSHmlw,21556
|
|
51
|
+
liger_kernel_nightly-0.4.1.dev20241114041219.dist-info/NOTICE,sha256=njwnoPZLh9AN8SJQzxvCGLHi-8X__AvWRze6joNXIY8,2066
|
|
52
|
+
liger_kernel_nightly-0.4.1.dev20241114041219.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
|
|
53
|
+
liger_kernel_nightly-0.4.1.dev20241114041219.dist-info/top_level.txt,sha256=2eghu4hA3LnkM7ElW92tQ8zegWKgSbeo-k-aGe1YnvY,13
|
|
54
|
+
liger_kernel_nightly-0.4.1.dev20241114041219.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|