project-llm-trainer 0.8.3__py3-none-any.whl → 0.9.1__py3-none-any.whl

llm_trainer/generate_utils.py

@@ -1,5 +1,4 @@
 from typing import Union, Optional, List
-from contextlib import nullcontext
 import torch
 from llm_model import VlmModel, KVCache
 from .tools import TrainerTools

llm_trainer/train_configs.py

@@ -123,7 +123,7 @@ class OptimConfig:
 class LossConfig:
     critical_tokens: Optional[List[int]] = None
     critical_alpha: float = 1.0
-    aux_loss_coef: Optional[float] = 1.0
+    aux_loss_coef: Optional[float] = 0.001
 
 
 @dataclass(kw_only=True)

llm_trainer/trainer.py

@@ -1,5 +1,7 @@
 from typing import Optional, Tuple, List, Dict, Any
 import copy
+import importlib.metadata
+from packaging import version
 
 import torch
 import torch.distributed as dist
@@ -168,12 +170,14 @@ class Trainer:
             if ('zero_optimization' in self.parallel_kwargs
                     and 'offload_optimizer' in self.parallel_kwargs['zero_optimization']
                     and self.parallel_kwargs['zero_optimization']['offload_optimizer']['device'] == 'cpu'):
-                # offline optimizer to cpu
-                # 不能使用 deepspeed.ops.lion.cpu_lion.DeepSpeedCPULion???
-                # 所以，这里忽略lion判断
-                optimizer = deepspeed.ops.adam.DeepSpeedCPUAdam
                 if self.train_config.optim_config.optim_type == 'lion':
-                    log('When set offload_optimizer, lion optim is unsupported, so set optim to adam!!!!!')
+                    if version.parse(importlib.metadata.version("deepspeed")) >= version.parse('0.17.6'):
+                        optimizer = deepspeed.ops.lion.DeepSpeedCPULion
+                    else:
+                        optimizer = deepspeed.ops.adam.DeepSpeedCPUAdam
+                        log('When set offload_optimizer, lion optim is unsupported, so set optim to adam!!!!!')
+                else:
+                    optimizer = deepspeed.ops.adam.DeepSpeedCPUAdam
             else:
                 if self.train_config.optim_config.optim_type == 'lion':
                     optimizer = deepspeed.ops.lion.FusedLion

{project_llm_trainer-0.8.3.data → project_llm_trainer-0.9.1.data}/scripts/plot_loss

@@ -35,5 +35,3 @@ if __name__ == '__main__':
 
     plt.plot(steps, losses)
     plt.show()
-
-

{project_llm_trainer-0.8.3.dist-info → project_llm_trainer-0.9.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: project_llm_trainer
-Version: 0.8.3
+Version: 0.9.1
 Summary: LLM and VLM trainer
 Author: qibin
 Author-email: qibin0506@gmail.com

{project_llm_trainer-0.8.3.dist-info → project_llm_trainer-0.9.1.dist-info}/RECORD

@@ -4,7 +4,7 @@ llm_trainer/dataset.py,sha256=UL3fGeM4XSlyNQRZH-139u3LujqAQx3YyaxNRewk6LE,8935
 llm_trainer/dpo_trainer.py,sha256=Qi7WKhFO4fdnj9W8BNIF_so6-F8g_YKUoPU9sNjWK_M,13320
 llm_trainer/ds_checkpoint.py,sha256=X2IWgpgi0yOtogph7n6DEwvK_0Ceb7juu1WMutv3HSk,2270
 llm_trainer/eval.py,sha256=ZyUfSo2Q8P-lrCdPEnGkoo5pGubd0AabREK5eMISRII,1109
-llm_trainer/generate_utils.py,sha256=8K3YFbp7IF_lCkmkzjHhqTW26EBFb2AilQmarVcfMvs,15001
+llm_trainer/generate_utils.py,sha256=9-F29aWKDnAnnDCU_D8uKUkVcM2r6xktwyqV4nlggTg,14966
 llm_trainer/grpo_trainer.py,sha256=3CcV-cuyV4ZUTymN9vz3au4uf3gZdyo8SGgSj2NEofs,16774
 llm_trainer/log.py,sha256=XwychwKF6gvFPhthCIZCAEUZ0G3DY3fiQrOHqPWsxz0,463
 llm_trainer/loss.py,sha256=RhTxftLMj1Tqc5pkUvJiZumfbMEPWL8GBGxdTfQggmk,6744
@@ -17,17 +17,17 @@ llm_trainer/scheduler.py,sha256=LAI_0VxClsIQkix0bRoduRD4vPfVuIZDhZgTAT_KK8k,4901
 llm_trainer/sft_trainer.py,sha256=rSOGZx53jMgOuJdztfxQASYJ62uD0dVaih4IAnSwGBc,1787
 llm_trainer/tokenizer.py,sha256=0-xQCMz1xiPTDAZiYsVsiECSoZ_1eIvW9XsZOoFfakQ,7250
 llm_trainer/tools.py,sha256=5op5qrjjkK-Lr9oes5VxIVnOVYOYGoAdlIJq9mPUf64,2637
-llm_trainer/train_configs.py,sha256=pPZkbliRdTnWSv3TUuTM23x9RDdMhGSPrxbNAyzDklY,7636
-llm_trainer/trainer.py,sha256=sz350VaLR7odGm0vYGTGvFWcr5bC7tmwMx7PUY6LA2o,28980
+llm_trainer/train_configs.py,sha256=u3b3RhZn7S6vm2DKNF5sNRHzp73-eYnActGSlzcIbzs,7638
+llm_trainer/trainer.py,sha256=Wf1JLWYhmxKKef49asxkaR4ZocKO-jlupxksv-BNfZI,29170
 llm_trainer/utils.py,sha256=xC5plG-8-_Al5yIF5xIU5lroOcBBk98TEhtUJrazZPE,12305
-project_llm_trainer-0.8.3.data/scripts/calc_intermediate_size,sha256=AggpgNHokJiJMbEtVdOnolqr_4bH3i1UYuZNEAzC2Gc,460
-project_llm_trainer-0.8.3.data/scripts/ddp_train,sha256=eZSud6KYQAoKLsYB5QB-FI2zq5AZm6Apq1azKdupV3o,477
-project_llm_trainer-0.8.3.data/scripts/ds_train,sha256=41q4rOxwbvZDUY0FDdAIpG13PEaUWBpthhvFvww8uOc,388
-project_llm_trainer-0.8.3.data/scripts/plot_loss,sha256=MzFcdJESlVr1srj4Td6-AxPGUKkfB_QEcJwm0Bd-5fU,910
-project_llm_trainer-0.8.3.data/scripts/plot_lr,sha256=w_7XR_x3KYYyboeOVAeu_I4fveLFI-C0wBmRrNlmWUI,894
-project_llm_trainer-0.8.3.data/scripts/py_train,sha256=tOp9TquORQeU8XN5H7OVIk5O0Ypwi34p_GENxTwgwdk,265
-project_llm_trainer-0.8.3.data/scripts/smart_train,sha256=N8dp2n7k6bghGczedBVwOdtf1O66oM_cNPh9QmZt0bM,914
-project_llm_trainer-0.8.3.dist-info/METADATA,sha256=WxVWmIDlcVdlNbOIglZo7asnIQDmfvobKEFiloTvjrs,195
-project_llm_trainer-0.8.3.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
-project_llm_trainer-0.8.3.dist-info/top_level.txt,sha256=LtRFg28i0QIG7iBCD2t095oSco99LCtkijibS9cMGik,12
-project_llm_trainer-0.8.3.dist-info/RECORD,,
+project_llm_trainer-0.9.1.data/scripts/calc_intermediate_size,sha256=AggpgNHokJiJMbEtVdOnolqr_4bH3i1UYuZNEAzC2Gc,460
+project_llm_trainer-0.9.1.data/scripts/ddp_train,sha256=eZSud6KYQAoKLsYB5QB-FI2zq5AZm6Apq1azKdupV3o,477
+project_llm_trainer-0.9.1.data/scripts/ds_train,sha256=41q4rOxwbvZDUY0FDdAIpG13PEaUWBpthhvFvww8uOc,388
+project_llm_trainer-0.9.1.data/scripts/plot_loss,sha256=O9ooioAJ-79-X06LosgqF8XOqQe-beRxYm3LsLunmoU,908
+project_llm_trainer-0.9.1.data/scripts/plot_lr,sha256=w_7XR_x3KYYyboeOVAeu_I4fveLFI-C0wBmRrNlmWUI,894
+project_llm_trainer-0.9.1.data/scripts/py_train,sha256=tOp9TquORQeU8XN5H7OVIk5O0Ypwi34p_GENxTwgwdk,265
+project_llm_trainer-0.9.1.data/scripts/smart_train,sha256=N8dp2n7k6bghGczedBVwOdtf1O66oM_cNPh9QmZt0bM,914
+project_llm_trainer-0.9.1.dist-info/METADATA,sha256=1Rtg7wi3BLy-9cFnNouWF3bVkexiMHeFSyaSmvrOFpA,195
+project_llm_trainer-0.9.1.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
+project_llm_trainer-0.9.1.dist-info/top_level.txt,sha256=LtRFg28i0QIG7iBCD2t095oSco99LCtkijibS9cMGik,12
+project_llm_trainer-0.9.1.dist-info/RECORD,,