project-llm-trainer 0.8.1__py3-none-any.whl → 0.8.3__py3-none-any.whl

llm_trainer/trainer.py

@@ -383,15 +383,19 @@ class Trainer:
 
     def _calc_loss(self, inputs, attention_mask, logits, labels):
         # calc loss
-        loss = self.criterion(logits, labels)
+        if not self.kd_loss or self.train_config.kd_config.kd_coef == 0.0:
+            # 不用计算kd_loss
+            return self.criterion(logits, labels)
 
-        # 知识蒸馏loss
-        if self.kd_loss:
-            teacher_logits = self.train_config.kd_config.teacher_logits_provider(inputs, attention_mask)
-            distil_loss = self.kd_loss(logits, teacher_logits, labels)
-            loss = (1 - self.train_config.kd_config.kd_coef) * loss + self.train_config.kd_config.kd_coef * distil_loss
+        teacher_logits = self.train_config.kd_config.teacher_logits_provider(inputs, attention_mask)
+        loss = self.kd_loss(logits, teacher_logits, labels)
 
-        return loss
+        if self.train_config.kd_config.kd_coef == 1.0:
+            # 不用计算ce loss
+            return loss
+
+        ce_loss = self.criterion(logits, labels)
+        return (1 - self.train_config.kd_config.kd_coef) * ce_loss + self.train_config.kd_config.kd_coef * loss
 
     def _backward_loss(self, loss):
         if isinstance(TrainerTools().parallel, DsParallel):

project_llm_trainer-0.8.3.data/scripts/ddp_train

@@ -0,0 +1,21 @@
+#!python
+
+if __name__ == '__main__':
+    import os, sys
+    arguments = sys.argv[1:]
+    # file_name
+    run_file_name = arguments[0]
+
+    extra_args = ''
+    if len(arguments) > 1:
+        extra_args = f"{' '.join(arguments[1:])} "
+
+    os.environ['PARALLEL_TYPE'] = 'ddp'
+
+    if len(extra_args) == 0:
+        extra_args = '--standalone --nproc_per_node=gpu '
+
+    command = f'torchrun {extra_args}{run_file_name}'
+
+    print(f'run command {command}')
+    os.system(command)

project_llm_trainer-0.8.3.data/scripts/ds_train

@@ -0,0 +1,17 @@
+#!python
+
+if __name__ == '__main__':
+    import os, sys
+    arguments = sys.argv[1:]
+    # file_name
+    run_file_name = arguments[0]
+
+    extra_args = ''
+    if len(arguments) > 1:
+        extra_args = f"{' '.join(arguments[1:])} "
+
+    os.environ['PARALLEL_TYPE'] = 'ds'
+    command = f'deepspeed {extra_args}{run_file_name}'
+
+    print(f'run command {command}')
+    os.system(command)

{project_llm_trainer-0.8.1.data → project_llm_trainer-0.8.3.data}/scripts/smart_train

@@ -7,18 +7,9 @@ if __name__ == '__main__':
     # file name
     run_file_name = arguments[0]
 
-    # cuda_visible_devive
+    extra_args = ''
     if len(arguments) > 1:
-        # 0,1,2,3
-        cuda_visible_devive = arguments[1]
-    else:
-        cuda_visible_devive = None
-
-    # cuda location
-    if len(arguments) > 2:
-        cuda_loc = arguments[2]
-    else:
-        cuda_loc = 'localhost'
+        extra_args = f"{' '.join(arguments[1:])} "
 
     try:
         import deepspeed
@@ -33,12 +24,12 @@ if __name__ == '__main__':
     os.environ['PARALLEL_TYPE'] = parallel_type
 
     if parallel_type == 'ds':
-        cuda_ctrl = f' --include {cuda_loc}:{cuda_visible_devive}' if cuda_visible_devive else ''
-        command = f'deepspeed{cuda_ctrl} {run_file_name}'
+        command = f'deepspeed {extra_args}{run_file_name}'
     elif parallel_type == 'ddp':
-        if cuda_visible_devive:
-            os.environ['CUDA_VISIBLE_DEVICES'] = cuda_visible_devive
-        command = f'torchrun --standalone --nproc_per_node=gpu {run_file_name}'
+        if len(extra_args) == 0:
+            extra_args = '--standalone --nproc_per_node=gpu '
+
+        command = f'torchrun {extra_args}{run_file_name}'
     else:
         command = f'python3 {run_file_name}'
 

{project_llm_trainer-0.8.1.dist-info → project_llm_trainer-0.8.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: project_llm_trainer
-Version: 0.8.1
+Version: 0.8.3
 Summary: LLM and VLM trainer
 Author: qibin
 Author-email: qibin0506@gmail.com

{project_llm_trainer-0.8.1.dist-info → project_llm_trainer-0.8.3.dist-info}/RECORD

@@ -18,16 +18,16 @@ llm_trainer/sft_trainer.py,sha256=rSOGZx53jMgOuJdztfxQASYJ62uD0dVaih4IAnSwGBc,17
 llm_trainer/tokenizer.py,sha256=0-xQCMz1xiPTDAZiYsVsiECSoZ_1eIvW9XsZOoFfakQ,7250
 llm_trainer/tools.py,sha256=5op5qrjjkK-Lr9oes5VxIVnOVYOYGoAdlIJq9mPUf64,2637
 llm_trainer/train_configs.py,sha256=pPZkbliRdTnWSv3TUuTM23x9RDdMhGSPrxbNAyzDklY,7636
-llm_trainer/trainer.py,sha256=diP-1suOf2U5dY_R8QH5arAx4MgBrKW-GBQ2_ScGNM8,28799
+llm_trainer/trainer.py,sha256=sz350VaLR7odGm0vYGTGvFWcr5bC7tmwMx7PUY6LA2o,28980
 llm_trainer/utils.py,sha256=xC5plG-8-_Al5yIF5xIU5lroOcBBk98TEhtUJrazZPE,12305
-project_llm_trainer-0.8.1.data/scripts/calc_intermediate_size,sha256=AggpgNHokJiJMbEtVdOnolqr_4bH3i1UYuZNEAzC2Gc,460
-project_llm_trainer-0.8.1.data/scripts/ddp_train,sha256=Z-309mM56CN0m3bxoeC5us4LUuwuNnoiOm3-fDdLMjQ,566
-project_llm_trainer-0.8.1.data/scripts/ds_train,sha256=tME0xmMdX1D9XuVo07D9dilW5VIWavBS3UK9DoY67WI,709
-project_llm_trainer-0.8.1.data/scripts/plot_loss,sha256=MzFcdJESlVr1srj4Td6-AxPGUKkfB_QEcJwm0Bd-5fU,910
-project_llm_trainer-0.8.1.data/scripts/plot_lr,sha256=w_7XR_x3KYYyboeOVAeu_I4fveLFI-C0wBmRrNlmWUI,894
-project_llm_trainer-0.8.1.data/scripts/py_train,sha256=tOp9TquORQeU8XN5H7OVIk5O0Ypwi34p_GENxTwgwdk,265
-project_llm_trainer-0.8.1.data/scripts/smart_train,sha256=3oLIDuuqb4U4TU1lXy9V8lw_0gIf7i8tGsxlQ_s6bro,1220
-project_llm_trainer-0.8.1.dist-info/METADATA,sha256=07L7qqkujmk6YAwD5jPKe6dzyWPRu1Jirmp-6BqzMzA,195
-project_llm_trainer-0.8.1.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
-project_llm_trainer-0.8.1.dist-info/top_level.txt,sha256=LtRFg28i0QIG7iBCD2t095oSco99LCtkijibS9cMGik,12
-project_llm_trainer-0.8.1.dist-info/RECORD,,
+project_llm_trainer-0.8.3.data/scripts/calc_intermediate_size,sha256=AggpgNHokJiJMbEtVdOnolqr_4bH3i1UYuZNEAzC2Gc,460
+project_llm_trainer-0.8.3.data/scripts/ddp_train,sha256=eZSud6KYQAoKLsYB5QB-FI2zq5AZm6Apq1azKdupV3o,477
+project_llm_trainer-0.8.3.data/scripts/ds_train,sha256=41q4rOxwbvZDUY0FDdAIpG13PEaUWBpthhvFvww8uOc,388
+project_llm_trainer-0.8.3.data/scripts/plot_loss,sha256=MzFcdJESlVr1srj4Td6-AxPGUKkfB_QEcJwm0Bd-5fU,910
+project_llm_trainer-0.8.3.data/scripts/plot_lr,sha256=w_7XR_x3KYYyboeOVAeu_I4fveLFI-C0wBmRrNlmWUI,894
+project_llm_trainer-0.8.3.data/scripts/py_train,sha256=tOp9TquORQeU8XN5H7OVIk5O0Ypwi34p_GENxTwgwdk,265
+project_llm_trainer-0.8.3.data/scripts/smart_train,sha256=N8dp2n7k6bghGczedBVwOdtf1O66oM_cNPh9QmZt0bM,914
+project_llm_trainer-0.8.3.dist-info/METADATA,sha256=WxVWmIDlcVdlNbOIglZo7asnIQDmfvobKEFiloTvjrs,195
+project_llm_trainer-0.8.3.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
+project_llm_trainer-0.8.3.dist-info/top_level.txt,sha256=LtRFg28i0QIG7iBCD2t095oSco99LCtkijibS9cMGik,12
+project_llm_trainer-0.8.3.dist-info/RECORD,,

project_llm_trainer-0.8.1.data/scripts/ddp_train

@@ -1,24 +0,0 @@
-#!python
-
-if __name__ == '__main__':
-    import os, sys
-    arguments = sys.argv[1:]
-    # file_name
-    run_file_name = arguments[0]
-
-    # cuda_visible_devive
-    if len(arguments) > 1:
-        # 0,1,2,3
-        cuda_visible_devive = arguments[1]
-    else:
-        cuda_visible_devive = None
-
-    os.environ['PARALLEL_TYPE'] = 'ddp'
-
-    if cuda_visible_devive:
-        os.environ['CUDA_VISIBLE_DEVICES'] = cuda_visible_devive
-
-    command = f'torchrun --standalone --nproc_per_node=gpu {run_file_name}'
-
-    print(f'run command {command}')
-    os.system(command)

project_llm_trainer-0.8.1.data/scripts/ds_train

@@ -1,30 +0,0 @@
-#!python
-
-if __name__ == '__main__':
-    import os, sys
-    arguments = sys.argv[1:]
-    # file_name
-    run_file_name = arguments[0]
-
-    # cuda_visible_devive
-    if len(arguments) > 1:
-        # 0,1,2,3
-        cuda_visible_devive = arguments[1]
-
-        # cuda location
-        if len(arguments) > 2:
-            cuda_loc = arguments[2]
-        else:
-            cuda_loc = 'localhost'
-    else:
-        cuda_visible_devive = None
-        cuda_loc = None
-
-    os.environ['PARALLEL_TYPE'] = 'ds'
-
-    cuda_ctrl = f' --include {cuda_loc}:{cuda_visible_devive}' if cuda_visible_devive else ''
-
-    command = f'deepspeed{cuda_ctrl} {run_file_name}'
-
-    print(f'run command {command}')
-    os.system(command)