project-llm-trainer 0.5.14__py3-none-any.whl → 0.5.16__py3-none-any.whl

llm_trainer/checkpoint.py

@@ -65,7 +65,7 @@ def save_best_checkpoint(
         except:
             pass
 
-    TrainerTools().parallel.wait()
+    TrainerTools().parallel.wait('save best checkpoint')
     return need_replace
 
 

llm_trainer/dpo_trainer.py

@@ -170,14 +170,19 @@ class DPOTrainer(Trainer):
                         skipping_train = True
                         continue
 
-                    skipping_train = False
-
                     # 是否需要更新梯度
-                    if gradient_accumulation_steps > 1:
+                    if skipping_train:
+                        need_update_grad = False
+                    elif gradient_accumulation_steps > 1:
                         need_update_grad = (batch + 1) % gradient_accumulation_steps == 0 or batch == batch_count_per_file - 1
                     else:
                         need_update_grad = True
 
+                    # 要放在need_update_grad赋值下面，解决在继续训练时未知原因的卡死现象
+                    if skipping_train:
+                        TrainerTools().parallel.wait('skip train')
+                        skipping_train = False
+
                     try:
                         chosen_inputs: torch.Tensor = batch_data['chosen_inputs'].to(TrainerTools().parallel.device)
                         chosen_labels: torch.Tensor = batch_data['chosen_labels'].to(TrainerTools().parallel.device)

llm_trainer/ds_checkpoint.py

@@ -26,8 +26,6 @@ def save_ds_checkpoint(model: nn.Module):
         model.save_checkpoint(save_dir=ckpt_dir)
     except: ...
 
-    TrainerTools().parallel.wait()
-
     # 只在main rank上执行
     if TrainerTools().parallel.is_main_process:
         # 删除历史checkpoint
@@ -39,7 +37,7 @@ def save_ds_checkpoint(model: nn.Module):
                 shutil.rmtree(oldest_ckpt)
             except: ...
 
-    TrainerTools().parallel.wait()
+    TrainerTools().parallel.wait('remove old ds checkpoint')
 
 
 def load_ds_checkpoint(

llm_trainer/grpo_trainer.py

@@ -317,7 +317,9 @@ class GRPOTrainer(Trainer):
                         skipping_train = True
                         continue
 
-                    skipping_train = False
+                    if skipping_train:
+                        TrainerTools().parallel.wait('skip train')
+                        skipping_train = False
 
                     # start generate
                     if TrainerTools().parallel.is_main_process:

llm_trainer/parallel.py

@@ -139,9 +139,8 @@ class Parallel(ABC):
             return dist.get_world_size()
         return 1
 
-    def wait(self):
-        try:
-            log(f'wait at {self.device}')
-            dist.barrier()
-        except: ...
-        log(f'continue at {self.device}')
+    def wait(self, msg=None):
+        msg = f' for {msg}' if msg else None
+        log(f'wait at {self.device}{msg}')
+        dist.barrier()
+        log(f'continue at {self.device}{msg}')

llm_trainer/trainer.py

@@ -449,7 +449,7 @@ class Trainer:
                 )
                 generate_model.train()
 
-        TrainerTools().parallel.wait()
+        TrainerTools().parallel.wait('eval')
 
     def _on_batch_end(self, tag: str):
         self._eval(f'sign:batch/{tag}')
@@ -500,14 +500,19 @@ class Trainer:
                         skipping_train = True
                         continue
 
-                    skipping_train = False
-
                     # 是否需要更新梯度
-                    if gradient_accumulation_steps > 1:
+                    if skipping_train:
+                        need_update_grad = False
+                    elif gradient_accumulation_steps > 1:
                         need_update_grad = (batch + 1) % gradient_accumulation_steps == 0 or batch == batch_count_per_file - 1
                     else:
                         need_update_grad = True
 
+                    # 要放在need_update_grad赋值下面，解决在继续训练时未知原因的卡死现象
+                    if skipping_train:
+                        TrainerTools().parallel.wait('skip train')
+                        skipping_train = False
+
                     inputs = batch_data['inputs']
                     labels = batch_data['labels']
 

{project_llm_trainer-0.5.14.dist-info → project_llm_trainer-0.5.16.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: project_llm_trainer
-Version: 0.5.14
+Version: 0.5.16
 Summary: LLM and VLM trainer
 Author: qibin
 Author-email: qibin0506@gmail.com

{project_llm_trainer-0.5.14.dist-info → project_llm_trainer-0.5.16.dist-info}/RECORD

@@ -1,14 +1,14 @@
 llm_trainer/__init__.py,sha256=HWgtTEVeQSnZmEyYQm2K6eFEG4X2QAoigMlB5Z2tcXE,260
-llm_trainer/checkpoint.py,sha256=Wh5CwceIajTgJ9i_mH3I1R9N2nOLFqVFmlEMkTiGcD4,4306
+llm_trainer/checkpoint.py,sha256=gz31pZbbQvRTYrBhxV-MFaBAIFeqpe7rM6nFsjwT9lY,4328
 llm_trainer/dataset.py,sha256=4QlOo0SFB5816BUYegQjgobUqTUMQvdmZMM_OEAMSjE,4347
-llm_trainer/dpo_trainer.py,sha256=8LYRxviJKcB-rN_XprVsWr5YshU8KolggMm7irjbXvI,11990
-llm_trainer/ds_checkpoint.py,sha256=U7f79uWLysiyLnbJET_G4RwWh53Z0C9HCcXmNKq8UvM,2151
+llm_trainer/dpo_trainer.py,sha256=--ItH-rkkq24Da3M_Kf0VxpQ3t-k0fpZrzFGqkYsjks,12304
+llm_trainer/ds_checkpoint.py,sha256=Wzy7PvVVWR794-BW4uragWFTAkkgDvjvkF-qMdyB4fc,2141
 llm_trainer/eval.py,sha256=ZyUfSo2Q8P-lrCdPEnGkoo5pGubd0AabREK5eMISRII,1109
 llm_trainer/generate_utils.py,sha256=wrZoG2g7CsOyG4sb3px9vURHQFV6_9j5kQmpFc5A8yg,15335
-llm_trainer/grpo_trainer.py,sha256=PVTlKOEJpI0AMlh7Siw_MHpLm9CAZepCAMjjSZF6eRU,15996
+llm_trainer/grpo_trainer.py,sha256=g_ivzQop2SkvhlKAEWb0zUnIvNuHTfsOoIG6y29oTCw,16106
 llm_trainer/log.py,sha256=LxqTGRNZUGMTSQCePRpk-rYyxSnSIbT4kOdP8Fbzr0M,462
 llm_trainer/loss.py,sha256=NZCQeUXnLSj__mmDflE8g89KgE0emAJXIab0IERCLno,6023
-llm_trainer/parallel.py,sha256=DQu8GqEFxD99HQ6hKuIxxyKi-05dMO33eMhImYlPuOI,4468
+llm_trainer/parallel.py,sha256=j1L4n-JmDkDZblURrNKpEAWEqqGIAXAN9PT_fSS_OnE,4492
 llm_trainer/parallel_ddp.py,sha256=Pob9vUlBZnkL4oP1Re11kFob7nufMSE96pn7m7fuOEM,1345
 llm_trainer/parallel_ds.py,sha256=oy8RRxHud3rACWubFlJqqd0pjPEQhKeAPGPQUSdJX2c,1145
 llm_trainer/parallel_none.py,sha256=TG6Pm829Dg-yQu-97O-EHV3FCARBlNcP47KkGFAs16E,676
@@ -18,16 +18,16 @@ llm_trainer/sft_trainer.py,sha256=gxQA7T1o1QGUsHp2CX1Qb_fO5LppBJuNbc0H4ixCYUA,17
 llm_trainer/tokenizer.py,sha256=SSpgXtb0e1NtQqRW0gCq09TTZi47umggy-Fh5EMHKJg,6708
 llm_trainer/tools.py,sha256=yF17lp6oOfLe2XJeKDQ1juZcbv-6vFamJSLwEeArduA,2975
 llm_trainer/train_configs.py,sha256=992wy0YhBG2WvxwdLEPL4_-JUl4NkwMPT-jj_BIHo6A,7347
-llm_trainer/trainer.py,sha256=FF75J-BRUp34No2TvQIgomvNozWYzVhDeOfaBgQLV9g,26079
+llm_trainer/trainer.py,sha256=YqWhD9jXbrUdm3KEjEHLyg_qHiXCy5R7PK-arCXxJ6M,26399
 llm_trainer/utils.py,sha256=LWNhyQ0NDEZ9mZtk2Ryvh6EulvHIaUGIflugSpqmeFI,6791
-project_llm_trainer-0.5.14.data/scripts/calc_intermediate_size,sha256=AggpgNHokJiJMbEtVdOnolqr_4bH3i1UYuZNEAzC2Gc,460
-project_llm_trainer-0.5.14.data/scripts/ddp_train,sha256=x81AasaN2-9TwARFFF1l7iV1LmfMQ0bLw0i_CGbOwSw,299
-project_llm_trainer-0.5.14.data/scripts/ds_train,sha256=qL3qc3TcedBCw98UZUjW07ONcErRawLE1HymW2AmscA,265
-project_llm_trainer-0.5.14.data/scripts/plot_loss,sha256=MzFcdJESlVr1srj4Td6-AxPGUKkfB_QEcJwm0Bd-5fU,910
-project_llm_trainer-0.5.14.data/scripts/plot_lr,sha256=w_7XR_x3KYYyboeOVAeu_I4fveLFI-C0wBmRrNlmWUI,894
-project_llm_trainer-0.5.14.data/scripts/py_train,sha256=tOp9TquORQeU8XN5H7OVIk5O0Ypwi34p_GENxTwgwdk,265
-project_llm_trainer-0.5.14.data/scripts/smart_train,sha256=Pmt4Q0to4Hoz82iB9uFPZuz7uahNUbfE7FR1940EBy8,716
-project_llm_trainer-0.5.14.dist-info/METADATA,sha256=639zcd1nZ1iEJDswOjEL8eudA39c6RhqPVi-H-xttWE,196
-project_llm_trainer-0.5.14.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
-project_llm_trainer-0.5.14.dist-info/top_level.txt,sha256=LtRFg28i0QIG7iBCD2t095oSco99LCtkijibS9cMGik,12
-project_llm_trainer-0.5.14.dist-info/RECORD,,
+project_llm_trainer-0.5.16.data/scripts/calc_intermediate_size,sha256=AggpgNHokJiJMbEtVdOnolqr_4bH3i1UYuZNEAzC2Gc,460
+project_llm_trainer-0.5.16.data/scripts/ddp_train,sha256=x81AasaN2-9TwARFFF1l7iV1LmfMQ0bLw0i_CGbOwSw,299
+project_llm_trainer-0.5.16.data/scripts/ds_train,sha256=qL3qc3TcedBCw98UZUjW07ONcErRawLE1HymW2AmscA,265
+project_llm_trainer-0.5.16.data/scripts/plot_loss,sha256=MzFcdJESlVr1srj4Td6-AxPGUKkfB_QEcJwm0Bd-5fU,910
+project_llm_trainer-0.5.16.data/scripts/plot_lr,sha256=w_7XR_x3KYYyboeOVAeu_I4fveLFI-C0wBmRrNlmWUI,894
+project_llm_trainer-0.5.16.data/scripts/py_train,sha256=tOp9TquORQeU8XN5H7OVIk5O0Ypwi34p_GENxTwgwdk,265
+project_llm_trainer-0.5.16.data/scripts/smart_train,sha256=Pmt4Q0to4Hoz82iB9uFPZuz7uahNUbfE7FR1940EBy8,716
+project_llm_trainer-0.5.16.dist-info/METADATA,sha256=h0TMNrZMUU875tVasbuqt69EuPPMbo_nv6tHQLKeNbQ,196
+project_llm_trainer-0.5.16.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
+project_llm_trainer-0.5.16.dist-info/top_level.txt,sha256=LtRFg28i0QIG7iBCD2t095oSco99LCtkijibS9cMGik,12
+project_llm_trainer-0.5.16.dist-info/RECORD,,