project-llm-trainer 0.9.0__py3-none-any.whl → 0.9.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of project-llm-trainer might be problematic. Click here for more details.

Files changed (12) hide show
  1. llm_trainer/trainer.py +9 -5
  2. {project_llm_trainer-0.9.0.dist-info → project_llm_trainer-0.9.1.dist-info}/METADATA +1 -1
  3. {project_llm_trainer-0.9.0.dist-info → project_llm_trainer-0.9.1.dist-info}/RECORD +12 -12
  4. {project_llm_trainer-0.9.0.data → project_llm_trainer-0.9.1.data}/scripts/calc_intermediate_size +0 -0
  5. {project_llm_trainer-0.9.0.data → project_llm_trainer-0.9.1.data}/scripts/ddp_train +0 -0
  6. {project_llm_trainer-0.9.0.data → project_llm_trainer-0.9.1.data}/scripts/ds_train +0 -0
  7. {project_llm_trainer-0.9.0.data → project_llm_trainer-0.9.1.data}/scripts/plot_loss +0 -0
  8. {project_llm_trainer-0.9.0.data → project_llm_trainer-0.9.1.data}/scripts/plot_lr +0 -0
  9. {project_llm_trainer-0.9.0.data → project_llm_trainer-0.9.1.data}/scripts/py_train +0 -0
  10. {project_llm_trainer-0.9.0.data → project_llm_trainer-0.9.1.data}/scripts/smart_train +0 -0
  11. {project_llm_trainer-0.9.0.dist-info → project_llm_trainer-0.9.1.dist-info}/WHEEL +0 -0
  12. {project_llm_trainer-0.9.0.dist-info → project_llm_trainer-0.9.1.dist-info}/top_level.txt +0 -0
llm_trainer/trainer.py CHANGED
@@ -1,5 +1,7 @@
1
1
  from typing import Optional, Tuple, List, Dict, Any
2
2
  import copy
3
+ import importlib.metadata
4
+ from packaging import version
3
5
 
4
6
  import torch
5
7
  import torch.distributed as dist
@@ -168,12 +170,14 @@ class Trainer:
168
170
  if ('zero_optimization' in self.parallel_kwargs
169
171
  and 'offload_optimizer' in self.parallel_kwargs['zero_optimization']
170
172
  and self.parallel_kwargs['zero_optimization']['offload_optimizer']['device'] == 'cpu'):
171
- # offline optimizer to cpu
172
- # 不能使用 deepspeed.ops.lion.cpu_lion.DeepSpeedCPULion???
173
- # 所以，这里忽略lion判断
174
- optimizer = deepspeed.ops.adam.DeepSpeedCPUAdam
175
173
  if self.train_config.optim_config.optim_type == 'lion':
176
- log('When set offload_optimizer, lion optim is unsupported, so set optim to adam!!!!!')
174
+ if version.parse(importlib.metadata.version("deepspeed")) >= version.parse('0.17.6'):
175
+ optimizer = deepspeed.ops.lion.DeepSpeedCPULion
176
+ else:
177
+ optimizer = deepspeed.ops.adam.DeepSpeedCPUAdam
178
+ log('When set offload_optimizer, lion optim is unsupported, so set optim to adam!!!!!')
179
+ else:
180
+ optimizer = deepspeed.ops.adam.DeepSpeedCPUAdam
177
181
  else:
178
182
  if self.train_config.optim_config.optim_type == 'lion':
179
183
  optimizer = deepspeed.ops.lion.FusedLion
{project_llm_trainer-0.9.0.dist-info → project_llm_trainer-0.9.1.dist-info}/METADATA RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: project_llm_trainer
3
- Version: 0.9.0
3
+ Version: 0.9.1
4
4
  Summary: LLM and VLM trainer
5
5
  Author: qibin
6
6
  Author-email: qibin0506@gmail.com
{project_llm_trainer-0.9.0.dist-info → project_llm_trainer-0.9.1.dist-info}/RECORD RENAMED
@@ -18,16 +18,16 @@ llm_trainer/sft_trainer.py,sha256=rSOGZx53jMgOuJdztfxQASYJ62uD0dVaih4IAnSwGBc,17
18
18
  llm_trainer/tokenizer.py,sha256=0-xQCMz1xiPTDAZiYsVsiECSoZ_1eIvW9XsZOoFfakQ,7250
19
19
  llm_trainer/tools.py,sha256=5op5qrjjkK-Lr9oes5VxIVnOVYOYGoAdlIJq9mPUf64,2637
20
20
  llm_trainer/train_configs.py,sha256=u3b3RhZn7S6vm2DKNF5sNRHzp73-eYnActGSlzcIbzs,7638
21
- llm_trainer/trainer.py,sha256=sz350VaLR7odGm0vYGTGvFWcr5bC7tmwMx7PUY6LA2o,28980
21
+ llm_trainer/trainer.py,sha256=Wf1JLWYhmxKKef49asxkaR4ZocKO-jlupxksv-BNfZI,29170
22
22
  llm_trainer/utils.py,sha256=xC5plG-8-_Al5yIF5xIU5lroOcBBk98TEhtUJrazZPE,12305
23
- project_llm_trainer-0.9.0.data/scripts/calc_intermediate_size,sha256=AggpgNHokJiJMbEtVdOnolqr_4bH3i1UYuZNEAzC2Gc,460
24
- project_llm_trainer-0.9.0.data/scripts/ddp_train,sha256=eZSud6KYQAoKLsYB5QB-FI2zq5AZm6Apq1azKdupV3o,477
25
- project_llm_trainer-0.9.0.data/scripts/ds_train,sha256=41q4rOxwbvZDUY0FDdAIpG13PEaUWBpthhvFvww8uOc,388
26
- project_llm_trainer-0.9.0.data/scripts/plot_loss,sha256=O9ooioAJ-79-X06LosgqF8XOqQe-beRxYm3LsLunmoU,908
27
- project_llm_trainer-0.9.0.data/scripts/plot_lr,sha256=w_7XR_x3KYYyboeOVAeu_I4fveLFI-C0wBmRrNlmWUI,894
28
- project_llm_trainer-0.9.0.data/scripts/py_train,sha256=tOp9TquORQeU8XN5H7OVIk5O0Ypwi34p_GENxTwgwdk,265
29
- project_llm_trainer-0.9.0.data/scripts/smart_train,sha256=N8dp2n7k6bghGczedBVwOdtf1O66oM_cNPh9QmZt0bM,914
30
- project_llm_trainer-0.9.0.dist-info/METADATA,sha256=PSaXkuGCDIwRi8gPHD3LEYXmDhmMj9rC_8xlZP5ftxw,195
31
- project_llm_trainer-0.9.0.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
32
- project_llm_trainer-0.9.0.dist-info/top_level.txt,sha256=LtRFg28i0QIG7iBCD2t095oSco99LCtkijibS9cMGik,12
33
- project_llm_trainer-0.9.0.dist-info/RECORD,,
23
+ project_llm_trainer-0.9.1.data/scripts/calc_intermediate_size,sha256=AggpgNHokJiJMbEtVdOnolqr_4bH3i1UYuZNEAzC2Gc,460
24
+ project_llm_trainer-0.9.1.data/scripts/ddp_train,sha256=eZSud6KYQAoKLsYB5QB-FI2zq5AZm6Apq1azKdupV3o,477
25
+ project_llm_trainer-0.9.1.data/scripts/ds_train,sha256=41q4rOxwbvZDUY0FDdAIpG13PEaUWBpthhvFvww8uOc,388
26
+ project_llm_trainer-0.9.1.data/scripts/plot_loss,sha256=O9ooioAJ-79-X06LosgqF8XOqQe-beRxYm3LsLunmoU,908
27
+ project_llm_trainer-0.9.1.data/scripts/plot_lr,sha256=w_7XR_x3KYYyboeOVAeu_I4fveLFI-C0wBmRrNlmWUI,894
28
+ project_llm_trainer-0.9.1.data/scripts/py_train,sha256=tOp9TquORQeU8XN5H7OVIk5O0Ypwi34p_GENxTwgwdk,265
29
+ project_llm_trainer-0.9.1.data/scripts/smart_train,sha256=N8dp2n7k6bghGczedBVwOdtf1O66oM_cNPh9QmZt0bM,914
30
+ project_llm_trainer-0.9.1.dist-info/METADATA,sha256=1Rtg7wi3BLy-9cFnNouWF3bVkexiMHeFSyaSmvrOFpA,195
31
+ project_llm_trainer-0.9.1.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
32
+ project_llm_trainer-0.9.1.dist-info/top_level.txt,sha256=LtRFg28i0QIG7iBCD2t095oSco99LCtkijibS9cMGik,12
33
+ project_llm_trainer-0.9.1.dist-info/RECORD,,