project-llm-trainer 0.4.6__py3-none-any.whl → 0.4.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of project-llm-trainer might be problematic. Click here for more details.
- llm_trainer/ds_checkpoint.py +1 -2
- llm_trainer/generate_utils.py +5 -5
- {project_llm_trainer-0.4.6.dist-info → project_llm_trainer-0.4.7.dist-info}/METADATA +1 -1
- {project_llm_trainer-0.4.6.dist-info → project_llm_trainer-0.4.7.dist-info}/RECORD +13 -13
- {project_llm_trainer-0.4.6.data → project_llm_trainer-0.4.7.data}/scripts/calc_intermediate_size +0 -0
- {project_llm_trainer-0.4.6.data → project_llm_trainer-0.4.7.data}/scripts/ddp_train +0 -0
- {project_llm_trainer-0.4.6.data → project_llm_trainer-0.4.7.data}/scripts/ds_train +0 -0
- {project_llm_trainer-0.4.6.data → project_llm_trainer-0.4.7.data}/scripts/plot_loss +0 -0
- {project_llm_trainer-0.4.6.data → project_llm_trainer-0.4.7.data}/scripts/plot_lr +0 -0
- {project_llm_trainer-0.4.6.data → project_llm_trainer-0.4.7.data}/scripts/py_train +0 -0
- {project_llm_trainer-0.4.6.data → project_llm_trainer-0.4.7.data}/scripts/smart_train +0 -0
- {project_llm_trainer-0.4.6.dist-info → project_llm_trainer-0.4.7.dist-info}/WHEEL +0 -0
- {project_llm_trainer-0.4.6.dist-info → project_llm_trainer-0.4.7.dist-info}/top_level.txt +0 -0
llm_trainer/ds_checkpoint.py
CHANGED
|
@@ -69,7 +69,7 @@ def load_ds_checkpoint_for_eval(model: nn.Module):
|
|
|
69
69
|
|
|
70
70
|
def _get_ds_full_state_dict_on_rank0(model: DeepSpeedEngine) -> Optional[dict]:
|
|
71
71
|
"""
|
|
72
|
-
|
|
72
|
+
需要在所有rank上调用，然后只有rank0有值
|
|
73
73
|
"""
|
|
74
74
|
|
|
75
75
|
if model.zero_optimization_stage() != 3:
|
|
@@ -99,7 +99,6 @@ def _get_ds_full_state_dict_on_rank0(model: DeepSpeedEngine) -> Optional[dict]:
|
|
|
99
99
|
# if TrainerTools().parallel.is_main_process:
|
|
100
100
|
# state_dict_on_rank_0[param_name] = param.data.to(torch.float32).cpu().clone()
|
|
101
101
|
# else:
|
|
102
|
-
# print("22222222")
|
|
103
102
|
# if TrainerTools().parallel.is_main_process:
|
|
104
103
|
# state_dict_on_rank_0[param_name] = param.data.to(torch.float32).cpu().clone()
|
|
105
104
|
#
|
llm_trainer/generate_utils.py
CHANGED
|
@@ -175,7 +175,7 @@ def _generate(
|
|
|
175
175
|
if k and k != 0:
|
|
176
176
|
logits = _top_k_warper(logits, k, device)
|
|
177
177
|
|
|
178
|
-
if p and
|
|
178
|
+
if p and 0 < p <= 1:
|
|
179
179
|
logits = _top_p_warper(logits, p)
|
|
180
180
|
|
|
181
181
|
if multinomial:
|
|
@@ -210,7 +210,7 @@ def _streaming_generate(
|
|
|
210
210
|
max_new_tokens: int,
|
|
211
211
|
temperature: Optional[float] = 1.0,
|
|
212
212
|
k: Optional[int] = None,
|
|
213
|
-
p: Optional[float] =
|
|
213
|
+
p: Optional[float] = None,
|
|
214
214
|
pixel_values: Optional[torch.Tensor] = None,
|
|
215
215
|
tokens_per_image: int = -1,
|
|
216
216
|
suppress_tokens: Optional[List[int]] = None,
|
|
@@ -245,7 +245,7 @@ def streaming_generate(
|
|
|
245
245
|
max_new_tokens: int,
|
|
246
246
|
temperature: Optional[float] = 1.0,
|
|
247
247
|
k: Optional[int] = None,
|
|
248
|
-
p: Optional[float] =
|
|
248
|
+
p: Optional[float] = None,
|
|
249
249
|
pixel_values: Optional[torch.Tensor] = None,
|
|
250
250
|
tokens_per_image: int = -1,
|
|
251
251
|
suppress_tokens: Optional[List[int]] = None,
|
|
@@ -278,7 +278,7 @@ def generate(
|
|
|
278
278
|
max_new_tokens: int,
|
|
279
279
|
temperature: Optional[float] = 1.0,
|
|
280
280
|
k: Optional[int] = None,
|
|
281
|
-
p: Optional[float] =
|
|
281
|
+
p: Optional[float] = None,
|
|
282
282
|
pixel_values: Optional[torch.Tensor] = None,
|
|
283
283
|
tokens_per_image: int = -1,
|
|
284
284
|
suppress_tokens: Optional[List[int]] = None,
|
|
@@ -382,7 +382,7 @@ def batch_generate(
|
|
|
382
382
|
if k and k != 0:
|
|
383
383
|
logits = _top_k_warper(logits, k, device)
|
|
384
384
|
|
|
385
|
-
if p and
|
|
385
|
+
if p and 0 < p <= 1:
|
|
386
386
|
logits = _top_p_warper(logits, p)
|
|
387
387
|
|
|
388
388
|
prob = logits.softmax(dim=-1)
|
|
@@ -3,10 +3,10 @@ llm_trainer/checkpoint.py,sha256=yZcExxneN2yzvWxRiK-pstMWs35LV7GiOfqcLq-S6vc,574
|
|
|
3
3
|
llm_trainer/dataset.py,sha256=4QlOo0SFB5816BUYegQjgobUqTUMQvdmZMM_OEAMSjE,4347
|
|
4
4
|
llm_trainer/dcp.py,sha256=PkD97DyrOtoTKn4FJsfL3VqAy4dxufgjdzJEz8-Cnoc,3635
|
|
5
5
|
llm_trainer/dpo_trainer.py,sha256=rC_I5ipesSlP3gFK_SG2GB8NbgJAMu4K7KLxkAS-aRY,13406
|
|
6
|
-
llm_trainer/ds_checkpoint.py,sha256=
|
|
6
|
+
llm_trainer/ds_checkpoint.py,sha256=x_tjgJR47P8gVwV4qAnTUCGwx7eVq2Epw0vOVV7fkYo,4925
|
|
7
7
|
llm_trainer/eval.py,sha256=NDm8PbXLch7xT81xPYPRCNrcrB_Xj5GDJSCxyVwUOp4,1524
|
|
8
8
|
llm_trainer/fsdp_checkpoint.py,sha256=lqZFzHyWyfzuCq_81kQNtJd2qaiMeY1N5BCEMnrJTBw,3192
|
|
9
|
-
llm_trainer/generate_utils.py,sha256=
|
|
9
|
+
llm_trainer/generate_utils.py,sha256=BmjpCrus_jvJ3SM2KS1bQNzJWAFnpJ9mI28iBWXZpvo,15206
|
|
10
10
|
llm_trainer/grpo_trainer.py,sha256=bZPrxhyPQLAnFzWhI7hhA6fpuKVNwj7nOm9k0ku9aK4,15977
|
|
11
11
|
llm_trainer/log.py,sha256=LxqTGRNZUGMTSQCePRpk-rYyxSnSIbT4kOdP8Fbzr0M,462
|
|
12
12
|
llm_trainer/loss.py,sha256=Yv3fsaVuZ5AhnGPJOr5vEMb_tM2urR6mCb4DBbrHHI8,6030
|
|
@@ -22,14 +22,14 @@ llm_trainer/tools.py,sha256=O45-20wRmh-nyTfU-U-XtjbKAoe7boEIsUvWT_NaKx4,3041
|
|
|
22
22
|
llm_trainer/train_configs.py,sha256=arnet3tIzgVnwshod08F1jE7r4I7e-SIgMy55IagPnE,15971
|
|
23
23
|
llm_trainer/trainer.py,sha256=Zy1oesBfsFlDedZ4hn3gcAkTrpi5fr76bFFQikfAkak,25351
|
|
24
24
|
llm_trainer/utils.py,sha256=-ivhMF0d999va13S1wt2uBvtVw8Nvr3uBzhaUFKL04Q,6826
|
|
25
|
-
project_llm_trainer-0.4.
|
|
26
|
-
project_llm_trainer-0.4.
|
|
27
|
-
project_llm_trainer-0.4.
|
|
28
|
-
project_llm_trainer-0.4.
|
|
29
|
-
project_llm_trainer-0.4.
|
|
30
|
-
project_llm_trainer-0.4.
|
|
31
|
-
project_llm_trainer-0.4.
|
|
32
|
-
project_llm_trainer-0.4.
|
|
33
|
-
project_llm_trainer-0.4.
|
|
34
|
-
project_llm_trainer-0.4.
|
|
35
|
-
project_llm_trainer-0.4.
|
|
25
|
+
project_llm_trainer-0.4.7.data/scripts/calc_intermediate_size,sha256=AggpgNHokJiJMbEtVdOnolqr_4bH3i1UYuZNEAzC2Gc,460
|
|
26
|
+
project_llm_trainer-0.4.7.data/scripts/ddp_train,sha256=x81AasaN2-9TwARFFF1l7iV1LmfMQ0bLw0i_CGbOwSw,299
|
|
27
|
+
project_llm_trainer-0.4.7.data/scripts/ds_train,sha256=qL3qc3TcedBCw98UZUjW07ONcErRawLE1HymW2AmscA,265
|
|
28
|
+
project_llm_trainer-0.4.7.data/scripts/plot_loss,sha256=MzFcdJESlVr1srj4Td6-AxPGUKkfB_QEcJwm0Bd-5fU,910
|
|
29
|
+
project_llm_trainer-0.4.7.data/scripts/plot_lr,sha256=w_7XR_x3KYYyboeOVAeu_I4fveLFI-C0wBmRrNlmWUI,894
|
|
30
|
+
project_llm_trainer-0.4.7.data/scripts/py_train,sha256=tOp9TquORQeU8XN5H7OVIk5O0Ypwi34p_GENxTwgwdk,265
|
|
31
|
+
project_llm_trainer-0.4.7.data/scripts/smart_train,sha256=Pmt4Q0to4Hoz82iB9uFPZuz7uahNUbfE7FR1940EBy8,716
|
|
32
|
+
project_llm_trainer-0.4.7.dist-info/METADATA,sha256=u4_cQkQaH9QKqG_XcWiXzGHD5rnrzqHjvJWQvgVnkZQ,195
|
|
33
|
+
project_llm_trainer-0.4.7.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
|
|
34
|
+
project_llm_trainer-0.4.7.dist-info/top_level.txt,sha256=LtRFg28i0QIG7iBCD2t095oSco99LCtkijibS9cMGik,12
|
|
35
|
+
project_llm_trainer-0.4.7.dist-info/RECORD,,
|
{project_llm_trainer-0.4.6.data → project_llm_trainer-0.4.7.data}/scripts/calc_intermediate_size
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|