together 1.4.1__py3-none-any.whl → 1.4.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- together/cli/api/finetune.py +67 -5
- together/constants.py +6 -0
- together/legacy/finetune.py +1 -1
- together/resources/finetune.py +173 -15
- together/types/__init__.py +6 -0
- together/types/chat_completions.py +6 -0
- together/types/endpoints.py +3 -3
- together/types/finetune.py +45 -0
- together/utils/__init__.py +4 -0
- together/utils/files.py +139 -66
- together/utils/tools.py +53 -2
- {together-1.4.1.dist-info → together-1.4.4.dist-info}/METADATA +93 -23
- {together-1.4.1.dist-info → together-1.4.4.dist-info}/RECORD +16 -16
- {together-1.4.1.dist-info → together-1.4.4.dist-info}/WHEEL +1 -1
- {together-1.4.1.dist-info → together-1.4.4.dist-info}/LICENSE +0 -0
- {together-1.4.1.dist-info → together-1.4.4.dist-info}/entry_points.txt +0 -0
together/cli/api/finetune.py
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import json
|
|
4
|
-
from datetime import datetime
|
|
4
|
+
from datetime import datetime, timezone
|
|
5
5
|
from textwrap import wrap
|
|
6
6
|
from typing import Any, Literal
|
|
7
|
+
import re
|
|
7
8
|
|
|
8
9
|
import click
|
|
9
10
|
from click.core import ParameterSource # type: ignore[attr-defined]
|
|
@@ -17,8 +18,13 @@ from together.utils import (
|
|
|
17
18
|
log_warn,
|
|
18
19
|
log_warn_once,
|
|
19
20
|
parse_timestamp,
|
|
21
|
+
format_timestamp,
|
|
22
|
+
)
|
|
23
|
+
from together.types.finetune import (
|
|
24
|
+
DownloadCheckpointType,
|
|
25
|
+
FinetuneTrainingLimits,
|
|
26
|
+
FinetuneEventType,
|
|
20
27
|
)
|
|
21
|
-
from together.types.finetune import DownloadCheckpointType, FinetuneTrainingLimits
|
|
22
28
|
|
|
23
29
|
|
|
24
30
|
_CONFIRMATION_MESSAGE = (
|
|
@@ -104,6 +110,18 @@ def fine_tuning(ctx: click.Context) -> None:
|
|
|
104
110
|
default="all-linear",
|
|
105
111
|
help="Trainable modules for LoRA adapters. For example, 'all-linear', 'q_proj,v_proj'",
|
|
106
112
|
)
|
|
113
|
+
@click.option(
|
|
114
|
+
"--training-method",
|
|
115
|
+
type=click.Choice(["sft", "dpo"]),
|
|
116
|
+
default="sft",
|
|
117
|
+
help="Training method to use. Options: sft (supervised fine-tuning), dpo (Direct Preference Optimization)",
|
|
118
|
+
)
|
|
119
|
+
@click.option(
|
|
120
|
+
"--dpo-beta",
|
|
121
|
+
type=float,
|
|
122
|
+
default=0.1,
|
|
123
|
+
help="Beta parameter for DPO training (only used when '--training-method' is 'dpo')",
|
|
124
|
+
)
|
|
107
125
|
@click.option(
|
|
108
126
|
"--suffix", type=str, default=None, help="Suffix for the fine-tuned model name"
|
|
109
127
|
)
|
|
@@ -126,6 +144,14 @@ def fine_tuning(ctx: click.Context) -> None:
|
|
|
126
144
|
help="Whether to mask the user messages in conversational data or prompts in instruction data. "
|
|
127
145
|
"`auto` will automatically determine whether to mask the inputs based on the data format.",
|
|
128
146
|
)
|
|
147
|
+
@click.option(
|
|
148
|
+
"--from-checkpoint",
|
|
149
|
+
type=str,
|
|
150
|
+
default=None,
|
|
151
|
+
help="The checkpoint identifier to continue training from a previous fine-tuning job. "
|
|
152
|
+
"The format: {$JOB_ID/$OUTPUT_MODEL_NAME}:{$STEP}. "
|
|
153
|
+
"The step value is optional, without it the final checkpoint will be used.",
|
|
154
|
+
)
|
|
129
155
|
def create(
|
|
130
156
|
ctx: click.Context,
|
|
131
157
|
training_file: str,
|
|
@@ -152,6 +178,9 @@ def create(
|
|
|
152
178
|
wandb_name: str,
|
|
153
179
|
confirm: bool,
|
|
154
180
|
train_on_inputs: bool | Literal["auto"],
|
|
181
|
+
training_method: str,
|
|
182
|
+
dpo_beta: float,
|
|
183
|
+
from_checkpoint: str,
|
|
155
184
|
) -> None:
|
|
156
185
|
"""Start fine-tuning"""
|
|
157
186
|
client: Together = ctx.obj
|
|
@@ -180,6 +209,9 @@ def create(
|
|
|
180
209
|
wandb_project_name=wandb_project_name,
|
|
181
210
|
wandb_name=wandb_name,
|
|
182
211
|
train_on_inputs=train_on_inputs,
|
|
212
|
+
training_method=training_method,
|
|
213
|
+
dpo_beta=dpo_beta,
|
|
214
|
+
from_checkpoint=from_checkpoint,
|
|
183
215
|
)
|
|
184
216
|
|
|
185
217
|
model_limits: FinetuneTrainingLimits = client.fine_tuning.get_model_limits(
|
|
@@ -261,7 +293,9 @@ def list(ctx: click.Context) -> None:
|
|
|
261
293
|
|
|
262
294
|
response.data = response.data or []
|
|
263
295
|
|
|
264
|
-
|
|
296
|
+
# Use a default datetime for None values to make sure the key function always returns a comparable value
|
|
297
|
+
epoch_start = datetime.fromtimestamp(0, tz=timezone.utc)
|
|
298
|
+
response.data.sort(key=lambda x: parse_timestamp(x.created_at or "") or epoch_start)
|
|
265
299
|
|
|
266
300
|
display_list = []
|
|
267
301
|
for i in response.data:
|
|
@@ -344,6 +378,34 @@ def list_events(ctx: click.Context, fine_tune_id: str) -> None:
|
|
|
344
378
|
click.echo(table)
|
|
345
379
|
|
|
346
380
|
|
|
381
|
+
@fine_tuning.command()
|
|
382
|
+
@click.pass_context
|
|
383
|
+
@click.argument("fine_tune_id", type=str, required=True)
|
|
384
|
+
def list_checkpoints(ctx: click.Context, fine_tune_id: str) -> None:
|
|
385
|
+
"""List available checkpoints for a fine-tuning job"""
|
|
386
|
+
client: Together = ctx.obj
|
|
387
|
+
|
|
388
|
+
checkpoints = client.fine_tuning.list_checkpoints(fine_tune_id)
|
|
389
|
+
|
|
390
|
+
display_list = []
|
|
391
|
+
for checkpoint in checkpoints:
|
|
392
|
+
display_list.append(
|
|
393
|
+
{
|
|
394
|
+
"Type": checkpoint.type,
|
|
395
|
+
"Timestamp": format_timestamp(checkpoint.timestamp),
|
|
396
|
+
"Name": checkpoint.name,
|
|
397
|
+
}
|
|
398
|
+
)
|
|
399
|
+
|
|
400
|
+
if display_list:
|
|
401
|
+
click.echo(f"Job {fine_tune_id} contains the following checkpoints:")
|
|
402
|
+
table = tabulate(display_list, headers="keys", tablefmt="grid")
|
|
403
|
+
click.echo(table)
|
|
404
|
+
click.echo("\nTo download a checkpoint, use `together fine-tuning download`")
|
|
405
|
+
else:
|
|
406
|
+
click.echo(f"No checkpoints found for job {fine_tune_id}")
|
|
407
|
+
|
|
408
|
+
|
|
347
409
|
@fine_tuning.command()
|
|
348
410
|
@click.pass_context
|
|
349
411
|
@click.argument("fine_tune_id", type=str, required=True)
|
|
@@ -358,7 +420,7 @@ def list_events(ctx: click.Context, fine_tune_id: str) -> None:
|
|
|
358
420
|
"--checkpoint-step",
|
|
359
421
|
type=int,
|
|
360
422
|
required=False,
|
|
361
|
-
default
|
|
423
|
+
default=None,
|
|
362
424
|
help="Download fine-tuning checkpoint. Defaults to latest.",
|
|
363
425
|
)
|
|
364
426
|
@click.option(
|
|
@@ -372,7 +434,7 @@ def download(
|
|
|
372
434
|
ctx: click.Context,
|
|
373
435
|
fine_tune_id: str,
|
|
374
436
|
output_dir: str,
|
|
375
|
-
checkpoint_step: int,
|
|
437
|
+
checkpoint_step: int | None,
|
|
376
438
|
checkpoint_type: DownloadCheckpointType,
|
|
377
439
|
) -> None:
|
|
378
440
|
"""Download fine-tuning checkpoint"""
|
together/constants.py
CHANGED
|
@@ -39,12 +39,18 @@ class DatasetFormat(enum.Enum):
|
|
|
39
39
|
GENERAL = "general"
|
|
40
40
|
CONVERSATION = "conversation"
|
|
41
41
|
INSTRUCTION = "instruction"
|
|
42
|
+
PREFERENCE_OPENAI = "preference_openai"
|
|
42
43
|
|
|
43
44
|
|
|
44
45
|
JSONL_REQUIRED_COLUMNS_MAP = {
|
|
45
46
|
DatasetFormat.GENERAL: ["text"],
|
|
46
47
|
DatasetFormat.CONVERSATION: ["messages"],
|
|
47
48
|
DatasetFormat.INSTRUCTION: ["prompt", "completion"],
|
|
49
|
+
DatasetFormat.PREFERENCE_OPENAI: [
|
|
50
|
+
"input",
|
|
51
|
+
"preferred_output",
|
|
52
|
+
"non_preferred_output",
|
|
53
|
+
],
|
|
48
54
|
}
|
|
49
55
|
REQUIRED_COLUMNS_MESSAGE = ["role", "content"]
|
|
50
56
|
POSSIBLE_ROLES_CONVERSATION = ["system", "user", "assistant"]
|
together/legacy/finetune.py
CHANGED
together/resources/finetune.py
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import re
|
|
3
4
|
from pathlib import Path
|
|
4
|
-
from typing import Literal
|
|
5
|
+
from typing import Literal, List
|
|
5
6
|
|
|
6
7
|
from rich import print as rprint
|
|
7
8
|
|
|
@@ -22,9 +23,28 @@ from together.types import (
|
|
|
22
23
|
TrainingType,
|
|
23
24
|
FinetuneLRScheduler,
|
|
24
25
|
FinetuneLinearLRSchedulerArgs,
|
|
26
|
+
TrainingMethodDPO,
|
|
27
|
+
TrainingMethodSFT,
|
|
28
|
+
FinetuneCheckpoint,
|
|
25
29
|
)
|
|
26
|
-
from together.types.finetune import
|
|
27
|
-
|
|
30
|
+
from together.types.finetune import (
|
|
31
|
+
DownloadCheckpointType,
|
|
32
|
+
FinetuneEventType,
|
|
33
|
+
FinetuneEvent,
|
|
34
|
+
)
|
|
35
|
+
from together.utils import (
|
|
36
|
+
log_warn_once,
|
|
37
|
+
normalize_key,
|
|
38
|
+
get_event_step,
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
_FT_JOB_WITH_STEP_REGEX = r"^ft-[\dabcdef-]+:\d+$"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
AVAILABLE_TRAINING_METHODS = {
|
|
45
|
+
TrainingMethodSFT().method,
|
|
46
|
+
TrainingMethodDPO().method,
|
|
47
|
+
}
|
|
28
48
|
|
|
29
49
|
|
|
30
50
|
def createFinetuneRequest(
|
|
@@ -52,7 +72,11 @@ def createFinetuneRequest(
|
|
|
52
72
|
wandb_project_name: str | None = None,
|
|
53
73
|
wandb_name: str | None = None,
|
|
54
74
|
train_on_inputs: bool | Literal["auto"] = "auto",
|
|
75
|
+
training_method: str = "sft",
|
|
76
|
+
dpo_beta: float | None = None,
|
|
77
|
+
from_checkpoint: str | None = None,
|
|
55
78
|
) -> FinetuneRequest:
|
|
79
|
+
|
|
56
80
|
if batch_size == "max":
|
|
57
81
|
log_warn_once(
|
|
58
82
|
"Starting from together>=1.3.0, "
|
|
@@ -100,11 +124,20 @@ def createFinetuneRequest(
|
|
|
100
124
|
if weight_decay is not None and (weight_decay < 0):
|
|
101
125
|
raise ValueError("Weight decay should be non-negative")
|
|
102
126
|
|
|
127
|
+
if training_method not in AVAILABLE_TRAINING_METHODS:
|
|
128
|
+
raise ValueError(
|
|
129
|
+
f"training_method must be one of {', '.join(AVAILABLE_TRAINING_METHODS)}"
|
|
130
|
+
)
|
|
131
|
+
|
|
103
132
|
lrScheduler = FinetuneLRScheduler(
|
|
104
133
|
lr_scheduler_type="linear",
|
|
105
134
|
lr_scheduler_args=FinetuneLinearLRSchedulerArgs(min_lr_ratio=min_lr_ratio),
|
|
106
135
|
)
|
|
107
136
|
|
|
137
|
+
training_method_cls: TrainingMethodSFT | TrainingMethodDPO = TrainingMethodSFT()
|
|
138
|
+
if training_method == "dpo":
|
|
139
|
+
training_method_cls = TrainingMethodDPO(dpo_beta=dpo_beta)
|
|
140
|
+
|
|
108
141
|
finetune_request = FinetuneRequest(
|
|
109
142
|
model=model,
|
|
110
143
|
training_file=training_file,
|
|
@@ -125,11 +158,77 @@ def createFinetuneRequest(
|
|
|
125
158
|
wandb_project_name=wandb_project_name,
|
|
126
159
|
wandb_name=wandb_name,
|
|
127
160
|
train_on_inputs=train_on_inputs,
|
|
161
|
+
training_method=training_method_cls,
|
|
162
|
+
from_checkpoint=from_checkpoint,
|
|
128
163
|
)
|
|
129
164
|
|
|
130
165
|
return finetune_request
|
|
131
166
|
|
|
132
167
|
|
|
168
|
+
def _process_checkpoints_from_events(
|
|
169
|
+
events: List[FinetuneEvent], id: str
|
|
170
|
+
) -> List[FinetuneCheckpoint]:
|
|
171
|
+
"""
|
|
172
|
+
Helper function to process events and create checkpoint list.
|
|
173
|
+
|
|
174
|
+
Args:
|
|
175
|
+
events (List[FinetuneEvent]): List of fine-tune events to process
|
|
176
|
+
id (str): Fine-tune job ID
|
|
177
|
+
|
|
178
|
+
Returns:
|
|
179
|
+
List[FinetuneCheckpoint]: List of available checkpoints
|
|
180
|
+
"""
|
|
181
|
+
checkpoints: List[FinetuneCheckpoint] = []
|
|
182
|
+
|
|
183
|
+
for event in events:
|
|
184
|
+
event_type = event.type
|
|
185
|
+
|
|
186
|
+
if event_type == FinetuneEventType.CHECKPOINT_SAVE:
|
|
187
|
+
step = get_event_step(event)
|
|
188
|
+
checkpoint_name = f"{id}:{step}" if step is not None else id
|
|
189
|
+
|
|
190
|
+
checkpoints.append(
|
|
191
|
+
FinetuneCheckpoint(
|
|
192
|
+
type=(
|
|
193
|
+
f"Intermediate (step {step})"
|
|
194
|
+
if step is not None
|
|
195
|
+
else "Intermediate"
|
|
196
|
+
),
|
|
197
|
+
timestamp=event.created_at,
|
|
198
|
+
name=checkpoint_name,
|
|
199
|
+
)
|
|
200
|
+
)
|
|
201
|
+
elif event_type == FinetuneEventType.JOB_COMPLETE:
|
|
202
|
+
if hasattr(event, "model_path"):
|
|
203
|
+
checkpoints.append(
|
|
204
|
+
FinetuneCheckpoint(
|
|
205
|
+
type=(
|
|
206
|
+
"Final Merged"
|
|
207
|
+
if hasattr(event, "adapter_path")
|
|
208
|
+
else "Final"
|
|
209
|
+
),
|
|
210
|
+
timestamp=event.created_at,
|
|
211
|
+
name=id,
|
|
212
|
+
)
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
if hasattr(event, "adapter_path"):
|
|
216
|
+
checkpoints.append(
|
|
217
|
+
FinetuneCheckpoint(
|
|
218
|
+
type=(
|
|
219
|
+
"Final Adapter" if hasattr(event, "model_path") else "Final"
|
|
220
|
+
),
|
|
221
|
+
timestamp=event.created_at,
|
|
222
|
+
name=id,
|
|
223
|
+
)
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
# Sort by timestamp (newest first)
|
|
227
|
+
checkpoints.sort(key=lambda x: x.timestamp, reverse=True)
|
|
228
|
+
|
|
229
|
+
return checkpoints
|
|
230
|
+
|
|
231
|
+
|
|
133
232
|
class FineTuning:
|
|
134
233
|
def __init__(self, client: TogetherClient) -> None:
|
|
135
234
|
self._client = client
|
|
@@ -162,6 +261,9 @@ class FineTuning:
|
|
|
162
261
|
verbose: bool = False,
|
|
163
262
|
model_limits: FinetuneTrainingLimits | None = None,
|
|
164
263
|
train_on_inputs: bool | Literal["auto"] = "auto",
|
|
264
|
+
training_method: str = "sft",
|
|
265
|
+
dpo_beta: float | None = None,
|
|
266
|
+
from_checkpoint: str | None = None,
|
|
165
267
|
) -> FinetuneResponse:
|
|
166
268
|
"""
|
|
167
269
|
Method to initiate a fine-tuning job
|
|
@@ -207,6 +309,12 @@ class FineTuning:
|
|
|
207
309
|
For datasets with the "messages" field (conversational format) or "prompt" and "completion" fields
|
|
208
310
|
(Instruction format), inputs will be masked.
|
|
209
311
|
Defaults to "auto".
|
|
312
|
+
training_method (str, optional): Training method. Defaults to "sft".
|
|
313
|
+
Supported methods: "sft", "dpo".
|
|
314
|
+
dpo_beta (float, optional): DPO beta parameter. Defaults to None.
|
|
315
|
+
from_checkpoint (str, optional): The checkpoint identifier to continue training from a previous fine-tuning job.
|
|
316
|
+
The format: {$JOB_ID/$OUTPUT_MODEL_NAME}:{$STEP}.
|
|
317
|
+
The step value is optional, without it the final checkpoint will be used.
|
|
210
318
|
|
|
211
319
|
Returns:
|
|
212
320
|
FinetuneResponse: Object containing information about fine-tuning job.
|
|
@@ -218,7 +326,6 @@ class FineTuning:
|
|
|
218
326
|
|
|
219
327
|
if model_limits is None:
|
|
220
328
|
model_limits = self.get_model_limits(model=model)
|
|
221
|
-
|
|
222
329
|
finetune_request = createFinetuneRequest(
|
|
223
330
|
model_limits=model_limits,
|
|
224
331
|
training_file=training_file,
|
|
@@ -244,6 +351,9 @@ class FineTuning:
|
|
|
244
351
|
wandb_project_name=wandb_project_name,
|
|
245
352
|
wandb_name=wandb_name,
|
|
246
353
|
train_on_inputs=train_on_inputs,
|
|
354
|
+
training_method=training_method,
|
|
355
|
+
dpo_beta=dpo_beta,
|
|
356
|
+
from_checkpoint=from_checkpoint,
|
|
247
357
|
)
|
|
248
358
|
|
|
249
359
|
if verbose:
|
|
@@ -261,7 +371,6 @@ class FineTuning:
|
|
|
261
371
|
),
|
|
262
372
|
stream=False,
|
|
263
373
|
)
|
|
264
|
-
|
|
265
374
|
assert isinstance(response, TogetherResponse)
|
|
266
375
|
|
|
267
376
|
return FinetuneResponse(**response.data)
|
|
@@ -366,17 +475,29 @@ class FineTuning:
|
|
|
366
475
|
),
|
|
367
476
|
stream=False,
|
|
368
477
|
)
|
|
369
|
-
|
|
370
478
|
assert isinstance(response, TogetherResponse)
|
|
371
479
|
|
|
372
480
|
return FinetuneListEvents(**response.data)
|
|
373
481
|
|
|
482
|
+
def list_checkpoints(self, id: str) -> List[FinetuneCheckpoint]:
|
|
483
|
+
"""
|
|
484
|
+
List available checkpoints for a fine-tuning job
|
|
485
|
+
|
|
486
|
+
Args:
|
|
487
|
+
id (str): Unique identifier of the fine-tune job to list checkpoints for
|
|
488
|
+
|
|
489
|
+
Returns:
|
|
490
|
+
List[FinetuneCheckpoint]: List of available checkpoints
|
|
491
|
+
"""
|
|
492
|
+
events = self.list_events(id).data or []
|
|
493
|
+
return _process_checkpoints_from_events(events, id)
|
|
494
|
+
|
|
374
495
|
def download(
|
|
375
496
|
self,
|
|
376
497
|
id: str,
|
|
377
498
|
*,
|
|
378
499
|
output: Path | str | None = None,
|
|
379
|
-
checkpoint_step: int =
|
|
500
|
+
checkpoint_step: int | None = None,
|
|
380
501
|
checkpoint_type: DownloadCheckpointType = DownloadCheckpointType.DEFAULT,
|
|
381
502
|
) -> FinetuneDownloadResult:
|
|
382
503
|
"""
|
|
@@ -397,9 +518,19 @@ class FineTuning:
|
|
|
397
518
|
FinetuneDownloadResult: Object containing downloaded model metadata
|
|
398
519
|
"""
|
|
399
520
|
|
|
521
|
+
if re.match(_FT_JOB_WITH_STEP_REGEX, id) is not None:
|
|
522
|
+
if checkpoint_step is None:
|
|
523
|
+
checkpoint_step = int(id.split(":")[1])
|
|
524
|
+
id = id.split(":")[0]
|
|
525
|
+
else:
|
|
526
|
+
raise ValueError(
|
|
527
|
+
"Fine-tuning job ID {id} contains a colon to specify the step to download, but `checkpoint_step` "
|
|
528
|
+
"was also set. Remove one of the step specifiers to proceed."
|
|
529
|
+
)
|
|
530
|
+
|
|
400
531
|
url = f"finetune/download?ft_id={id}"
|
|
401
532
|
|
|
402
|
-
if checkpoint_step
|
|
533
|
+
if checkpoint_step is not None:
|
|
403
534
|
url += f"&checkpoint_step={checkpoint_step}"
|
|
404
535
|
|
|
405
536
|
ft_job = self.retrieve(id)
|
|
@@ -503,6 +634,9 @@ class AsyncFineTuning:
|
|
|
503
634
|
verbose: bool = False,
|
|
504
635
|
model_limits: FinetuneTrainingLimits | None = None,
|
|
505
636
|
train_on_inputs: bool | Literal["auto"] = "auto",
|
|
637
|
+
training_method: str = "sft",
|
|
638
|
+
dpo_beta: float | None = None,
|
|
639
|
+
from_checkpoint: str | None = None,
|
|
506
640
|
) -> FinetuneResponse:
|
|
507
641
|
"""
|
|
508
642
|
Async method to initiate a fine-tuning job
|
|
@@ -548,6 +682,12 @@ class AsyncFineTuning:
|
|
|
548
682
|
For datasets with the "messages" field (conversational format) or "prompt" and "completion" fields
|
|
549
683
|
(Instruction format), inputs will be masked.
|
|
550
684
|
Defaults to "auto".
|
|
685
|
+
training_method (str, optional): Training method. Defaults to "sft".
|
|
686
|
+
Supported methods: "sft", "dpo".
|
|
687
|
+
dpo_beta (float, optional): DPO beta parameter. Defaults to None.
|
|
688
|
+
from_checkpoint (str, optional): The checkpoint identifier to continue training from a previous fine-tuning job.
|
|
689
|
+
The format: {$JOB_ID/$OUTPUT_MODEL_NAME}:{$STEP}.
|
|
690
|
+
The step value is optional, without it the final checkpoint will be used.
|
|
551
691
|
|
|
552
692
|
Returns:
|
|
553
693
|
FinetuneResponse: Object containing information about fine-tuning job.
|
|
@@ -585,6 +725,9 @@ class AsyncFineTuning:
|
|
|
585
725
|
wandb_project_name=wandb_project_name,
|
|
586
726
|
wandb_name=wandb_name,
|
|
587
727
|
train_on_inputs=train_on_inputs,
|
|
728
|
+
training_method=training_method,
|
|
729
|
+
dpo_beta=dpo_beta,
|
|
730
|
+
from_checkpoint=from_checkpoint,
|
|
588
731
|
)
|
|
589
732
|
|
|
590
733
|
if verbose:
|
|
@@ -687,30 +830,45 @@ class AsyncFineTuning:
|
|
|
687
830
|
|
|
688
831
|
async def list_events(self, id: str) -> FinetuneListEvents:
|
|
689
832
|
"""
|
|
690
|
-
|
|
833
|
+
List fine-tuning events
|
|
691
834
|
|
|
692
835
|
Args:
|
|
693
|
-
id (str):
|
|
836
|
+
id (str): Unique identifier of the fine-tune job to list events for
|
|
694
837
|
|
|
695
838
|
Returns:
|
|
696
|
-
FinetuneListEvents: Object containing list of fine-tune events
|
|
839
|
+
FinetuneListEvents: Object containing list of fine-tune job events
|
|
697
840
|
"""
|
|
698
841
|
|
|
699
842
|
requestor = api_requestor.APIRequestor(
|
|
700
843
|
client=self._client,
|
|
701
844
|
)
|
|
702
845
|
|
|
703
|
-
|
|
846
|
+
events_response, _, _ = await requestor.arequest(
|
|
704
847
|
options=TogetherRequest(
|
|
705
848
|
method="GET",
|
|
706
|
-
url=f"fine-tunes/{id}/events",
|
|
849
|
+
url=f"fine-tunes/{normalize_key(id)}/events",
|
|
707
850
|
),
|
|
708
851
|
stream=False,
|
|
709
852
|
)
|
|
710
853
|
|
|
711
|
-
|
|
854
|
+
# FIXME: API returns "data" field with no object type (should be "list")
|
|
855
|
+
events_list = FinetuneListEvents(object="list", **events_response.data)
|
|
712
856
|
|
|
713
|
-
return
|
|
857
|
+
return events_list
|
|
858
|
+
|
|
859
|
+
async def list_checkpoints(self, id: str) -> List[FinetuneCheckpoint]:
|
|
860
|
+
"""
|
|
861
|
+
List available checkpoints for a fine-tuning job
|
|
862
|
+
|
|
863
|
+
Args:
|
|
864
|
+
id (str): Unique identifier of the fine-tune job to list checkpoints for
|
|
865
|
+
|
|
866
|
+
Returns:
|
|
867
|
+
List[FinetuneCheckpoint]: Object containing list of available checkpoints
|
|
868
|
+
"""
|
|
869
|
+
events_list = await self.list_events(id)
|
|
870
|
+
events = events_list.data or []
|
|
871
|
+
return _process_checkpoints_from_events(events, id)
|
|
714
872
|
|
|
715
873
|
async def download(
|
|
716
874
|
self, id: str, *, output: str | None = None, checkpoint_step: int = -1
|
together/types/__init__.py
CHANGED
|
@@ -31,6 +31,9 @@ from together.types.files import (
|
|
|
31
31
|
FileType,
|
|
32
32
|
)
|
|
33
33
|
from together.types.finetune import (
|
|
34
|
+
TrainingMethodDPO,
|
|
35
|
+
TrainingMethodSFT,
|
|
36
|
+
FinetuneCheckpoint,
|
|
34
37
|
FinetuneDownloadResult,
|
|
35
38
|
FinetuneLinearLRSchedulerArgs,
|
|
36
39
|
FinetuneList,
|
|
@@ -59,6 +62,7 @@ __all__ = [
|
|
|
59
62
|
"ChatCompletionResponse",
|
|
60
63
|
"EmbeddingRequest",
|
|
61
64
|
"EmbeddingResponse",
|
|
65
|
+
"FinetuneCheckpoint",
|
|
62
66
|
"FinetuneRequest",
|
|
63
67
|
"FinetuneResponse",
|
|
64
68
|
"FinetuneList",
|
|
@@ -79,6 +83,8 @@ __all__ = [
|
|
|
79
83
|
"TrainingType",
|
|
80
84
|
"FullTrainingType",
|
|
81
85
|
"LoRATrainingType",
|
|
86
|
+
"TrainingMethodDPO",
|
|
87
|
+
"TrainingMethodSFT",
|
|
82
88
|
"RerankRequest",
|
|
83
89
|
"RerankResponse",
|
|
84
90
|
"FinetuneTrainingLimits",
|
|
@@ -44,16 +44,22 @@ class ToolCalls(BaseModel):
|
|
|
44
44
|
class ChatCompletionMessageContentType(str, Enum):
|
|
45
45
|
TEXT = "text"
|
|
46
46
|
IMAGE_URL = "image_url"
|
|
47
|
+
VIDEO_URL = "video_url"
|
|
47
48
|
|
|
48
49
|
|
|
49
50
|
class ChatCompletionMessageContentImageURL(BaseModel):
|
|
50
51
|
url: str
|
|
51
52
|
|
|
52
53
|
|
|
54
|
+
class ChatCompletionMessageContentVideoURL(BaseModel):
|
|
55
|
+
url: str
|
|
56
|
+
|
|
57
|
+
|
|
53
58
|
class ChatCompletionMessageContent(BaseModel):
|
|
54
59
|
type: ChatCompletionMessageContentType
|
|
55
60
|
text: str | None = None
|
|
56
61
|
image_url: ChatCompletionMessageContentImageURL | None = None
|
|
62
|
+
video_url: ChatCompletionMessageContentVideoURL | None = None
|
|
57
63
|
|
|
58
64
|
|
|
59
65
|
class ChatCompletionMessage(BaseModel):
|
together/types/endpoints.py
CHANGED
|
@@ -86,9 +86,9 @@ class BaseEndpoint(TogetherJSONModel):
|
|
|
86
86
|
model: str = Field(description="The model deployed on this endpoint")
|
|
87
87
|
type: str = Field(description="The type of endpoint")
|
|
88
88
|
owner: str = Field(description="The owner of this endpoint")
|
|
89
|
-
state: Literal[
|
|
90
|
-
|
|
91
|
-
)
|
|
89
|
+
state: Literal[
|
|
90
|
+
"PENDING", "STARTING", "STARTED", "STOPPING", "STOPPED", "FAILED", "ERROR"
|
|
91
|
+
] = Field(description="Current state of the endpoint")
|
|
92
92
|
created_at: datetime = Field(description="Timestamp when the endpoint was created")
|
|
93
93
|
|
|
94
94
|
|
together/types/finetune.py
CHANGED
|
@@ -135,6 +135,31 @@ class LoRATrainingType(TrainingType):
|
|
|
135
135
|
type: str = "Lora"
|
|
136
136
|
|
|
137
137
|
|
|
138
|
+
class TrainingMethod(BaseModel):
|
|
139
|
+
"""
|
|
140
|
+
Training method type
|
|
141
|
+
"""
|
|
142
|
+
|
|
143
|
+
method: str
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
class TrainingMethodSFT(TrainingMethod):
|
|
147
|
+
"""
|
|
148
|
+
Training method type for SFT training
|
|
149
|
+
"""
|
|
150
|
+
|
|
151
|
+
method: Literal["sft"] = "sft"
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
class TrainingMethodDPO(TrainingMethod):
|
|
155
|
+
"""
|
|
156
|
+
Training method type for DPO training
|
|
157
|
+
"""
|
|
158
|
+
|
|
159
|
+
method: Literal["dpo"] = "dpo"
|
|
160
|
+
dpo_beta: float | None = None
|
|
161
|
+
|
|
162
|
+
|
|
138
163
|
class FinetuneRequest(BaseModel):
|
|
139
164
|
"""
|
|
140
165
|
Fine-tune request type
|
|
@@ -178,6 +203,12 @@ class FinetuneRequest(BaseModel):
|
|
|
178
203
|
training_type: FullTrainingType | LoRATrainingType | None = None
|
|
179
204
|
# train on inputs
|
|
180
205
|
train_on_inputs: StrictBool | Literal["auto"] = "auto"
|
|
206
|
+
# training method
|
|
207
|
+
training_method: TrainingMethodSFT | TrainingMethodDPO = Field(
|
|
208
|
+
default_factory=TrainingMethodSFT
|
|
209
|
+
)
|
|
210
|
+
# from step
|
|
211
|
+
from_checkpoint: str | None = None
|
|
181
212
|
|
|
182
213
|
|
|
183
214
|
class FinetuneResponse(BaseModel):
|
|
@@ -256,6 +287,7 @@ class FinetuneResponse(BaseModel):
|
|
|
256
287
|
training_file_num_lines: int | None = Field(None, alias="TrainingFileNumLines")
|
|
257
288
|
training_file_size: int | None = Field(None, alias="TrainingFileSize")
|
|
258
289
|
train_on_inputs: StrictBool | Literal["auto"] | None = "auto"
|
|
290
|
+
from_checkpoint: str | None = None
|
|
259
291
|
|
|
260
292
|
@field_validator("training_type")
|
|
261
293
|
@classmethod
|
|
@@ -320,3 +352,16 @@ class FinetuneLRScheduler(BaseModel):
|
|
|
320
352
|
|
|
321
353
|
class FinetuneLinearLRSchedulerArgs(BaseModel):
|
|
322
354
|
min_lr_ratio: float | None = 0.0
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
class FinetuneCheckpoint(BaseModel):
|
|
358
|
+
"""
|
|
359
|
+
Fine-tuning checkpoint information
|
|
360
|
+
"""
|
|
361
|
+
|
|
362
|
+
# checkpoint type (e.g. "Intermediate", "Final", "Final Merged", "Final Adapter")
|
|
363
|
+
type: str
|
|
364
|
+
# timestamp when the checkpoint was created
|
|
365
|
+
timestamp: str
|
|
366
|
+
# checkpoint name/identifier
|
|
367
|
+
name: str
|
together/utils/__init__.py
CHANGED
|
@@ -8,6 +8,8 @@ from together.utils.tools import (
|
|
|
8
8
|
finetune_price_to_dollars,
|
|
9
9
|
normalize_key,
|
|
10
10
|
parse_timestamp,
|
|
11
|
+
format_timestamp,
|
|
12
|
+
get_event_step,
|
|
11
13
|
)
|
|
12
14
|
|
|
13
15
|
|
|
@@ -23,6 +25,8 @@ __all__ = [
|
|
|
23
25
|
"enforce_trailing_slash",
|
|
24
26
|
"normalize_key",
|
|
25
27
|
"parse_timestamp",
|
|
28
|
+
"format_timestamp",
|
|
29
|
+
"get_event_step",
|
|
26
30
|
"finetune_price_to_dollars",
|
|
27
31
|
"convert_bytes",
|
|
28
32
|
"convert_unix_timestamp",
|
together/utils/files.py
CHANGED
|
@@ -4,7 +4,7 @@ import json
|
|
|
4
4
|
import os
|
|
5
5
|
from pathlib import Path
|
|
6
6
|
from traceback import format_exc
|
|
7
|
-
from typing import Any, Dict
|
|
7
|
+
from typing import Any, Dict, List
|
|
8
8
|
|
|
9
9
|
from pyarrow import ArrowInvalid, parquet
|
|
10
10
|
|
|
@@ -96,6 +96,140 @@ def check_file(
|
|
|
96
96
|
return report_dict
|
|
97
97
|
|
|
98
98
|
|
|
99
|
+
def validate_messages(messages: List[Dict[str, str | bool]], idx: int) -> None:
|
|
100
|
+
"""Validate the messages column."""
|
|
101
|
+
if not isinstance(messages, list):
|
|
102
|
+
raise InvalidFileFormatError(
|
|
103
|
+
message=f"Invalid format on line {idx + 1} of the input file. "
|
|
104
|
+
f"Expected a list of messages. Found {type(messages)}",
|
|
105
|
+
line_number=idx + 1,
|
|
106
|
+
error_source="key_value",
|
|
107
|
+
)
|
|
108
|
+
if not messages:
|
|
109
|
+
raise InvalidFileFormatError(
|
|
110
|
+
message=f"Invalid format on line {idx + 1} of the input file. "
|
|
111
|
+
f"Expected a non-empty list of messages. Found empty list",
|
|
112
|
+
line_number=idx + 1,
|
|
113
|
+
error_source="key_value",
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
has_weights = any("weight" in message for message in messages)
|
|
117
|
+
|
|
118
|
+
previous_role = None
|
|
119
|
+
for message in messages:
|
|
120
|
+
if not isinstance(message, dict):
|
|
121
|
+
raise InvalidFileFormatError(
|
|
122
|
+
message=f"Invalid format on line {idx + 1} of the input file. "
|
|
123
|
+
f"Expected a dictionary in the messages list. Found {type(message)}",
|
|
124
|
+
line_number=idx + 1,
|
|
125
|
+
error_source="key_value",
|
|
126
|
+
)
|
|
127
|
+
for column in REQUIRED_COLUMNS_MESSAGE:
|
|
128
|
+
if column not in message:
|
|
129
|
+
raise InvalidFileFormatError(
|
|
130
|
+
message=f"Field `{column}` is missing for a turn `{message}` on line {idx + 1} "
|
|
131
|
+
"of the the input file.",
|
|
132
|
+
line_number=idx + 1,
|
|
133
|
+
error_source="key_value",
|
|
134
|
+
)
|
|
135
|
+
else:
|
|
136
|
+
if not isinstance(message[column], str):
|
|
137
|
+
raise InvalidFileFormatError(
|
|
138
|
+
message=f"Invalid format on line {idx + 1} in the column {column} for turn `{message}` "
|
|
139
|
+
f"of the input file. Expected string. Found {type(message[column])}",
|
|
140
|
+
line_number=idx + 1,
|
|
141
|
+
error_source="text_field",
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
if has_weights and "weight" in message:
|
|
145
|
+
weight = message["weight"]
|
|
146
|
+
if not isinstance(weight, int):
|
|
147
|
+
raise InvalidFileFormatError(
|
|
148
|
+
message="Weight must be an integer",
|
|
149
|
+
line_number=idx + 1,
|
|
150
|
+
error_source="key_value",
|
|
151
|
+
)
|
|
152
|
+
if weight not in {0, 1}:
|
|
153
|
+
raise InvalidFileFormatError(
|
|
154
|
+
message="Weight must be either 0 or 1",
|
|
155
|
+
line_number=idx + 1,
|
|
156
|
+
error_source="key_value",
|
|
157
|
+
)
|
|
158
|
+
if message["role"] not in POSSIBLE_ROLES_CONVERSATION:
|
|
159
|
+
raise InvalidFileFormatError(
|
|
160
|
+
message=f"Found invalid role `{message['role']}` in the messages on the line {idx + 1}. "
|
|
161
|
+
f"Possible roles in the conversation are: {POSSIBLE_ROLES_CONVERSATION}",
|
|
162
|
+
line_number=idx + 1,
|
|
163
|
+
error_source="key_value",
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
if previous_role == message["role"]:
|
|
167
|
+
raise InvalidFileFormatError(
|
|
168
|
+
message=f"Invalid role turns on line {idx + 1} of the input file. "
|
|
169
|
+
"`user` and `assistant` roles must alternate user/assistant/user/assistant/...",
|
|
170
|
+
line_number=idx + 1,
|
|
171
|
+
error_source="key_value",
|
|
172
|
+
)
|
|
173
|
+
previous_role = message["role"]
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def validate_preference_openai(example: Dict[str, Any], idx: int = 0) -> None:
|
|
177
|
+
"""Validate the OpenAI preference dataset format.
|
|
178
|
+
|
|
179
|
+
Args:
|
|
180
|
+
example (dict): Input entry to be checked.
|
|
181
|
+
idx (int): Line number in the file.
|
|
182
|
+
|
|
183
|
+
Raises:
|
|
184
|
+
InvalidFileFormatError: If the dataset format is invalid.
|
|
185
|
+
"""
|
|
186
|
+
if not isinstance(example["input"], dict):
|
|
187
|
+
raise InvalidFileFormatError(
|
|
188
|
+
message="The dataset is malformed, the `input` field must be a dictionary.",
|
|
189
|
+
line_number=idx + 1,
|
|
190
|
+
error_source="key_value",
|
|
191
|
+
)
|
|
192
|
+
|
|
193
|
+
if "messages" not in example["input"]:
|
|
194
|
+
raise InvalidFileFormatError(
|
|
195
|
+
message="The dataset is malformed, the `input` dictionary must contain a `messages` field.",
|
|
196
|
+
line_number=idx + 1,
|
|
197
|
+
error_source="key_value",
|
|
198
|
+
)
|
|
199
|
+
|
|
200
|
+
validate_messages(example["input"]["messages"], idx)
|
|
201
|
+
|
|
202
|
+
for output_field in ["preferred_output", "non_preferred_output"]:
|
|
203
|
+
if not isinstance(example[output_field], list):
|
|
204
|
+
raise InvalidFileFormatError(
|
|
205
|
+
message=f"The dataset is malformed, the `{output_field}` field must be a list.",
|
|
206
|
+
line_number=idx + 1,
|
|
207
|
+
error_source="key_value",
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
if len(example[output_field]) != 1:
|
|
211
|
+
raise InvalidFileFormatError(
|
|
212
|
+
message=f"The dataset is malformed, the `{output_field}` list must contain exactly one message.",
|
|
213
|
+
line_number=idx + 1,
|
|
214
|
+
error_source="key_value",
|
|
215
|
+
)
|
|
216
|
+
if "role" not in example[output_field][0]:
|
|
217
|
+
raise InvalidFileFormatError(
|
|
218
|
+
message=f"The dataset is malformed, the `{output_field}` message is missing the `role` field.",
|
|
219
|
+
line_number=idx + 1,
|
|
220
|
+
error_source="key_value",
|
|
221
|
+
)
|
|
222
|
+
elif example[output_field][0]["role"] != "assistant":
|
|
223
|
+
raise InvalidFileFormatError(
|
|
224
|
+
message=f"The dataset is malformed, the `{output_field}` must contain an assistant message.",
|
|
225
|
+
line_number=idx + 1,
|
|
226
|
+
error_source="key_value",
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
validate_messages(example["preferred_output"], idx)
|
|
230
|
+
validate_messages(example["non_preferred_output"], idx)
|
|
231
|
+
|
|
232
|
+
|
|
99
233
|
def _check_jsonl(file: Path) -> Dict[str, Any]:
|
|
100
234
|
report_dict: Dict[str, Any] = {}
|
|
101
235
|
# Check that the file is UTF-8 encoded. If not report where the error occurs.
|
|
@@ -164,74 +298,13 @@ def _check_jsonl(file: Path) -> Dict[str, Any]:
|
|
|
164
298
|
line_number=idx + 1,
|
|
165
299
|
error_source="format",
|
|
166
300
|
)
|
|
167
|
-
|
|
168
|
-
|
|
301
|
+
if current_format == DatasetFormat.PREFERENCE_OPENAI:
|
|
302
|
+
validate_preference_openai(json_line, idx)
|
|
303
|
+
elif current_format == DatasetFormat.CONVERSATION:
|
|
169
304
|
message_column = JSONL_REQUIRED_COLUMNS_MAP[
|
|
170
305
|
DatasetFormat.CONVERSATION
|
|
171
306
|
][0]
|
|
172
|
-
|
|
173
|
-
raise InvalidFileFormatError(
|
|
174
|
-
message=f"Invalid format on line {idx + 1} of the input file. "
|
|
175
|
-
f"Expected a list of messages. Found {type(json_line[message_column])}",
|
|
176
|
-
line_number=idx + 1,
|
|
177
|
-
error_source="key_value",
|
|
178
|
-
)
|
|
179
|
-
|
|
180
|
-
if len(json_line[message_column]) == 0:
|
|
181
|
-
raise InvalidFileFormatError(
|
|
182
|
-
message=f"Invalid format on line {idx + 1} of the input file. "
|
|
183
|
-
f"Expected a non-empty list of messages. Found empty list",
|
|
184
|
-
line_number=idx + 1,
|
|
185
|
-
error_source="key_value",
|
|
186
|
-
)
|
|
187
|
-
|
|
188
|
-
for turn_id, turn in enumerate(json_line[message_column]):
|
|
189
|
-
if not isinstance(turn, dict):
|
|
190
|
-
raise InvalidFileFormatError(
|
|
191
|
-
message=f"Invalid format on line {idx + 1} of the input file. "
|
|
192
|
-
f"Expected a dictionary in the {turn_id + 1} turn. Found {type(turn)}",
|
|
193
|
-
line_number=idx + 1,
|
|
194
|
-
error_source="key_value",
|
|
195
|
-
)
|
|
196
|
-
|
|
197
|
-
previous_role = None
|
|
198
|
-
for turn in json_line[message_column]:
|
|
199
|
-
for column in REQUIRED_COLUMNS_MESSAGE:
|
|
200
|
-
if column not in turn:
|
|
201
|
-
raise InvalidFileFormatError(
|
|
202
|
-
message=f"Field `{column}` is missing for a turn `{turn}` on line {idx + 1} "
|
|
203
|
-
"of the the input file.",
|
|
204
|
-
line_number=idx + 1,
|
|
205
|
-
error_source="key_value",
|
|
206
|
-
)
|
|
207
|
-
else:
|
|
208
|
-
if not isinstance(turn[column], str):
|
|
209
|
-
raise InvalidFileFormatError(
|
|
210
|
-
message=f"Invalid format on line {idx + 1} in the column {column} for turn `{turn}` "
|
|
211
|
-
f"of the input file. Expected string. Found {type(turn[column])}",
|
|
212
|
-
line_number=idx + 1,
|
|
213
|
-
error_source="text_field",
|
|
214
|
-
)
|
|
215
|
-
role = turn["role"]
|
|
216
|
-
|
|
217
|
-
if role not in POSSIBLE_ROLES_CONVERSATION:
|
|
218
|
-
raise InvalidFileFormatError(
|
|
219
|
-
message=f"Found invalid role `{role}` in the messages on the line {idx + 1}. "
|
|
220
|
-
f"Possible roles in the conversation are: {POSSIBLE_ROLES_CONVERSATION}",
|
|
221
|
-
line_number=idx + 1,
|
|
222
|
-
error_source="key_value",
|
|
223
|
-
)
|
|
224
|
-
|
|
225
|
-
if previous_role == role:
|
|
226
|
-
raise InvalidFileFormatError(
|
|
227
|
-
message=f"Invalid role turns on line {idx + 1} of the input file. "
|
|
228
|
-
"`user` and `assistant` roles must alternate user/assistant/user/assistant/...",
|
|
229
|
-
line_number=idx + 1,
|
|
230
|
-
error_source="key_value",
|
|
231
|
-
)
|
|
232
|
-
|
|
233
|
-
previous_role = role
|
|
234
|
-
|
|
307
|
+
validate_messages(json_line[message_column], idx)
|
|
235
308
|
else:
|
|
236
309
|
for column in JSONL_REQUIRED_COLUMNS_MAP[current_format]:
|
|
237
310
|
if not isinstance(json_line[column], str):
|
together/utils/tools.py
CHANGED
|
@@ -3,6 +3,8 @@ from __future__ import annotations
|
|
|
3
3
|
import logging
|
|
4
4
|
import os
|
|
5
5
|
from datetime import datetime
|
|
6
|
+
import re
|
|
7
|
+
from typing import Any
|
|
6
8
|
|
|
7
9
|
|
|
8
10
|
logger = logging.getLogger("together")
|
|
@@ -23,18 +25,67 @@ def normalize_key(key: str) -> str:
|
|
|
23
25
|
return key.replace("/", "--").replace("_", "-").replace(" ", "-").lower()
|
|
24
26
|
|
|
25
27
|
|
|
26
|
-
def parse_timestamp(timestamp: str) -> datetime:
|
|
28
|
+
def parse_timestamp(timestamp: str) -> datetime | None:
|
|
29
|
+
"""Parse a timestamp string into a datetime object or None if the string is empty.
|
|
30
|
+
|
|
31
|
+
Args:
|
|
32
|
+
timestamp (str): Timestamp
|
|
33
|
+
|
|
34
|
+
Returns:
|
|
35
|
+
datetime | None: Parsed datetime, or None if the string is empty
|
|
36
|
+
"""
|
|
37
|
+
if timestamp == "":
|
|
38
|
+
return None
|
|
39
|
+
|
|
27
40
|
formats = ["%Y-%m-%dT%H:%M:%S.%fZ", "%Y-%m-%dT%H:%M:%SZ"]
|
|
28
41
|
for fmt in formats:
|
|
29
42
|
try:
|
|
30
43
|
return datetime.strptime(timestamp, fmt)
|
|
31
44
|
except ValueError:
|
|
32
45
|
continue
|
|
46
|
+
|
|
33
47
|
raise ValueError("Timestamp does not match any expected format")
|
|
34
48
|
|
|
35
49
|
|
|
36
|
-
|
|
50
|
+
def format_timestamp(timestamp_str: str) -> str:
|
|
51
|
+
"""Format timestamp to a readable date string.
|
|
52
|
+
|
|
53
|
+
Args:
|
|
54
|
+
timestamp: A timestamp string
|
|
55
|
+
|
|
56
|
+
Returns:
|
|
57
|
+
str: Formatted timestamp string (MM/DD/YYYY, HH:MM AM/PM)
|
|
58
|
+
"""
|
|
59
|
+
timestamp = parse_timestamp(timestamp_str)
|
|
60
|
+
if timestamp is None:
|
|
61
|
+
return ""
|
|
62
|
+
return timestamp.strftime("%m/%d/%Y, %I:%M %p")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def get_event_step(event: Any) -> str | None:
|
|
66
|
+
"""Extract the step number from a checkpoint event.
|
|
67
|
+
|
|
68
|
+
Args:
|
|
69
|
+
event: A checkpoint event object
|
|
70
|
+
|
|
71
|
+
Returns:
|
|
72
|
+
str | None: The step number as a string, or None if not found
|
|
73
|
+
"""
|
|
74
|
+
step = getattr(event, "step", None)
|
|
75
|
+
if step is not None:
|
|
76
|
+
return str(step)
|
|
77
|
+
return None
|
|
78
|
+
|
|
79
|
+
|
|
37
80
|
def finetune_price_to_dollars(price: float) -> float:
|
|
81
|
+
"""Convert fine-tuning job price to dollars
|
|
82
|
+
|
|
83
|
+
Args:
|
|
84
|
+
price (float): Fine-tuning job price in billing units
|
|
85
|
+
|
|
86
|
+
Returns:
|
|
87
|
+
float: Price in dollars
|
|
88
|
+
"""
|
|
38
89
|
return price / NANODOLLAR
|
|
39
90
|
|
|
40
91
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: together
|
|
3
|
-
Version: 1.4.
|
|
3
|
+
Version: 1.4.4
|
|
4
4
|
Summary: Python client for Together's Cloud Platform!
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
Author: Together AI
|
|
@@ -87,25 +87,101 @@ This repo contains both a Python Library and a CLI. We'll demonstrate how to use
|
|
|
87
87
|
### Chat Completions
|
|
88
88
|
|
|
89
89
|
```python
|
|
90
|
-
import os
|
|
91
90
|
from together import Together
|
|
92
91
|
|
|
93
|
-
client = Together(
|
|
92
|
+
client = Together()
|
|
94
93
|
|
|
94
|
+
# Simple text message
|
|
95
95
|
response = client.chat.completions.create(
|
|
96
96
|
model="mistralai/Mixtral-8x7B-Instruct-v0.1",
|
|
97
97
|
messages=[{"role": "user", "content": "tell me about new york"}],
|
|
98
98
|
)
|
|
99
99
|
print(response.choices[0].message.content)
|
|
100
|
+
|
|
101
|
+
# Multi-modal message with text and image
|
|
102
|
+
response = client.chat.completions.create(
|
|
103
|
+
model="meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo",
|
|
104
|
+
messages=[{
|
|
105
|
+
"role": "user",
|
|
106
|
+
"content": [
|
|
107
|
+
{
|
|
108
|
+
"type": "text",
|
|
109
|
+
"text": "What's in this image?"
|
|
110
|
+
},
|
|
111
|
+
{
|
|
112
|
+
"type": "image_url",
|
|
113
|
+
"image_url": {
|
|
114
|
+
"url": "https://huggingface.co/datasets/patrickvonplaten/random_img/resolve/main/yosemite.png"
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
]
|
|
118
|
+
}]
|
|
119
|
+
)
|
|
120
|
+
print(response.choices[0].message.content)
|
|
121
|
+
|
|
122
|
+
# Multi-modal message with multiple images
|
|
123
|
+
response = client.chat.completions.create(
|
|
124
|
+
model="Qwen/Qwen2.5-VL-72B-Instruct",
|
|
125
|
+
messages=[{
|
|
126
|
+
"role": "user",
|
|
127
|
+
"content": [
|
|
128
|
+
{
|
|
129
|
+
"type": "text",
|
|
130
|
+
"text": "Compare these two images."
|
|
131
|
+
},
|
|
132
|
+
{
|
|
133
|
+
"type": "image_url",
|
|
134
|
+
"image_url": {
|
|
135
|
+
"url": "https://huggingface.co/datasets/patrickvonplaten/random_img/resolve/main/yosemite.png"
|
|
136
|
+
}
|
|
137
|
+
},
|
|
138
|
+
{
|
|
139
|
+
"type": "image_url",
|
|
140
|
+
"image_url": {
|
|
141
|
+
"url": "https://huggingface.co/datasets/patrickvonplaten/random_img/resolve/main/slack.png"
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
]
|
|
145
|
+
}]
|
|
146
|
+
)
|
|
147
|
+
print(response.choices[0].message.content)
|
|
148
|
+
|
|
149
|
+
# Multi-modal message with text and video
|
|
150
|
+
response = client.chat.completions.create(
|
|
151
|
+
model="Qwen/Qwen2.5-VL-72B-Instruct",
|
|
152
|
+
messages=[{
|
|
153
|
+
"role": "user",
|
|
154
|
+
"content": [
|
|
155
|
+
{
|
|
156
|
+
"type": "text",
|
|
157
|
+
"text": "What's happening in this video?"
|
|
158
|
+
},
|
|
159
|
+
{
|
|
160
|
+
"type": "video_url",
|
|
161
|
+
"video_url": {
|
|
162
|
+
"url": "http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerFun.mp4"
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
]
|
|
166
|
+
}]
|
|
167
|
+
)
|
|
168
|
+
print(response.choices[0].message.content)
|
|
100
169
|
```
|
|
101
170
|
|
|
171
|
+
The chat completions API supports three types of content:
|
|
172
|
+
- Plain text messages using the `content` field directly
|
|
173
|
+
- Multi-modal messages with images using `type: "image_url"`
|
|
174
|
+
- Multi-modal messages with videos using `type: "video_url"`
|
|
175
|
+
|
|
176
|
+
When using multi-modal content, the `content` field becomes an array of content objects, each with its own type and corresponding data.
|
|
177
|
+
|
|
102
178
|
#### Streaming
|
|
103
179
|
|
|
104
180
|
```python
|
|
105
181
|
import os
|
|
106
182
|
from together import Together
|
|
107
183
|
|
|
108
|
-
client = Together(
|
|
184
|
+
client = Together()
|
|
109
185
|
stream = client.chat.completions.create(
|
|
110
186
|
model="mistralai/Mixtral-8x7B-Instruct-v0.1",
|
|
111
187
|
messages=[{"role": "user", "content": "tell me about new york"}],
|
|
@@ -119,17 +195,17 @@ for chunk in stream:
|
|
|
119
195
|
#### Async usage
|
|
120
196
|
|
|
121
197
|
```python
|
|
122
|
-
import
|
|
198
|
+
import asyncio
|
|
123
199
|
from together import AsyncTogether
|
|
124
200
|
|
|
125
|
-
async_client = AsyncTogether(
|
|
201
|
+
async_client = AsyncTogether()
|
|
126
202
|
messages = [
|
|
127
203
|
"What are the top things to do in San Francisco?",
|
|
128
204
|
"What country is Paris in?",
|
|
129
205
|
]
|
|
130
206
|
|
|
131
207
|
async def async_chat_completion(messages):
|
|
132
|
-
async_client = AsyncTogether(
|
|
208
|
+
async_client = AsyncTogether()
|
|
133
209
|
tasks = [
|
|
134
210
|
async_client.chat.completions.create(
|
|
135
211
|
model="mistralai/Mixtral-8x7B-Instruct-v0.1",
|
|
@@ -150,10 +226,9 @@ asyncio.run(async_chat_completion(messages))
|
|
|
150
226
|
Completions are for code and language models shown [here](https://docs.together.ai/docs/inference-models). Below, a code model example is shown.
|
|
151
227
|
|
|
152
228
|
```python
|
|
153
|
-
import os
|
|
154
229
|
from together import Together
|
|
155
230
|
|
|
156
|
-
client = Together(
|
|
231
|
+
client = Together()
|
|
157
232
|
|
|
158
233
|
response = client.completions.create(
|
|
159
234
|
model="codellama/CodeLlama-34b-Python-hf",
|
|
@@ -166,10 +241,9 @@ print(response.choices[0].text)
|
|
|
166
241
|
#### Streaming
|
|
167
242
|
|
|
168
243
|
```python
|
|
169
|
-
import os
|
|
170
244
|
from together import Together
|
|
171
245
|
|
|
172
|
-
client = Together(
|
|
246
|
+
client = Together()
|
|
173
247
|
stream = client.completions.create(
|
|
174
248
|
model="codellama/CodeLlama-34b-Python-hf",
|
|
175
249
|
prompt="Write a Next.js component with TailwindCSS for a header component.",
|
|
@@ -183,10 +257,10 @@ for chunk in stream:
|
|
|
183
257
|
#### Async usage
|
|
184
258
|
|
|
185
259
|
```python
|
|
186
|
-
import
|
|
260
|
+
import asyncio
|
|
187
261
|
from together import AsyncTogether
|
|
188
262
|
|
|
189
|
-
async_client = AsyncTogether(
|
|
263
|
+
async_client = AsyncTogether()
|
|
190
264
|
prompts = [
|
|
191
265
|
"Write a Next.js component with TailwindCSS for a header component.",
|
|
192
266
|
"Write a python function for the fibonacci sequence",
|
|
@@ -211,10 +285,9 @@ asyncio.run(async_chat_completion(prompts))
|
|
|
211
285
|
### Image generation
|
|
212
286
|
|
|
213
287
|
```python
|
|
214
|
-
import os
|
|
215
288
|
from together import Together
|
|
216
289
|
|
|
217
|
-
client = Together(
|
|
290
|
+
client = Together()
|
|
218
291
|
|
|
219
292
|
response = client.images.generate(
|
|
220
293
|
prompt="space robots",
|
|
@@ -231,7 +304,7 @@ print(response.data[0].b64_json)
|
|
|
231
304
|
from typing import List
|
|
232
305
|
from together import Together
|
|
233
306
|
|
|
234
|
-
client = Together(
|
|
307
|
+
client = Together()
|
|
235
308
|
|
|
236
309
|
def get_embeddings(texts: List[str], model: str) -> List[List[float]]:
|
|
237
310
|
texts = [text.replace("\n", " ") for text in texts]
|
|
@@ -250,7 +323,7 @@ print(embeddings)
|
|
|
250
323
|
from typing import List
|
|
251
324
|
from together import Together
|
|
252
325
|
|
|
253
|
-
client = Together(
|
|
326
|
+
client = Together()
|
|
254
327
|
|
|
255
328
|
def get_reranked_documents(query: str, documents: List[str], model: str, top_n: int = 3) -> List[str]:
|
|
256
329
|
outputs = client.rerank.create(model=model, query=query, documents=documents, top_n=top_n)
|
|
@@ -272,10 +345,9 @@ Read more about Reranking [here](https://docs.together.ai/docs/rerank-overview).
|
|
|
272
345
|
The files API is used for fine-tuning and allows developers to upload data to fine-tune on. It also has several methods to list all files, retrive files, and delete files. Please refer to our fine-tuning docs [here](https://docs.together.ai/docs/fine-tuning-python).
|
|
273
346
|
|
|
274
347
|
```python
|
|
275
|
-
import os
|
|
276
348
|
from together import Together
|
|
277
349
|
|
|
278
|
-
client = Together(
|
|
350
|
+
client = Together()
|
|
279
351
|
|
|
280
352
|
client.files.upload(file="somedata.jsonl") # uploads a file
|
|
281
353
|
client.files.list() # lists all uploaded files
|
|
@@ -289,10 +361,9 @@ client.files.delete(id="file-d0d318cb-b7d9-493a-bd70-1cfe089d3815") # deletes a
|
|
|
289
361
|
The finetune API is used for fine-tuning and allows developers to create finetuning jobs. It also has several methods to list all jobs, retrive statuses and get checkpoints. Please refer to our fine-tuning docs [here](https://docs.together.ai/docs/fine-tuning-python).
|
|
290
362
|
|
|
291
363
|
```python
|
|
292
|
-
import os
|
|
293
364
|
from together import Together
|
|
294
365
|
|
|
295
|
-
client = Together(
|
|
366
|
+
client = Together()
|
|
296
367
|
|
|
297
368
|
client.fine_tuning.create(
|
|
298
369
|
training_file = 'file-d0d318cb-b7d9-493a-bd70-1cfe089d3815',
|
|
@@ -316,10 +387,9 @@ client.fine_tuning.download(id="ft-c66a5c18-1d6d-43c9-94bd-32d756425b4b") # down
|
|
|
316
387
|
This lists all the models that Together supports.
|
|
317
388
|
|
|
318
389
|
```python
|
|
319
|
-
import os
|
|
320
390
|
from together import Together
|
|
321
391
|
|
|
322
|
-
client = Together(
|
|
392
|
+
client = Together()
|
|
323
393
|
|
|
324
394
|
models = client.models.list()
|
|
325
395
|
|
|
@@ -7,13 +7,13 @@ together/cli/api/chat.py,sha256=2PHRb-9T-lUEKhUJFtc7SxJv3shCVx40gq_8pzfsewM,9234
|
|
|
7
7
|
together/cli/api/completions.py,sha256=l-Zw5t7hojL3w8xd_mitS2NRB72i5Z0xwkzH0rT5XMc,4263
|
|
8
8
|
together/cli/api/endpoints.py,sha256=LUIuK4DLs-VYor1nvOPzUNq0WeA7nIgIBHBD5Erdd5I,12470
|
|
9
9
|
together/cli/api/files.py,sha256=QLYEXRkY8J2Gg1SbTCtzGfoTMvosoeACNK83L_oLubs,3397
|
|
10
|
-
together/cli/api/finetune.py,sha256=
|
|
10
|
+
together/cli/api/finetune.py,sha256=0Md5FOzl0D6QfAmku628CGy43VzsjJ9-RbtY6ln5W1g,15018
|
|
11
11
|
together/cli/api/images.py,sha256=GADSeaNUHUVMtWovmccGuKc28IJ9E_v4vAEwYHJhu5o,2645
|
|
12
12
|
together/cli/api/models.py,sha256=xWEzu8ZpxM_Pz9KEjRPRVuv_v22RayYZ4QcgiezT5tE,1126
|
|
13
13
|
together/cli/api/utils.py,sha256=IuqYWPnLI38_Bqd7lj8V_SnGdYc59pRmMbQmciS4FsM,1326
|
|
14
14
|
together/cli/cli.py,sha256=YCDzbXpC5is0rs2PEkUPrIhYuzdyrihQ8GVR_TlDv5s,2054
|
|
15
15
|
together/client.py,sha256=vOe9NOgDyDlrT5ppvNfJGzdOHnMWEPmJX2RbXUQXKno,5081
|
|
16
|
-
together/constants.py,sha256=
|
|
16
|
+
together/constants.py,sha256=UDJhEylJFmdm4bedBDpvqYXBj5Or3k7z9GWtkRY_dZQ,1526
|
|
17
17
|
together/error.py,sha256=HU6247CyzCFjaxL9A0XYbXZ6fY_ebRg0FEYjI4Skogs,5515
|
|
18
18
|
together/filemanager.py,sha256=QHhBn73oVFdgUpSYXYLmJzHJ9c5wYEMJC0ur6ZgDeYo,11269
|
|
19
19
|
together/legacy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -21,7 +21,7 @@ together/legacy/base.py,sha256=ehrX1SCfRbK5OA83wL1q7-tfF-yuZOUxzjxYfFtdvvQ,727
|
|
|
21
21
|
together/legacy/complete.py,sha256=NRJX-vjnkg4HrgDo9LS3jFfhwfXpeGxcl24dcrLPK3A,2439
|
|
22
22
|
together/legacy/embeddings.py,sha256=nyTERjyPLTm7Sc987a9FJt1adnW7gIa7xs2CwXLE9EI,635
|
|
23
23
|
together/legacy/files.py,sha256=qmAqMiNTPWb6WvLV5Tsv6kxGRfQ31q7OkHZNFwkw8v0,4082
|
|
24
|
-
together/legacy/finetune.py,sha256=
|
|
24
|
+
together/legacy/finetune.py,sha256=XjZ4Dn2hSjMUVm64s6u1bbh9F7r9GbDKp-WLmzyEKRw,5123
|
|
25
25
|
together/legacy/images.py,sha256=bJJRs-6C7-NexPyaeyHiYlHOU51yls5-QAiqtO4xrZU,626
|
|
26
26
|
together/legacy/models.py,sha256=85ZN9Ids_FjdYNDRv5k7sgrtVWPKPHqkDplORtVUGHg,1087
|
|
27
27
|
together/resources/__init__.py,sha256=OQ8tW9mUIX0Ezk0wvYEnnEym6wGsjBKgXFLU9Ffgb-o,984
|
|
@@ -33,33 +33,33 @@ together/resources/completions.py,sha256=5Wa-ZjPCxRcam6CDe7KgGYlTA7yJZMmd5TrRgGC
|
|
|
33
33
|
together/resources/embeddings.py,sha256=PTvLb82yjG_-iQOyuhsilp77Fr7gZ0o6WD2KeRnKoxs,2675
|
|
34
34
|
together/resources/endpoints.py,sha256=tk_Ih94F9CXDmdRqsmOHS4yedmyxiUfIjFodh6pbCl8,15865
|
|
35
35
|
together/resources/files.py,sha256=bnPbaF25e4InBRPvHwXHXT-oSX1Z1sZRsnQW5wq82U4,4990
|
|
36
|
-
together/resources/finetune.py,sha256=
|
|
36
|
+
together/resources/finetune.py,sha256=euTGbSlFb7fIoRWGD4bc6Q-PKlXkOW7cAbfZALS4DTU,32945
|
|
37
37
|
together/resources/images.py,sha256=LQUjKPaFxWTqOAPnyF1Pp7Rz4NLOYhmoKwshpYiprEM,4923
|
|
38
38
|
together/resources/models.py,sha256=2dtHhXAqTDOOpwSbYLzWcKTC0-m2Szlb7LDYvp7Jr4w,1786
|
|
39
39
|
together/resources/rerank.py,sha256=3Ju_aRSyZ1s_3zCSNZnSnEJErUVmt2xa3M8z1nvejMA,3931
|
|
40
40
|
together/together_response.py,sha256=a3dgKMPDrlfKQwxYENfNt2T4l2vSZxRWMixhHSy-q3E,1308
|
|
41
|
-
together/types/__init__.py,sha256=
|
|
41
|
+
together/types/__init__.py,sha256=edHguHW7OeCPZZWts80Uw6mF406rPzWAcoCQLueO1_0,2552
|
|
42
42
|
together/types/abstract.py,sha256=1lFQI_3WjsR_t1128AeKW0aTk6EiM6Gh1J3ZuyLLPao,642
|
|
43
43
|
together/types/audio_speech.py,sha256=jlj8BZf3dkIDARF1P11fuenVLj4try8Yx4RN-EAkhOU,2609
|
|
44
|
-
together/types/chat_completions.py,sha256=
|
|
44
|
+
together/types/chat_completions.py,sha256=ggwt1LlBXTB_hZKbtLsjg8j-gXxO8pUUQfTrxUmRXHU,5078
|
|
45
45
|
together/types/common.py,sha256=kxZ-N9xtBsGYZBmbIWnZ0rfT3Pn8PFB7sAbp3iv96pw,1525
|
|
46
46
|
together/types/completions.py,sha256=o3FR5ixsTUj-a3pmOUzbSQg-hESVhpqrC9UD__VCqr4,2971
|
|
47
47
|
together/types/embeddings.py,sha256=J7grkYYn7xhqeKaBO2T-8XQRtHhkzYzymovtGdIUK5A,751
|
|
48
|
-
together/types/endpoints.py,sha256=
|
|
48
|
+
together/types/endpoints.py,sha256=EzNhHOoQ_D9fUdNQtxQPeSWiFzdFLqpNodN0YLmv_h0,4393
|
|
49
49
|
together/types/error.py,sha256=OVlCs3cx_2WhZK4JzHT8SQyRIIqKOP1AZQ4y1PydjAE,370
|
|
50
50
|
together/types/files.py,sha256=-rEUfsV6f2vZB9NrFxT4_933ubsDIUNkPB-3OlOFk4A,1954
|
|
51
|
-
together/types/finetune.py,sha256=
|
|
51
|
+
together/types/finetune.py,sha256=rsmzxUF2gEh6KzlxoagkuUEiJz1gHDwuRgZnmotyQ1k,9994
|
|
52
52
|
together/types/images.py,sha256=xnC-FZGdZU30WSFTybfGneWxb-kj0ZGufJsgHtB8j0k,980
|
|
53
53
|
together/types/models.py,sha256=nwQIZGHKZpX9I6mK8z56VW70YC6Ry6JGsVa0s99QVxc,1055
|
|
54
54
|
together/types/rerank.py,sha256=qZfuXOn7MZ6ly8hpJ_MZ7OU_Bi1-cgYNSB20Wja8Qkk,1061
|
|
55
|
-
together/utils/__init__.py,sha256=
|
|
55
|
+
together/utils/__init__.py,sha256=5fqvj4KT2rHxKSQot2TSyV_HcvkvkGiqAiaYuJwqtm0,786
|
|
56
56
|
together/utils/_log.py,sha256=5IYNI-jYzxyIS-pUvhb0vE_Muo3MA7GgBhsu66TKP2w,1951
|
|
57
57
|
together/utils/api_helpers.py,sha256=RSF7SRhbjHzroMOSWAXscflByM1r1ta_1SpxkAT22iE,2407
|
|
58
|
-
together/utils/files.py,sha256=
|
|
59
|
-
together/utils/tools.py,sha256=
|
|
58
|
+
together/utils/files.py,sha256=rfp10qU0urtWOXXFeasFtO9xp-1KIhM3S43JxcnHmL0,16438
|
|
59
|
+
together/utils/tools.py,sha256=H2MTJhEqtBllaDvOyZehIO_IVNK3P17rSDeILtJIVag,2964
|
|
60
60
|
together/version.py,sha256=p03ivHyE0SyWU4jAnRTBi_sOwywVWoZPU4g2gzRgG-Y,126
|
|
61
|
-
together-1.4.
|
|
62
|
-
together-1.4.
|
|
63
|
-
together-1.4.
|
|
64
|
-
together-1.4.
|
|
65
|
-
together-1.4.
|
|
61
|
+
together-1.4.4.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
62
|
+
together-1.4.4.dist-info/METADATA,sha256=ICtNOO5v35bKJFEtlgdVg4ORP0ofYO11vANs-QuowxY,14445
|
|
63
|
+
together-1.4.4.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
|
|
64
|
+
together-1.4.4.dist-info/entry_points.txt,sha256=G-b5NKW6lUUf1V1fH8IPTBb7jXnK7lhbX9H1zTEJXPs,50
|
|
65
|
+
together-1.4.4.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|