together 1.2.2__py3-none-any.whl → 1.2.3__py3-none-any.whl

together/cli/api/finetune.py

@@ -2,6 +2,7 @@ import json
 from textwrap import wrap
 
 import click
+from click.core import ParameterSource  # type: ignore[attr-defined]
 from tabulate import tabulate
 
 from together import Together
@@ -26,7 +27,22 @@ def fine_tuning(ctx: click.Context) -> None:
     "--n-checkpoints", type=int, default=1, help="Number of checkpoints to save"
 )
 @click.option("--batch-size", type=int, default=32, help="Train batch size")
-@click.option("--learning-rate", type=float, default=3e-5, help="Learning rate")
+@click.option("--learning-rate", type=float, default=1e-5, help="Learning rate")
+@click.option(
+    "--lora/--no-lora",
+    type=bool,
+    default=False,
+    help="Whether to use LoRA adapters for fine-tuning",
+)
+@click.option("--lora-r", type=int, default=8, help="LoRA adapters' rank")
+@click.option("--lora-dropout", type=float, default=0, help="LoRA adapters' dropout")
+@click.option("--lora-alpha", type=float, default=8, help="LoRA adapters' alpha")
+@click.option(
+    "--lora-trainable-modules",
+    type=str,
+    default="all-linear",
+    help="Trainable modules for LoRA adapters. For example, 'all-linear', 'q_proj,v_proj'",
+)
 @click.option(
     "--suffix", type=str, default=None, help="Suffix for the fine-tuned model name"
 )
@@ -39,12 +55,32 @@ def create(
     n_checkpoints: int,
     batch_size: int,
     learning_rate: float,
+    lora: bool,
+    lora_r: int,
+    lora_dropout: float,
+    lora_alpha: float,
+    lora_trainable_modules: str,
     suffix: str,
     wandb_api_key: str,
 ) -> None:
     """Start fine-tuning"""
     client: Together = ctx.obj
 
+    if lora:
+        learning_rate_source = click.get_current_context().get_parameter_source(  # type: ignore[attr-defined]
+            "learning_rate"
+        )
+        if learning_rate_source == ParameterSource.DEFAULT:
+            learning_rate = 1e-3
+    else:
+        for param in ["lora_r", "lora_dropout", "lora_alpha", "lora_trainable_modules"]:
+            param_source = click.get_current_context().get_parameter_source(param)  # type: ignore[attr-defined]
+            if param_source != ParameterSource.DEFAULT:
+                raise click.BadParameter(
+                    f"You set LoRA parameter `{param}` for a full fine-tuning job. "
+                    f"Please change the job type with --lora or remove `{param}` from the arguments"
+                )
+
     response = client.fine_tuning.create(
         training_file=training_file,
         model=model,
@@ -52,6 +88,11 @@ def create(
         n_checkpoints=n_checkpoints,
         batch_size=batch_size,
         learning_rate=learning_rate,
+        lora=lora,
+        lora_r=lora_r,
+        lora_dropout=lora_dropout,
+        lora_alpha=lora_alpha,
+        lora_trainable_modules=lora_trainable_modules,
         suffix=suffix,
         wandb_api_key=wandb_api_key,
     )

together/resources/finetune.py

@@ -11,8 +11,11 @@ from together.types import (
     FinetuneListEvents,
     FinetuneRequest,
     FinetuneResponse,
+    FullTrainingType,
+    LoRATrainingType,
     TogetherClient,
     TogetherRequest,
+    TrainingType,
 )
 from together.utils import normalize_key
 
@@ -30,6 +33,11 @@ class FineTuning:
         n_checkpoints: int | None = 1,
         batch_size: int | None = 32,
         learning_rate: float | None = 0.00001,
+        lora: bool = True,
+        lora_r: int | None = 8,
+        lora_dropout: float | None = 0,
+        lora_alpha: float | None = 8,
+        lora_trainable_modules: str | None = "all-linear",
         suffix: str | None = None,
         wandb_api_key: str | None = None,
     ) -> FinetuneResponse:
@@ -45,6 +53,11 @@ class FineTuning:
             batch_size (int, optional): Batch size for fine-tuning. Defaults to 32.
             learning_rate (float, optional): Learning rate multiplier to use for training
                 Defaults to 0.00001.
+            lora (bool, optional): Whether to use LoRA adapters. Defaults to True.
+            lora_r (int, optional): Rank of LoRA adapters. Defaults to 8.
+            lora_dropout (float, optional): Dropout rate for LoRA adapters. Defaults to 0.
+            lora_alpha (float, optional): Alpha for LoRA adapters. Defaults to 8.
+            lora_trainable_modules (str, optional): Trainable modules for LoRA adapters. Defaults to "all-linear".
             suffix (str, optional): Up to 40 character suffix that will be added to your fine-tuned model name.
                 Defaults to None.
             wandb_api_key (str, optional): API key for Weights & Biases integration.
@@ -58,6 +71,15 @@ class FineTuning:
             client=self._client,
         )
 
+        training_type: TrainingType = FullTrainingType()
+        if lora:
+            training_type = LoRATrainingType(
+                lora_r=lora_r,
+                lora_alpha=lora_alpha,
+                lora_dropout=lora_dropout,
+                lora_trainable_modules=lora_trainable_modules,
+            )
+
         parameter_payload = FinetuneRequest(
             model=model,
             training_file=training_file,
@@ -65,6 +87,7 @@ class FineTuning:
             n_checkpoints=n_checkpoints,
             batch_size=batch_size,
             learning_rate=learning_rate,
+            training_type=training_type,
             suffix=suffix,
             wandb_key=wandb_api_key,
         ).model_dump()

together/types/__init__.py

@@ -26,6 +26,9 @@ from together.types.finetune import (
     FinetuneListEvents,
     FinetuneRequest,
     FinetuneResponse,
+    FullTrainingType,
+    LoRATrainingType,
+    TrainingType,
 )
 from together.types.images import (
     ImageRequest,
@@ -60,4 +63,7 @@ __all__ = [
     "ImageRequest",
     "ImageResponse",
     "ModelObject",
+    "TrainingType",
+    "FullTrainingType",
+    "LoRATrainingType",
 ]

together/types/finetune.py

@@ -100,6 +100,34 @@ class FinetuneEvent(BaseModel):
     hash: str | None = None
 
 
+class TrainingType(BaseModel):
+    """
+    Abstract training type
+    """
+
+    type: str
+
+
+class FullTrainingType(TrainingType):
+    """
+    Training type for full fine-tuning
+    """
+
+    type: str = "Full"
+
+
+class LoRATrainingType(TrainingType):
+    """
+    Training type for LoRA adapters training
+    """
+
+    lora_r: int
+    lora_alpha: int
+    lora_dropout: float
+    lora_trainable_modules: str
+    type: str = "Lora"
+
+
 class FinetuneRequest(BaseModel):
     """
     Fine-tune request type
@@ -121,6 +149,7 @@ class FinetuneRequest(BaseModel):
     suffix: str | None = None
     # weights & biases api key
     wandb_key: str | None = None
+    training_type: FullTrainingType | LoRATrainingType | None = None
 
 
 class FinetuneResponse(BaseModel):
@@ -138,6 +167,8 @@ class FinetuneResponse(BaseModel):
     model: str | None = None
     # output model name
     output_name: str | None = Field(None, alias="model_output_name")
+    # adapter output name
+    adapter_output_name: str | None = None
     # number of epochs
     n_epochs: int | None = None
     # number of checkpoints to save
@@ -148,11 +179,8 @@ class FinetuneResponse(BaseModel):
     learning_rate: float | None = None
     # number of steps between evals
     eval_steps: int | None = None
-    # is LoRA finetune boolean
-    lora: bool | None = None
-    lora_r: int | None = None
-    lora_alpha: int | None = None
-    lora_dropout: int | None = None
+    # training type
+    training_type: FullTrainingType | LoRATrainingType | None = None
     # created/updated datetime stamps
     created_at: str | None = None
     updated_at: str | None = None

{together-1.2.2.dist-info → together-1.2.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: together
-Version: 1.2.2
+Version: 1.2.3
 Summary: Python client for Together's Cloud Platform!
 Home-page: https://github.com/togethercomputer/together-python
 License: Apache-2.0

{together-1.2.2.dist-info → together-1.2.3.dist-info}/RECORD

@@ -6,7 +6,7 @@ together/cli/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,
 together/cli/api/chat.py,sha256=qmDcYohGe1ODfvYEDN6Z2eZO3qODgmeUTll8P5qoKYQ,9170
 together/cli/api/completions.py,sha256=v6KvGsXnPCdSM7WIu_QmKXOA4iVjW93k3dLwfIBkxPQ,4199
 together/cli/api/files.py,sha256=hOiN1ftwbSAs3gcVqwmPymhmzDYiIUrFWXFHVcWWom8,3329
-together/cli/api/finetune.py,sha256=mSQSasmVQLtNzjJLeoEEwvewfEcpSkjKyozWqwb-7p0,5417
+together/cli/api/finetune.py,sha256=xg9yb1GEvtxj7RR6g2A4Mn9gM6M9Oeb0Tp-8EF8DSc0,7089
 together/cli/api/images.py,sha256=01dFYa2sK1HqUwVCD9FlwcjqkYWLoNxFZkzok13EriE,2363
 together/cli/api/models.py,sha256=xWEzu8ZpxM_Pz9KEjRPRVuv_v22RayYZ4QcgiezT5tE,1126
 together/cli/cli.py,sha256=RC0tgapkSOFjsRPg8p-8dx9D2LDzm8YmVCHUjk_aVyQ,1977
@@ -28,11 +28,11 @@ together/resources/chat/completions.py,sha256=QGrg0fGsn16k6jpxbQGoIBRivHWgEimPWA
 together/resources/completions.py,sha256=5s-jly2vbpI1rWhFwVijCmCOyhEZuDuwJblrm3HD0Gs,11353
 together/resources/embeddings.py,sha256=yRG0JaF4u_MEynQ4TBgCOVf9bKSvGI1fRaQx3CHUiB0,2546
 together/resources/files.py,sha256=bnPbaF25e4InBRPvHwXHXT-oSX1Z1sZRsnQW5wq82U4,4990
-together/resources/finetune.py,sha256=DcA0A3I9khS8nMv2KpVNjDVI92knnI2o-O_93Z8gP-U,12416
+together/resources/finetune.py,sha256=-MU25J2gHSFQnBIPg4XDle5cgfrkOnQtK9_T_hJqRy8,13471
 together/resources/images.py,sha256=gFzXy7gLzr20KsXJXHEsOZtJJpuR6pH1d8XHCd6Dl9E,4775
 together/resources/models.py,sha256=2dtHhXAqTDOOpwSbYLzWcKTC0-m2Szlb7LDYvp7Jr4w,1786
 together/together_response.py,sha256=MhczUCPem93cjX-A1TOAUrRj3sO-o3SLcEcTsZgVzQI,1319
-together/types/__init__.py,sha256=K7Gv6hLmobIfqfmijZbZwFrwxK_YKIDR_v94_ElFmVA,1432
+together/types/__init__.py,sha256=QeDcuLgW1lowGijdwEZXCibaBIb9k0YKG581TlXwzd4,1562
 together/types/abstract.py,sha256=1lFQI_3WjsR_t1128AeKW0aTk6EiM6Gh1J3ZuyLLPao,642
 together/types/chat_completions.py,sha256=VxuvcHM9IQVWfD1qKu6gDWdrV5UainjkV3pGm_4Oiyo,4335
 together/types/common.py,sha256=4ZeIgqGioqhIC-nNxY90czNPp-kAqboMulw6-1z6ShM,1511
@@ -40,7 +40,7 @@ together/types/completions.py,sha256=yydloTQePGaY97Lx-kbkvgCqBFhHFl7jU5s7uf9Ncg0
 together/types/embeddings.py,sha256=J7grkYYn7xhqeKaBO2T-8XQRtHhkzYzymovtGdIUK5A,751
 together/types/error.py,sha256=OVlCs3cx_2WhZK4JzHT8SQyRIIqKOP1AZQ4y1PydjAE,370
 together/types/files.py,sha256=-rEUfsV6f2vZB9NrFxT4_933ubsDIUNkPB-3OlOFk4A,1954
-together/types/finetune.py,sha256=eK2Zv47ccvbenGv87PmNCBsqPQleiA3KxoK7KjsuYTM,5850
+together/types/finetune.py,sha256=qApCHFPKmXuA4rOnW1qh3IK165gCKe5Kr-usn8ZOXLg,6344
 together/types/images.py,sha256=zX4Vt38tFDKU6yGb_hBY_N5eSTn3KPdpP5Ce_qnRHXQ,915
 together/types/models.py,sha256=3zag9x2fus2McNmLkr3fKzQL-2RNIT1-tiabI-z21p8,1013
 together/utils/__init__.py,sha256=VpjeRTya1m5eEE-Qe1zYTFsNAvuEA-dy7M2eG9Xu4fc,662
@@ -49,8 +49,8 @@ together/utils/api_helpers.py,sha256=RSF7SRhbjHzroMOSWAXscflByM1r1ta_1SpxkAT22iE
 together/utils/files.py,sha256=gMLthqfP5hKxVAerHMdy7gLXzdfY6lyOXdpW24Y4X3I,7165
 together/utils/tools.py,sha256=3-lXWP3cBCzOVSZg9tr5zOT1jaVeKAKVWxO2fcXZTh8,1788
 together/version.py,sha256=p03ivHyE0SyWU4jAnRTBi_sOwywVWoZPU4g2gzRgG-Y,126
-together-1.2.2.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-together-1.2.2.dist-info/METADATA,sha256=S9B78Nc5Ge6nBVMfLDFCK9qTQPrmhXMo4BbxCyYSsoo,11812
-together-1.2.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-together-1.2.2.dist-info/entry_points.txt,sha256=G-b5NKW6lUUf1V1fH8IPTBb7jXnK7lhbX9H1zTEJXPs,50
-together-1.2.2.dist-info/RECORD,,
+together-1.2.3.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+together-1.2.3.dist-info/METADATA,sha256=BdqQEU6vpCarSDazFhrqWoaZqBkz8tfTN7iH8HPjZH4,11812
+together-1.2.3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+together-1.2.3.dist-info/entry_points.txt,sha256=G-b5NKW6lUUf1V1fH8IPTBb7jXnK7lhbX9H1zTEJXPs,50
+together-1.2.3.dist-info/RECORD,,