together 2.0.0a10__py3-none-any.whl → 2.0.0a12__py3-none-any.whl

together/_base_client.py

@@ -1247,9 +1247,12 @@ class SyncAPIClient(BaseClient[httpx.Client, Stream[Any]]):
         *,
         cast_to: Type[ResponseT],
         body: Body | None = None,
+        files: RequestFiles | None = None,
         options: RequestOptions = {},
     ) -> ResponseT:
-        opts = FinalRequestOptions.construct(method="patch", url=path, json_data=body, **options)
+        opts = FinalRequestOptions.construct(
+            method="patch", url=path, json_data=body, files=to_httpx_files(files), **options
+        )
         return self.request(cast_to, opts)
 
     def put(
@@ -1767,9 +1770,12 @@ class AsyncAPIClient(BaseClient[httpx.AsyncClient, AsyncStream[Any]]):
         *,
         cast_to: Type[ResponseT],
         body: Body | None = None,
+        files: RequestFiles | None = None,
         options: RequestOptions = {},
     ) -> ResponseT:
-        opts = FinalRequestOptions.construct(method="patch", url=path, json_data=body, **options)
+        opts = FinalRequestOptions.construct(
+            method="patch", url=path, json_data=body, files=await async_to_httpx_files(files), **options
+        )
         return await self.request(cast_to, opts)
 
     async def put(

together/_version.py

@@ -1,4 +1,4 @@
 # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
 __title__ = "together"
-__version__ = "2.0.0-alpha.10"  # x-release-please-version
+__version__ = "2.0.0-alpha.12"  # x-release-please-version

together/lib/cli/api/fine_tuning.py

@@ -10,6 +10,7 @@ from textwrap import wrap
 import click
 from rich import print as rprint
 from tabulate import tabulate
+from rich.json import JSON
 from click.core import ParameterSource  # type: ignore[attr-defined]
 
 from together import Together
@@ -17,7 +18,7 @@ from together.types import fine_tuning_estimate_price_params as pe_params
 from together._types import NOT_GIVEN, NotGiven
 from together.lib.utils import log_warn
 from together.lib.utils.tools import format_timestamp, finetune_price_to_dollars
-from together.lib.cli.api.utils import INT_WITH_MAX, BOOL_WITH_AUTO
+from together.lib.cli.api.utils import INT_WITH_MAX, BOOL_WITH_AUTO, generate_progress_bar
 from together.lib.resources.files import DownloadManager
 from together.lib.utils.serializer import datetime_serializer
 from together.types.finetune_response import TrainingTypeFullTrainingType, TrainingTypeLoRaTrainingType
@@ -175,6 +176,12 @@ def fine_tuning(ctx: click.Context) -> None:
     help="Whether to mask the user messages in conversational data or prompts in instruction data. "
     "`auto` will automatically determine whether to mask the inputs based on the data format.",
 )
+@click.option(
+    "--train-vision",
+    type=bool,
+    default=False,
+    help="Whether to train the vision encoder. Only supported for multimodal models.",
+)
 @click.option(
     "--from-checkpoint",
     type=str,
@@ -230,6 +237,7 @@ def create(
     lora_dropout: float | None,
     lora_alpha: float | None,
     lora_trainable_modules: str | None,
+    train_vision: bool,
     suffix: str | None,
     wandb_api_key: str | None,
     wandb_base_url: str | None,
@@ -271,6 +279,7 @@ def create(
         lora_dropout=lora_dropout,
         lora_alpha=lora_alpha,
         lora_trainable_modules=lora_trainable_modules,
+        train_vision=train_vision,
         suffix=suffix,
         wandb_api_key=wandb_api_key,
         wandb_base_url=wandb_base_url,
@@ -361,7 +370,11 @@ def create(
             rpo_alpha=rpo_alpha or 0,
             simpo_gamma=simpo_gamma or 0,
         )
-    
+
+    if model_limits.supports_vision:
+        # Don't show price estimation for multimodal models yet
+        confirm = True
+
     finetune_price_estimation_result = client.fine_tuning.estimate_price(
         training_file=training_file,
         validation_file=validation_file,
@@ -425,6 +438,7 @@ def list(ctx: click.Context) -> None:
                 "Price": f"""${
                     finetune_price_to_dollars(float(str(i.total_price)))
                 }""",  # convert to string for mypy typing
+                "Progress": generate_progress_bar(i, datetime.now().astimezone(), use_rich=False),
             }
         )
     table = tabulate(display_list, headers="keys", tablefmt="grid", showindex=True)
@@ -444,7 +458,10 @@ def retrieve(ctx: click.Context, fine_tune_id: str) -> None:
     # remove events from response for cleaner output
     response.events = None
 
-    click.echo(json.dumps(response.model_dump(exclude_none=True), indent=4))
+    rprint(JSON.from_data(response.model_json_schema()))
+    progress_text = generate_progress_bar(response, datetime.now().astimezone(), use_rich=True)
+    prefix = f"Status: [bold]{response.status}[/bold],"
+    rprint(f"{prefix} {progress_text}")
 
 
 @fine_tuning.command()

together/lib/cli/api/utils.py

@@ -1,18 +1,25 @@
 from __future__ import annotations
 
-from typing import Literal
+import re
+import math
+from typing import List, Union, Literal
 from gettext import gettext as _
-from typing_extensions import override
+from datetime import datetime
 
 import click
 
+from together.lib.types.fine_tuning import COMPLETED_STATUSES, FinetuneResponse
+from together.types.finetune_response import FinetuneResponse as _FinetuneResponse
+from together.types.fine_tuning_list_response import Data
+
+_PROGRESS_BAR_WIDTH = 40
+
 
 class AutoIntParamType(click.ParamType):
     name = "integer_or_max"
     _number_class = int
 
-    @override
-    def convert(
+    def convert(  # pyright: ignore[reportImplicitOverride]
         self, value: str, param: click.Parameter | None, ctx: click.Context | None
     ) -> int | Literal["max"] | None:
         if value == "max":
@@ -30,8 +37,7 @@ class AutoIntParamType(click.ParamType):
 class BooleanWithAutoParamType(click.ParamType):
     name = "boolean_or_auto"
 
-    @override
-    def convert(
+    def convert(  # pyright: ignore[reportImplicitOverride]
         self, value: str, param: click.Parameter | None, ctx: click.Context | None
     ) -> bool | Literal["auto"] | None:
         if value == "auto":
@@ -48,3 +54,78 @@ class BooleanWithAutoParamType(click.ParamType):
 
 INT_WITH_MAX = AutoIntParamType()
 BOOL_WITH_AUTO = BooleanWithAutoParamType()
+
+
+def _human_readable_time(timedelta: float) -> str:
+    """Convert a timedelta to a compact human-readble string
+    Examples:
+        00:00:10 -> 10s
+        01:23:45 -> 1h 23min 45s
+        1 Month 23 days 04:56:07 -> 1month 23d 4h 56min 7s
+    Args:
+        timedelta (float): The timedelta in seconds to convert.
+    Returns:
+        A string representing the timedelta in a human-readable format.
+    """
+    units = [
+        (30 * 24 * 60 * 60, "month"),  # 30 days
+        (24 * 60 * 60, "d"),
+        (60 * 60, "h"),
+        (60, "min"),
+        (1, "s"),
+    ]
+
+    total_seconds = int(timedelta)
+    parts: List[str] = []
+
+    for unit_seconds, unit_name in units:
+        if total_seconds >= unit_seconds:
+            value = total_seconds // unit_seconds
+            total_seconds %= unit_seconds
+            parts.append(f"{value}{unit_name}")
+
+    return " ".join(parts) if parts else "0s"
+
+
+def generate_progress_bar(
+    finetune_job: Union[Data, FinetuneResponse, _FinetuneResponse], current_time: datetime, use_rich: bool = False
+) -> str:
+    """Generate a progress bar for a finetune job.
+    Args:
+        finetune_job: The finetune job to generate a progress bar for.
+        current_time: The current time.
+        use_rich: Whether to use rich formatting.
+    Returns:
+        A string representing the progress bar.
+    """
+    progress = "Progress: [bold red]unavailable[/bold red]"
+    if finetune_job.status in COMPLETED_STATUSES:
+        progress = "Progress: [bold green]completed[/bold green]"
+    elif finetune_job.updated_at is not None:
+        update_at = finetune_job.updated_at.astimezone()
+
+        if finetune_job.progress is not None:
+            if current_time < update_at:
+                return progress
+
+            if not finetune_job.progress.estimate_available:
+                return progress
+
+            if finetune_job.progress.seconds_remaining <= 0:
+                return progress
+
+            elapsed_time = (current_time - update_at).total_seconds()
+            ratio_filled = min(elapsed_time / finetune_job.progress.seconds_remaining, 1.0)
+            percentage = ratio_filled * 100
+            filled = math.ceil(ratio_filled * _PROGRESS_BAR_WIDTH)
+            bar = "█" * filled + "░" * (_PROGRESS_BAR_WIDTH - filled)
+            time_left = "N/A"
+            if finetune_job.progress.seconds_remaining > elapsed_time:
+                time_left = _human_readable_time(finetune_job.progress.seconds_remaining - elapsed_time)
+            time_text = f"{time_left} left"
+            progress = f"Progress: {bar} [bold]{percentage:>3.0f}%[/bold] [yellow]{time_text}[/yellow]"
+
+    if use_rich:
+        return progress
+
+    return re.sub(r"\[/?[^\]]+\]", "", progress)

together/lib/constants.py

@@ -14,6 +14,9 @@ import enum
 # Download defaults
 DOWNLOAD_BLOCK_SIZE = 10 * 1024 * 1024  # 10 MB
 DISABLE_TQDM = False
+MAX_DOWNLOAD_RETRIES = 5  # Maximum retries for download failures
+DOWNLOAD_INITIAL_RETRY_DELAY = 1.0  # Initial retry delay in seconds
+DOWNLOAD_MAX_RETRY_DELAY = 30.0  # Maximum retry delay in seconds
 
 # Upload defaults
 MAX_CONCURRENT_PARTS = 4  # Maximum concurrent parts for multipart upload
@@ -34,6 +37,12 @@ NUM_BYTES_IN_GB = 2**30
 # maximum number of GB sized files we support finetuning for
 MAX_FILE_SIZE_GB = 50.1
 
+# Multimodal limits
+MAX_IMAGES_PER_EXAMPLE = 10
+MAX_IMAGE_BYTES = 10 * 1024 * 1024  # 10MB
+# Max length = Header length + base64 factor (4/3) * image bytes
+MAX_BASE64_IMAGE_LENGTH = len("data:image/jpeg;base64,") + 4 * MAX_IMAGE_BYTES // 3
+
 # expected columns for Parquet files
 PARQUET_EXPECTED_COLUMNS = ["input_ids", "attention_mask", "labels"]
 

together/lib/resources/files.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import os
 import math
 import stat
+import time
 import uuid
 import shutil
 import asyncio
@@ -29,12 +30,15 @@ from ..constants import (
     MAX_MULTIPART_PARTS,
     TARGET_PART_SIZE_MB,
     MAX_CONCURRENT_PARTS,
+    MAX_DOWNLOAD_RETRIES,
     MULTIPART_THRESHOLD_GB,
+    DOWNLOAD_MAX_RETRY_DELAY,
     MULTIPART_UPLOAD_TIMEOUT,
+    DOWNLOAD_INITIAL_RETRY_DELAY,
 )
 from ..._resource import SyncAPIResource, AsyncAPIResource
 from ..types.error import DownloadError, FileTypeError
-from ..._exceptions import APIStatusError, AuthenticationError
+from ..._exceptions import APIStatusError, APIConnectionError, AuthenticationError
 
 log: logging.Logger = logging.getLogger(__name__)
 
@@ -198,6 +202,11 @@ class DownloadManager(SyncAPIResource):
 
                 assert file_size != 0, "Unable to retrieve remote file."
 
+                # Download with retry logic
+                bytes_downloaded = 0
+                retry_count = 0
+                retry_delay = DOWNLOAD_INITIAL_RETRY_DELAY
+
                 with tqdm(
                     total=file_size,
                     unit="B",
@@ -205,14 +214,64 @@ class DownloadManager(SyncAPIResource):
                     desc=f"Downloading file {file_path.name}",
                     disable=bool(DISABLE_TQDM),
                 ) as pbar:
-                    for chunk in response.iter_bytes(DOWNLOAD_BLOCK_SIZE):
-                        pbar.update(len(chunk))
-                        temp_file.write(chunk)  # type: ignore
+                    while bytes_downloaded < file_size:
+                        try:
+                            # If this is a retry, close the previous response and create a new one with Range header
+                            if bytes_downloaded > 0:
+                                response.close()
+
+                                log.info(f"Resuming download from byte {bytes_downloaded}")
+                                response = self._client.get(
+                                    path=url,
+                                    cast_to=httpx.Response,
+                                    stream=True,
+                                    options=RequestOptions(
+                                        headers={"Range": f"bytes={bytes_downloaded}-"},
+                                    ),
+                                )
+
+                            # Download chunks
+                            for chunk in response.iter_bytes(DOWNLOAD_BLOCK_SIZE):
+                                temp_file.write(chunk)  # type: ignore
+                                bytes_downloaded += len(chunk)
+                                pbar.update(len(chunk))
+
+                            # Successfully completed download
+                            break
+
+                        except (httpx.RequestError, httpx.StreamError, APIConnectionError) as e:
+                            if retry_count >= MAX_DOWNLOAD_RETRIES:
+                                log.error(f"Download failed after {retry_count} retries")
+                                raise DownloadError(
+                                    f"Download failed after {retry_count} retries. Last error: {str(e)}"
+                                ) from e
+
+                            retry_count += 1
+                            log.warning(
+                                f"Download interrupted at {bytes_downloaded}/{file_size} bytes. "
+                                f"Retry {retry_count}/{MAX_DOWNLOAD_RETRIES} in {retry_delay}s..."
+                            )
+                            time.sleep(retry_delay)
+
+                            # Exponential backoff with max delay cap
+                            retry_delay = min(retry_delay * 2, DOWNLOAD_MAX_RETRY_DELAY)
+
+                        except APIStatusError as e:
+                            # For API errors, don't retry
+                            log.error(f"API error during download: {e}")
+                            raise APIStatusError(
+                                "Error downloading file",
+                                response=e.response,
+                                body=e.response,
+                            ) from e
+
+                # Close the response
+                response.close()
 
             # Raise exception if remote file size does not match downloaded file size
             if os.stat(temp_file.name).st_size != file_size:
-                DownloadError(
-                    f"Downloaded file size `{pbar.n}` bytes does not match remote file size `{file_size}` bytes."
+                raise DownloadError(
+                    f"Downloaded file size `{bytes_downloaded}` bytes does not match remote file size `{file_size}` bytes."
                 )
 
             # Moves temp file to output file path

together/lib/resources/fine_tuning.py

@@ -22,6 +22,7 @@ from together.lib.types.fine_tuning import (
     CosineLRSchedulerArgs,
     LinearLRSchedulerArgs,
     FinetuneTrainingLimits,
+    FinetuneMultimodalParams,
 )
 
 AVAILABLE_TRAINING_METHODS = {
@@ -51,6 +52,7 @@ def create_finetune_request(
     lora_dropout: float | None = 0,
     lora_alpha: float | None = None,
     lora_trainable_modules: str | None = "all-linear",
+    train_vision: bool = False,
     suffix: str | None = None,
     wandb_api_key: str | None = None,
     wandb_base_url: str | None = None,
@@ -207,6 +209,13 @@ def create_finetune_request(
             simpo_gamma=simpo_gamma,
         )
 
+    if model_limits.supports_vision:
+        multimodal_params = FinetuneMultimodalParams(train_vision=train_vision)
+    elif not model_limits.supports_vision and train_vision:
+        raise ValueError(f"Vision encoder training is not supported for the non-multimodal model `{model}`")
+    else:
+        multimodal_params = None
+
     finetune_request = FinetuneRequest(
         model=model,
         training_file=training_file,
@@ -227,6 +236,7 @@ def create_finetune_request(
         wandb_project_name=wandb_project_name,
         wandb_name=wandb_name,
         training_method=training_method_cls,  # pyright: ignore[reportPossiblyUnboundVariable]
+        multimodal_params=multimodal_params,
         from_checkpoint=from_checkpoint,
         from_hf_model=from_hf_model,
         hf_model_revision=hf_model_revision,
@@ -238,7 +248,10 @@ def create_finetune_request(
 
     return finetune_request, training_type_pe, training_method_pe
 
-def create_price_estimation_params(finetune_request: FinetuneRequest) -> tuple[pe_params.TrainingType, pe_params.TrainingMethod]:
+
+def create_price_estimation_params(
+    finetune_request: FinetuneRequest,
+) -> tuple[pe_params.TrainingType, pe_params.TrainingMethod]:
     training_type_cls: pe_params.TrainingType
     if isinstance(finetune_request.training_type, FullTrainingType):
         training_type_cls = pe_params.TrainingTypeFullTrainingType(
@@ -275,6 +288,7 @@ def create_price_estimation_params(finetune_request: FinetuneRequest) -> tuple[p
 
     return training_type_cls, training_method_cls
 
+
 def get_model_limits(client: Together, model: str) -> FinetuneTrainingLimits:
     """
     Requests training limits for a specific model

together/lib/types/fine_tuning.py

@@ -25,6 +25,14 @@ class FinetuneJobStatus(str, Enum):
     STATUS_COMPLETED = "completed"
 
 
+COMPLETED_STATUSES = [
+    FinetuneJobStatus.STATUS_ERROR,
+    FinetuneJobStatus.STATUS_USER_ERROR,
+    FinetuneJobStatus.STATUS_COMPLETED,
+    FinetuneJobStatus.STATUS_CANCELLED,
+]
+
+
 class FinetuneEventType(str, Enum):
     """
     Fine-tune job event types
@@ -181,6 +189,7 @@ class TrainingMethodUnknown(BaseModel):
 
     method: str
 
+
 TrainingMethod: TypeAlias = Union[
     TrainingMethodSFT,
     TrainingMethodDPO,
@@ -194,6 +203,7 @@ class FinetuneTrainingLimits(BaseModel):
     min_learning_rate: float
     full_training: Optional[FinetuneFullTrainingLimits] = None
     lora_training: Optional[FinetuneLoraTrainingLimits] = None
+    supports_vision: bool = False
 
 
 class LinearLRSchedulerArgs(BaseModel):
@@ -241,6 +251,7 @@ class EmptyLRScheduler(BaseModel):
     lr_scheduler_type: Literal[""]
     lr_scheduler_args: None = None
 
+
 class UnknownLRScheduler(BaseModel):
     """
     Unknown learning rate scheduler
@@ -260,6 +271,23 @@ FinetuneLRScheduler: TypeAlias = Union[
 ]
 
 
+class FinetuneMultimodalParams(BaseModel):
+    """
+    Multimodal parameters
+    """
+
+    train_vision: bool = False
+
+
+class FinetuneProgress(BaseModel):
+    """
+    Fine-tune job progress
+    """
+
+    estimate_available: bool = False
+    seconds_remaining: float = 0
+
+
 class FinetuneResponse(BaseModel):
     """
     Fine-tune API response type
@@ -286,6 +314,9 @@ class FinetuneResponse(BaseModel):
     from_checkpoint: Optional[str] = None
     """Checkpoint used to continue training"""
 
+    multimodal_params: Optional[FinetuneMultimodalParams] = None
+    """Multimodal parameters"""
+
     from_hf_model: Optional[str] = None
     """Hugging Face Hub repo to start training from"""
 
@@ -393,6 +424,8 @@ class FinetuneResponse(BaseModel):
     training_file_size: Optional[int] = Field(None, alias="TrainingFileSize")
     train_on_inputs: Union[StrictBool, Literal["auto"], None] = "auto"
 
+    progress: Union[FinetuneProgress, None] = None
+
     @classmethod
     def validate_training_type(cls, v: TrainingType) -> TrainingType:
         if v.type == "Full" or v.type == "":
@@ -448,6 +481,9 @@ class FinetuneRequest(BaseModel):
     training_method: TrainingMethod = Field(default_factory=TrainingMethodSFT)
     # from step
     from_checkpoint: Union[str, None] = None
+    # multimodal parameters
+    multimodal_params: Union[FinetuneMultimodalParams, None] = None
+    # hugging face related fields
     from_hf_model: Union[str, None] = None
     hf_model_revision: Union[str, None] = None
     # hf related fields