together 2.0.0a11__py3-none-any.whl → 2.0.0a12__py3-none-any.whl

together/_base_client.py

@@ -1774,7 +1774,7 @@ class AsyncAPIClient(BaseClient[httpx.AsyncClient, AsyncStream[Any]]):
         options: RequestOptions = {},
     ) -> ResponseT:
         opts = FinalRequestOptions.construct(
-            method="patch", url=path, json_data=body, files=to_httpx_files(files), **options
+            method="patch", url=path, json_data=body, files=await async_to_httpx_files(files), **options
         )
         return await self.request(cast_to, opts)
 

together/_version.py

@@ -1,4 +1,4 @@
 # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
 __title__ = "together"
-__version__ = "2.0.0-alpha.11"  # x-release-please-version
+__version__ = "2.0.0-alpha.12"  # x-release-please-version

together/lib/cli/api/fine_tuning.py

@@ -176,6 +176,12 @@ def fine_tuning(ctx: click.Context) -> None:
     help="Whether to mask the user messages in conversational data or prompts in instruction data. "
     "`auto` will automatically determine whether to mask the inputs based on the data format.",
 )
+@click.option(
+    "--train-vision",
+    type=bool,
+    default=False,
+    help="Whether to train the vision encoder. Only supported for multimodal models.",
+)
 @click.option(
     "--from-checkpoint",
     type=str,
@@ -231,6 +237,7 @@ def create(
     lora_dropout: float | None,
     lora_alpha: float | None,
     lora_trainable_modules: str | None,
+    train_vision: bool,
     suffix: str | None,
     wandb_api_key: str | None,
     wandb_base_url: str | None,
@@ -272,6 +279,7 @@ def create(
         lora_dropout=lora_dropout,
         lora_alpha=lora_alpha,
         lora_trainable_modules=lora_trainable_modules,
+        train_vision=train_vision,
         suffix=suffix,
         wandb_api_key=wandb_api_key,
         wandb_base_url=wandb_base_url,
@@ -363,6 +371,10 @@ def create(
             simpo_gamma=simpo_gamma or 0,
         )
 
+    if model_limits.supports_vision:
+        # Don't show price estimation for multimodal models yet
+        confirm = True
+
     finetune_price_estimation_result = client.fine_tuning.estimate_price(
         training_file=training_file,
         validation_file=validation_file,
@@ -426,9 +438,7 @@ def list(ctx: click.Context) -> None:
                 "Price": f"""${
                     finetune_price_to_dollars(float(str(i.total_price)))
                 }""",  # convert to string for mypy typing
-                "Progress": generate_progress_bar(
-                    i, datetime.now().astimezone(), use_rich=False
-                ),
+                "Progress": generate_progress_bar(i, datetime.now().astimezone(), use_rich=False),
             }
         )
     table = tabulate(display_list, headers="keys", tablefmt="grid", showindex=True)
@@ -449,9 +459,7 @@ def retrieve(ctx: click.Context, fine_tune_id: str) -> None:
     response.events = None
 
     rprint(JSON.from_data(response.model_json_schema()))
-    progress_text = generate_progress_bar(
-        response, datetime.now().astimezone(), use_rich=True
-    )
+    progress_text = generate_progress_bar(response, datetime.now().astimezone(), use_rich=True)
     prefix = f"Status: [bold]{response.status}[/bold],"
     rprint(f"{prefix} {progress_text}")
 

together/lib/cli/api/utils.py

@@ -28,9 +28,7 @@ class AutoIntParamType(click.ParamType):
             return int(value)
         except ValueError:
             self.fail(
-                _("{value!r} is not a valid {number_type}.").format(
-                    value=value, number_type=self.name
-                ),
+                _("{value!r} is not a valid {number_type}.").format(value=value, number_type=self.name),
                 param,
                 ctx,
             )
@@ -39,7 +37,7 @@ class AutoIntParamType(click.ParamType):
 class BooleanWithAutoParamType(click.ParamType):
     name = "boolean_or_auto"
 
-    def convert( # pyright: ignore[reportImplicitOverride]
+    def convert(  # pyright: ignore[reportImplicitOverride]
         self, value: str, param: click.Parameter | None, ctx: click.Context | None
     ) -> bool | Literal["auto"] | None:
         if value == "auto":
@@ -48,9 +46,7 @@ class BooleanWithAutoParamType(click.ParamType):
             return bool(value)
         except ValueError:
             self.fail(
-                _("{value!r} is not a valid {type}.").format(
-                    value=value, type=self.name
-                ),
+                _("{value!r} is not a valid {type}.").format(value=value, type=self.name),
                 param,
                 ctx,
             )
@@ -119,17 +115,13 @@ def generate_progress_bar(
                 return progress
 
             elapsed_time = (current_time - update_at).total_seconds()
-            ratio_filled = min(
-                elapsed_time / finetune_job.progress.seconds_remaining, 1.0
-            )
+            ratio_filled = min(elapsed_time / finetune_job.progress.seconds_remaining, 1.0)
             percentage = ratio_filled * 100
             filled = math.ceil(ratio_filled * _PROGRESS_BAR_WIDTH)
             bar = "█" * filled + "░" * (_PROGRESS_BAR_WIDTH - filled)
             time_left = "N/A"
             if finetune_job.progress.seconds_remaining > elapsed_time:
-                time_left = _human_readable_time(
-                    finetune_job.progress.seconds_remaining - elapsed_time
-                )
+                time_left = _human_readable_time(finetune_job.progress.seconds_remaining - elapsed_time)
             time_text = f"{time_left} left"
             progress = f"Progress: {bar} [bold]{percentage:>3.0f}%[/bold] [yellow]{time_text}[/yellow]"
 

together/lib/constants.py

@@ -37,6 +37,12 @@ NUM_BYTES_IN_GB = 2**30
 # maximum number of GB sized files we support finetuning for
 MAX_FILE_SIZE_GB = 50.1
 
+# Multimodal limits
+MAX_IMAGES_PER_EXAMPLE = 10
+MAX_IMAGE_BYTES = 10 * 1024 * 1024  # 10MB
+# Max length = Header length + base64 factor (4/3) * image bytes
+MAX_BASE64_IMAGE_LENGTH = len("data:image/jpeg;base64,") + 4 * MAX_IMAGE_BYTES // 3
+
 # expected columns for Parquet files
 PARQUET_EXPECTED_COLUMNS = ["input_ids", "attention_mask", "labels"]
 

together/lib/resources/fine_tuning.py

@@ -22,6 +22,7 @@ from together.lib.types.fine_tuning import (
     CosineLRSchedulerArgs,
     LinearLRSchedulerArgs,
     FinetuneTrainingLimits,
+    FinetuneMultimodalParams,
 )
 
 AVAILABLE_TRAINING_METHODS = {
@@ -51,6 +52,7 @@ def create_finetune_request(
     lora_dropout: float | None = 0,
     lora_alpha: float | None = None,
     lora_trainable_modules: str | None = "all-linear",
+    train_vision: bool = False,
     suffix: str | None = None,
     wandb_api_key: str | None = None,
     wandb_base_url: str | None = None,
@@ -207,6 +209,13 @@ def create_finetune_request(
             simpo_gamma=simpo_gamma,
         )
 
+    if model_limits.supports_vision:
+        multimodal_params = FinetuneMultimodalParams(train_vision=train_vision)
+    elif not model_limits.supports_vision and train_vision:
+        raise ValueError(f"Vision encoder training is not supported for the non-multimodal model `{model}`")
+    else:
+        multimodal_params = None
+
     finetune_request = FinetuneRequest(
         model=model,
         training_file=training_file,
@@ -227,6 +236,7 @@ def create_finetune_request(
         wandb_project_name=wandb_project_name,
         wandb_name=wandb_name,
         training_method=training_method_cls,  # pyright: ignore[reportPossiblyUnboundVariable]
+        multimodal_params=multimodal_params,
         from_checkpoint=from_checkpoint,
         from_hf_model=from_hf_model,
         hf_model_revision=hf_model_revision,
@@ -238,7 +248,10 @@ def create_finetune_request(
 
     return finetune_request, training_type_pe, training_method_pe
 
-def create_price_estimation_params(finetune_request: FinetuneRequest) -> tuple[pe_params.TrainingType, pe_params.TrainingMethod]:
+
+def create_price_estimation_params(
+    finetune_request: FinetuneRequest,
+) -> tuple[pe_params.TrainingType, pe_params.TrainingMethod]:
     training_type_cls: pe_params.TrainingType
     if isinstance(finetune_request.training_type, FullTrainingType):
         training_type_cls = pe_params.TrainingTypeFullTrainingType(
@@ -275,6 +288,7 @@ def create_price_estimation_params(finetune_request: FinetuneRequest) -> tuple[p
 
     return training_type_cls, training_method_cls
 
+
 def get_model_limits(client: Together, model: str) -> FinetuneTrainingLimits:
     """
     Requests training limits for a specific model

together/lib/types/fine_tuning.py

@@ -189,6 +189,7 @@ class TrainingMethodUnknown(BaseModel):
 
     method: str
 
+
 TrainingMethod: TypeAlias = Union[
     TrainingMethodSFT,
     TrainingMethodDPO,
@@ -202,6 +203,7 @@ class FinetuneTrainingLimits(BaseModel):
     min_learning_rate: float
     full_training: Optional[FinetuneFullTrainingLimits] = None
     lora_training: Optional[FinetuneLoraTrainingLimits] = None
+    supports_vision: bool = False
 
 
 class LinearLRSchedulerArgs(BaseModel):
@@ -249,6 +251,7 @@ class EmptyLRScheduler(BaseModel):
     lr_scheduler_type: Literal[""]
     lr_scheduler_args: None = None
 
+
 class UnknownLRScheduler(BaseModel):
     """
     Unknown learning rate scheduler
@@ -268,6 +271,14 @@ FinetuneLRScheduler: TypeAlias = Union[
 ]
 
 
+class FinetuneMultimodalParams(BaseModel):
+    """
+    Multimodal parameters
+    """
+
+    train_vision: bool = False
+
+
 class FinetuneProgress(BaseModel):
     """
     Fine-tune job progress
@@ -303,6 +314,9 @@ class FinetuneResponse(BaseModel):
     from_checkpoint: Optional[str] = None
     """Checkpoint used to continue training"""
 
+    multimodal_params: Optional[FinetuneMultimodalParams] = None
+    """Multimodal parameters"""
+
     from_hf_model: Optional[str] = None
     """Hugging Face Hub repo to start training from"""
 
@@ -467,6 +481,9 @@ class FinetuneRequest(BaseModel):
     training_method: TrainingMethod = Field(default_factory=TrainingMethodSFT)
     # from step
     from_checkpoint: Union[str, None] = None
+    # multimodal parameters
+    multimodal_params: Union[FinetuneMultimodalParams, None] = None
+    # hugging face related fields
     from_hf_model: Union[str, None] = None
     hf_model_revision: Union[str, None] = None
     # hf related fields

together/lib/utils/files.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import os
 import csv
 import json
-from typing import Any, Dict, List, cast
+from typing import Any, Dict, List, Union, cast
 from pathlib import Path
 from traceback import format_exc
 
@@ -13,8 +13,11 @@ from together.types import FilePurpose
 from together.lib.constants import (
     MIN_SAMPLES,
     DISABLE_TQDM,
+    MAX_IMAGE_BYTES,
     NUM_BYTES_IN_GB,
     MAX_FILE_SIZE_GB,
+    MAX_IMAGES_PER_EXAMPLE,
+    MAX_BASE64_IMAGE_LENGTH,
     PARQUET_EXPECTED_COLUMNS,
     REQUIRED_COLUMNS_MESSAGE,
     JSONL_REQUIRED_COLUMNS_MAP,
@@ -22,6 +25,15 @@ from together.lib.constants import (
     DatasetFormat,
 )
 
+# MessageContent is a string or a list of dicts with 'type': 'text' or 'image_url', and 'text' or 'image_url.url'
+# Example: "Hello" or [
+#   {"type": "text", "text": "Hello"},
+#   {"type": "image_url", "image_url": {
+#     "url": "data:image/jpeg;base64,..."
+#   }}
+# ]
+MessageContent = Union[str, List[Dict[str, Any]]]
+
 
 class InvalidFileFormatError(ValueError):
     """Exception raised for invalid file formats during file checks."""
@@ -103,7 +115,7 @@ def check_file(
     return report_dict
 
 
-def _check_conversation_type(messages: List[Dict[str, str | bool]], idx: int) -> None:
+def _check_conversation_type(messages: List[Dict[str, str | int | MessageContent]], idx: int) -> None:
     """Check that the conversation has correct type.
 
     Args:
@@ -144,12 +156,6 @@ def _check_conversation_type(messages: List[Dict[str, str | bool]], idx: int) ->
                     line_number=idx + 1,
                     error_source="key_value",
                 )
-            if not isinstance(message[column], str):
-                raise InvalidFileFormatError(
-                    message=f"Column `{column}` is not a string on line {idx + 1}. Found {type(message[column])}",
-                    line_number=idx + 1,
-                    error_source="text_field",
-                )
 
 
 def _check_conversation_roles(require_assistant_role: bool, assistant_role_exists: bool, idx: int) -> None:
@@ -172,7 +178,7 @@ def _check_conversation_roles(require_assistant_role: bool, assistant_role_exist
         )
 
 
-def _check_message_weight(message: Dict[str, str | bool], idx: int) -> None:
+def _check_message_weight(message: Dict[str, str | int | MessageContent], idx: int) -> int | None:
     """Check that the message has a weight with the correct type and value.
 
     Args:
@@ -196,9 +202,12 @@ def _check_message_weight(message: Dict[str, str | bool], idx: int) -> None:
                 line_number=idx + 1,
                 error_source="key_value",
             )
+        return weight
+
+    return None
 
 
-def _check_message_role(message: Dict[str, str | bool], previous_role: str | bool | None, idx: int) -> str | bool:
+def _check_message_role(message: Dict[str, str | int | MessageContent], previous_role: str | None, idx: int) -> str:
     """Check that the message has correct roles.
 
     Args:
@@ -212,6 +221,14 @@ def _check_message_role(message: Dict[str, str | bool], previous_role: str | boo
     Raises:
         InvalidFileFormatError: If the message role is invalid.
     """
+    if not isinstance(message["role"], str):
+        raise InvalidFileFormatError(
+            message=f"Invalid role `{message['role']}` in conversation on line {idx + 1}. "
+            f"Role must be a string. Found {type(message['role'])}",
+            line_number=idx + 1,
+            error_source="key_value",
+        )
+
     if message["role"] not in POSSIBLE_ROLES_CONVERSATION:
         raise InvalidFileFormatError(
             message=f"Invalid role `{message['role']}` in conversation on line {idx + 1}. "
@@ -229,7 +246,130 @@ def _check_message_role(message: Dict[str, str | bool], previous_role: str | boo
     return message["role"]
 
 
-def validate_messages(messages: List[Dict[str, str | bool]], idx: int, require_assistant_role: bool = True) -> None:
+def _check_message_content(message_content: str | int | MessageContent, role: str, idx: int) -> tuple[bool, int]:
+    """Check that the message content has the correct type.
+    Message content can be either a) a string or b) an OpenAI-style multimodal list of content items
+    Example:
+        a) "Hello", or
+        b) [
+             {"type": "text", "text": "Hello"},
+             {"type": "image_url", "image_url": {
+                "url": "data:image/jpeg;base64,..."
+             }}
+           ]
+
+    Args:
+        message_content: The message content to check.
+        role: The role of the message.
+        idx: Line number in the file.
+
+    Returns:
+        tuple[bool, int]: A tuple with message is multimodal and the number of images in the message content.
+    """
+    # Text-only message content
+    if isinstance(message_content, str):
+        return False, 0
+
+    # Multimodal message content
+    if isinstance(message_content, list):
+        num_images = 0
+        for item in message_content:
+            if not isinstance(cast(Any, item), dict):
+                raise InvalidFileFormatError(
+                    "The dataset is malformed, the `content` field must be a list of dicts.",
+                    line_number=idx + 1,
+                    error_source="key_value",
+                )
+            if "type" not in item:
+                raise InvalidFileFormatError(
+                    "The dataset is malformed, the `content` field must be a list of dicts with a `type` field.",
+                    line_number=idx + 1,
+                    error_source="key_value",
+                )
+
+            if item["type"] == "text":
+                if "text" not in item or not isinstance(item["text"], str):
+                    raise InvalidFileFormatError(
+                        "The dataset is malformed, the `text` field must be present in the `content` item field and be"
+                        f" a string. Got '{item.get('text')!r}' instead.",
+                        line_number=idx + 1,
+                        error_source="key_value",
+                    )
+            elif item["type"] == "image_url":
+                if role != "user":
+                    raise InvalidFileFormatError(
+                        "The dataset is malformed, only user messages can contain images.",
+                        line_number=idx + 1,
+                        error_source="key_value",
+                    )
+
+                if "image_url" not in item or not isinstance(item["image_url"], dict):
+                    raise InvalidFileFormatError(
+                        "The dataset is malformed, the `image_url` field must be present in the `content` field and "
+                        f"be a dictionary. Got {item.get('image_url')!r} instead.",
+                        line_number=idx + 1,
+                        error_source="key_value",
+                    )
+
+                image_data = cast(Any, item["image_url"]).get("url")
+                if not image_data or not isinstance(image_data, str):
+                    raise InvalidFileFormatError(
+                        "The dataset is malformed, the `url` field must be present in the `image_url` field and be "
+                        f"a string. Got {image_data!r} instead.",
+                        line_number=idx + 1,
+                        error_source="key_value",
+                    )
+
+                if not any(image_data.startswith(f"data:image/{fmt};base64,") for fmt in ["jpeg", "png", "webp"]):
+                    raise InvalidFileFormatError(
+                        "The dataset is malformed, the `url` field must be either a JPEG, PNG or WEBP base64-encoded "
+                        "image in 'data:image/<format>;base64,<base64_encoded_image>' format. "
+                        f"Got '{image_data[:100]}...' instead.",
+                        line_number=idx + 1,
+                    )
+
+                if len(image_data) > MAX_BASE64_IMAGE_LENGTH:
+                    raise InvalidFileFormatError(
+                        "The dataset is malformed, the `url` field must contain base64-encoded image "
+                        f"that is less than {MAX_IMAGE_BYTES // (1024**2)}MB, found ~{len(image_data) * 3 // 4} bytes.",
+                        line_number=idx + 1,
+                        error_source="key_value",
+                    )
+
+                num_images += 1
+            else:
+                raise InvalidFileFormatError(
+                    "The dataset is malformed, the `type` field must be either 'text' or 'image_url'. "
+                    f"Got {item['type']!r}.",
+                    line_number=idx + 1,
+                    error_source="key_value",
+                )
+
+        if num_images > MAX_IMAGES_PER_EXAMPLE:
+            raise InvalidFileFormatError(
+                f"The dataset is malformed, the `content` field must contain at most "
+                f"{MAX_IMAGES_PER_EXAMPLE} images, found {num_images}.",
+                line_number=idx + 1,
+                error_source="key_value",
+            )
+
+        # We still consider text-only messages in such format as multimodal, even if they don't have any images
+        # included - so we can process datasets with rather sparse images (i.e. not in each sample) consistently.
+        return True, num_images
+
+    raise InvalidFileFormatError(
+        f"Invalid content type on line {idx + 1} of the input file. Expected string or multimodal list of dicts, "
+        f"found {type(message_content)}",
+        line_number=idx + 1,
+        error_source="key_value",
+    )
+
+
+def validate_messages(
+    messages: List[Dict[str, str | int | MessageContent]],
+    idx: int,
+    require_assistant_role: bool = True,
+) -> None:
     """Validate the messages column.
 
     Args:
@@ -242,15 +382,43 @@ def validate_messages(messages: List[Dict[str, str | bool]], idx: int, require_a
     """
     _check_conversation_type(messages, idx)
 
-    has_weights = any("weight" in message for message in messages)
     previous_role = None
     assistant_role_exists = False
 
+    messages_are_multimodal: bool | None = None
+    total_number_of_images = 0
+
     for message in messages:
-        if has_weights:
-            _check_message_weight(message, idx)
+        message_weight = _check_message_weight(message, idx)
         previous_role = _check_message_role(message, previous_role, idx)
         assistant_role_exists |= previous_role == "assistant"
+        is_multimodal, number_of_images = _check_message_content(message["content"], role=previous_role, idx=idx)
+        # Multimodal validation
+        if number_of_images > 0 and message_weight is not None and message_weight != 0:
+            raise InvalidFileFormatError(
+                "Messages with images cannot have non-zero weights.",
+                line_number=idx + 1,
+                error_source="key_value",
+            )
+        if messages_are_multimodal is None:
+            # Detect the format of the messages in the conversation.
+            messages_are_multimodal = is_multimodal
+        elif messages_are_multimodal != is_multimodal:
+            # Due to the format limitation, we cannot mix multimodal and text only messages in the same sample.
+            raise InvalidFileFormatError(
+                "Messages in the conversation must be either all in multimodal or all in text-only format.",
+                line_number=idx + 1,
+                error_source="key_value",
+            )
+        total_number_of_images += number_of_images
+
+    if total_number_of_images > MAX_IMAGES_PER_EXAMPLE:
+        raise InvalidFileFormatError(
+            f"The dataset is malformed, the `messages` must contain at most {MAX_IMAGES_PER_EXAMPLE} images. "
+            f"Found {total_number_of_images} images.",
+            line_number=idx + 1,
+            error_source="key_value",
+        )
 
     _check_conversation_roles(require_assistant_role, assistant_role_exists, idx)
 
@@ -279,7 +447,7 @@ def validate_preference_openai(example: Dict[str, Any], idx: int = 0) -> None:
             error_source="key_value",
         )
 
-    messages: List[Dict[str, str | bool]] = cast(Any, example["input"]["messages"])
+    messages: List[Dict[str, str | int | MessageContent]] = cast(Any, example["input"]["messages"])
     validate_messages(messages, idx, require_assistant_role=False)
 
     if example["input"]["messages"][-1]["role"] == "assistant":
@@ -341,12 +509,7 @@ def validate_preference_openai(example: Dict[str, Any], idx: int = 0) -> None:
                 error_source="key_value",
             )
 
-        if not isinstance(example[key][0]["content"], str):
-            raise InvalidFileFormatError(
-                message=f"The dataset is malformed, the 'content' field in `{key}` must be a string on line {idx + 1}.",
-                line_number=idx + 1,
-                error_source="key_value",
-            )
+        _check_message_content(example[key][0]["content"], role="assistant", idx=idx)
 
 
 def _check_utf8(file: Path) -> Dict[str, Any]:
@@ -514,7 +677,7 @@ def _check_jsonl(file: Path, purpose: FilePurpose | str) -> Dict[str, Any]:
                     elif current_format == DatasetFormat.CONVERSATION:
                         message_column = JSONL_REQUIRED_COLUMNS_MAP[DatasetFormat.CONVERSATION][0]
                         require_assistant = purpose != "eval"
-                        message: List[Dict[str, str | bool]] = cast(Any, json_line[message_column])
+                        message: List[Dict[str, str | int | MessageContent]] = cast(Any, json_line[message_column])
                         validate_messages(
                             message,
                             idx,
@@ -522,13 +685,8 @@ def _check_jsonl(file: Path, purpose: FilePurpose | str) -> Dict[str, Any]:
                         )
                     else:
                         for column in JSONL_REQUIRED_COLUMNS_MAP[current_format]:
-                            if not isinstance(json_line[column], str):
-                                raise InvalidFileFormatError(
-                                    message=f'Invalid value type for "{column}" key on line {idx + 1}. '
-                                    f"Expected string. Found {type(cast(Any, json_line[column]))}.",
-                                    line_number=idx + 1,
-                                    error_source="key_value",
-                                )
+                            role = "assistant" if column in {"completion"} else "user"
+                            _check_message_content(cast(Any, json_line[column]), role=role, idx=idx)
 
                     if dataset_format is None:
                         dataset_format = current_format

together/resources/fine_tuning.py

@@ -53,6 +53,7 @@ _WARNING_MESSAGE_INSUFFICIENT_FUNDS = (
     "Proceed at your own risk."
 )
 
+
 class FineTuningResource(SyncAPIResource):
     @cached_property
     def with_raw_response(self) -> FineTuningResourceWithRawResponse:
@@ -95,6 +96,7 @@ class FineTuningResource(SyncAPIResource):
         lora_dropout: float | None = 0,
         lora_alpha: float | None = None,
         lora_trainable_modules: str | None = "all-linear",
+        train_vision: bool = False,
         suffix: str | None = None,
         wandb_api_key: str | None = None,
         wandb_base_url: str | None = None,
@@ -140,6 +142,7 @@ class FineTuningResource(SyncAPIResource):
             lora_dropout (float, optional): Dropout rate for LoRA adapters. Defaults to 0.
             lora_alpha (float, optional): Alpha for LoRA adapters. Defaults to 8.
             lora_trainable_modules (str, optional): Trainable modules for LoRA adapters. Defaults to "all-linear".
+            train_vision (bool, optional): Whether to train the vision encoder (Only for multimodal models). Defaults to False.
             suffix (str, optional): Up to 40 character suffix that will be added to your fine-tuned model name.
                 Defaults to None.
             wandb_api_key (str, optional): API key for Weights & Biases integration.
@@ -214,6 +217,7 @@ class FineTuningResource(SyncAPIResource):
             lora_dropout=lora_dropout,
             lora_alpha=lora_alpha,
             lora_trainable_modules=lora_trainable_modules,
+            train_vision=train_vision,
             suffix=suffix,
             wandb_api_key=wandb_api_key,
             wandb_base_url=wandb_base_url,
@@ -232,29 +236,32 @@ class FineTuningResource(SyncAPIResource):
             hf_output_repo_name=hf_output_repo_name,
         )
 
-
-        price_estimation_result = self.estimate_price(
-            training_file=training_file,
-            from_checkpoint=from_checkpoint or Omit(),
-            validation_file=validation_file or Omit(),
-            model=model or "",
-            n_epochs=finetune_request.n_epochs,
-            n_evals=finetune_request.n_evals or 0,
-            training_type=training_type_cls,
-            training_method=training_method_cls,
-        )
-
+        if not model_limits.supports_vision:
+            price_estimation_result = self.estimate_price(
+                training_file=training_file,
+                from_checkpoint=from_checkpoint or Omit(),
+                validation_file=validation_file or Omit(),
+                model=model or "",
+                n_epochs=finetune_request.n_epochs,
+                n_evals=finetune_request.n_evals or 0,
+                training_type=training_type_cls,
+                training_method=training_method_cls,
+            )
+            price_limit_passed = price_estimation_result.allowed_to_proceed
+        else:
+            # unsupported case
+            price_limit_passed = True
 
         if verbose:
             rprint(
                 "Submitting a fine-tuning job with the following parameters:",
                 finetune_request,
             )
-            if not price_estimation_result.allowed_to_proceed:
+            if not price_limit_passed:
                 rprint(
                     "[red]"
                     + _WARNING_MESSAGE_INSUFFICIENT_FUNDS.format(
-                        price_estimation_result.estimated_total_price # pyright: ignore[reportPossiblyUnboundVariable]
+                        price_estimation_result.estimated_total_price  # pyright: ignore[reportPossiblyUnboundVariable]
                     )
                     + "[/red]",
                 )
@@ -627,6 +634,7 @@ class AsyncFineTuningResource(AsyncAPIResource):
         lora_dropout: float | None = 0,
         lora_alpha: float | None = None,
         lora_trainable_modules: str | None = "all-linear",
+        train_vision: bool = False,
         suffix: str | None = None,
         wandb_api_key: str | None = None,
         wandb_base_url: str | None = None,
@@ -672,6 +680,7 @@ class AsyncFineTuningResource(AsyncAPIResource):
             lora_dropout (float, optional): Dropout rate for LoRA adapters. Defaults to 0.
             lora_alpha (float, optional): Alpha for LoRA adapters. Defaults to 8.
             lora_trainable_modules (str, optional): Trainable modules for LoRA adapters. Defaults to "all-linear".
+            train_vision (bool, optional): Whether to train the vision encoder (Only for multimodal models). Defaults to False.
             suffix (str, optional): Up to 40 character suffix that will be added to your fine-tuned model name.
                 Defaults to None.
             wandb_api_key (str, optional): API key for Weights & Biases integration.
@@ -746,6 +755,7 @@ class AsyncFineTuningResource(AsyncAPIResource):
             lora_dropout=lora_dropout,
             lora_alpha=lora_alpha,
             lora_trainable_modules=lora_trainable_modules,
+            train_vision=train_vision,
             suffix=suffix,
             wandb_api_key=wandb_api_key,
             wandb_base_url=wandb_base_url,
@@ -764,7 +774,6 @@ class AsyncFineTuningResource(AsyncAPIResource):
             hf_output_repo_name=hf_output_repo_name,
         )
 
-
         price_estimation_result = await self.estimate_price(
             training_file=training_file,
             from_checkpoint=from_checkpoint or Omit(),
@@ -776,7 +785,6 @@ class AsyncFineTuningResource(AsyncAPIResource):
             training_method=training_method_cls,
         )
 
-
         if verbose:
             rprint(
                 "Submitting a fine-tuning job with the following parameters:",
@@ -786,7 +794,7 @@ class AsyncFineTuningResource(AsyncAPIResource):
                 rprint(
                     "[red]"
                     + _WARNING_MESSAGE_INSUFFICIENT_FUNDS.format(
-                        price_estimation_result.estimated_total_price # pyright: ignore[reportPossiblyUnboundVariable]
+                        price_estimation_result.estimated_total_price  # pyright: ignore[reportPossiblyUnboundVariable]
                     )
                     + "[/red]",
                 )

together/types/finetune_response.py

@@ -15,6 +15,7 @@ __all__ = [
     "LrSchedulerLrSchedulerArgs",
     "LrSchedulerLrSchedulerArgsLinearLrSchedulerArgs",
     "LrSchedulerLrSchedulerArgsCosineLrSchedulerArgs",
+    "MultimodalParams",
     "Progress",
     "TrainingMethod",
     "TrainingMethodTrainingMethodSft",
@@ -49,6 +50,14 @@ class LrScheduler(BaseModel):
     lr_scheduler_args: Optional[LrSchedulerLrSchedulerArgs] = None
 
 
+class MultimodalParams(BaseModel):
+    train_vision: Optional[bool] = None
+    """Whether to train the vision encoder of the model.
+
+    Only available for multimodal models.
+    """
+
+
 class Progress(BaseModel):
     """Progress information for a fine-tuning job"""
 
@@ -150,6 +159,8 @@ class FinetuneResponse(BaseModel):
 
     x_model_output_path: Optional[str] = FieldInfo(alias="model_output_path", default=None)
 
+    multimodal_params: Optional[MultimodalParams] = None
+
     n_checkpoints: Optional[int] = None
 
     n_epochs: Optional[int] = None

{together-2.0.0a11.dist-info → together-2.0.0a12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: together
-Version: 2.0.0a11
+Version: 2.0.0a12
 Summary: The official Python library for the together API
 Project-URL: Homepage, https://github.com/togethercomputer/together-py
 Project-URL: Repository, https://github.com/togethercomputer/together-py
@@ -183,7 +183,7 @@ stream = client.chat.completions.create(
     messages=[
         {
             "role": "user",
-            "content": "Say this is a test",
+            "content": "Say this is a test!",
         }
     ],
     model="meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo",
@@ -204,7 +204,7 @@ stream = await client.chat.completions.create(
     messages=[
         {
             "role": "user",
-            "content": "Say this is a test",
+            "content": "Say this is a test!",
         }
     ],
     model="meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo",

{together-2.0.0a11.dist-info → together-2.0.0a12.dist-info}/RECORD

@@ -1,5 +1,5 @@
 together/__init__.py,sha256=ghwEH6EUrPERUwHVSXaCJVqS7QmLN7NsUxKJNXQrOYM,2842
-together/_base_client.py,sha256=yP8nFDC-tFtGBrXHhcfcW4xF7KzAV5GQCLKlCZyp8sg,67237
+together/_base_client.py,sha256=U6Lhqesx9l1qhaYKaP2jR-_mIyqbNhAOo2r2bgTQegI,67249
 together/_client.py,sha256=Nw2fyh2kf3RABNaHuqWKKhTujaHttGXoUZENECGaqyI,37788
 together/_compat.py,sha256=DQBVORjFb33zch24jzkhM14msvnzY7mmSmgDLaVFUM8,6562
 together/_constants.py,sha256=i39tJ7BP8nnqvdHFJwMhN6LWR6-jg5LLYiFudwCD3Ic,463
@@ -11,7 +11,7 @@ together/_resource.py,sha256=-ZTq9O5qf2YsgjJk_gwJs-CM_OG4p6gdMLcNWjuxFwQ,1112
 together/_response.py,sha256=lvqEsCbpD8SRJTjlhhUFGbnLUR_4-Qva-OApxfVdiY4,28800
 together/_streaming.py,sha256=sk6fVYbpdO3Y-0S5iwZTHQJ3N24UkK0KaupgUTftWZk,11825
 together/_types.py,sha256=LzaeqN09mUAEvRg_XrLzihdOaW0D_R9qrG7jKsFjnQY,7297
-together/_version.py,sha256=F4XFYcFCYn77Ecwy5AT-x99HNaeVVRSV5vbTWx512a8,169
+together/_version.py,sha256=OH5dGztmdGmTCfPzpSFY8dmywHPRzyc6lLUVwFJNg8w,169
 together/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 together/_utils/__init__.py,sha256=7fch0GT9zpNnErbciSpUNa-SjTxxjY6kxHxKMOM4AGs,2305
 together/_utils/_compat.py,sha256=rN17SSvjMoQE1GmKFTLniRuG1sKj2WAD5VjdLPeRlF0,1231
@@ -27,25 +27,25 @@ together/_utils/_typing.py,sha256=N_5PPuFNsaygbtA_npZd98SVN1LQQvFTKL6bkWPBZGU,47
 together/_utils/_utils.py,sha256=g9ftElB09kVT6EVfCIlD_nUfANhDX5_vZO61FDWoIQI,12334
 together/lib/.keep,sha256=wuNrz-5SXo3jJaJOJgz4vFHM41YH_g20F5cRQo0vLes,224
 together/lib/__init__.py,sha256=Qtdi6geFNzxE-F51eNDk1ESXYyYDt8b82MR1POANQBQ,394
-together/lib/constants.py,sha256=pFWjUtItT-6_-Gpky0P2Q7bSj4iRmApbSqkcYetE7V0,1996
+together/lib/constants.py,sha256=w8-zVl8XZiJxqMdhbWekigHJ0JUMPoV9R3ejUHIcUJk,2237
 together/lib/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 together/lib/cli/cli.py,sha256=bNzYeLF8JdlMnSmIqFClp28MzjLGCwQ9hqSpaXHBQ0s,1939
 together/lib/cli/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 together/lib/cli/api/endpoints.py,sha256=MyliUrTuJWw2qFd80J27pFs9xTazIVAP0mqgRYxdVsw,14851
 together/lib/cli/api/evals.py,sha256=KkSvz2wIYmPQ3sFQBte6inNBZt1aptIkMVL5TKWTW5k,19074
 together/lib/cli/api/files.py,sha256=HbflC45PpzBIF0CE0TLucQaVr319ScL05VyAFKf2T6Y,3596
-together/lib/cli/api/fine_tuning.py,sha256=-PdNR_16WiwI_wZkqemb_m5SRceRuSvpOdEL8LGFGJU,22571
+together/lib/cli/api/fine_tuning.py,sha256=Tb4J9_LnHZHp73zNW1lHNSl_0UQfOd0yi9-IvMHBY08,22863
 together/lib/cli/api/models.py,sha256=Jfrl7gcbWAkbBQ1i1gCy485HHT2C4C784OMaaHZPiPw,4084
-together/lib/cli/api/utils.py,sha256=1W9Ebc_deJ6vss1IBGa80SFx61xENCyvnYRYwjCZ_Xo,4524
+together/lib/cli/api/utils.py,sha256=j4IYurqcoqisstAQuqWMsUKbMQITNX8ax2Vv6U6qF7I,4381
 together/lib/resources/__init__.py,sha256=ystIb0pBHQLuwUBtHJwhRgtjK3_TV6K0KuM8NGuuNoU,172
 together/lib/resources/files.py,sha256=Z_D23IvjYYWBpYrfYolCNfUslJBcE4PnU0WtuLsN67M,37277
-together/lib/resources/fine_tuning.py,sha256=-yCS6AIbgXpAcUkeqJcia8CPrnVcmyiaUN4dvksEjcc,12642
+together/lib/resources/fine_tuning.py,sha256=A-hOJqcGSPzw24wwX6K27OqV3B-u43dfdrK4nj4ItTg,13088
 together/lib/types/__init__.py,sha256=1-kHsAp9Sh9HxjTGKfdHnF1nTS_cM_Tazv-3Z9hrEbY,205
 together/lib/types/error.py,sha256=i-rnTZPRZuJDUf1lM-52abG2JHWOUBTCh55zPNGoakg,135
-together/lib/types/fine_tuning.py,sha256=uK_hke1huHjbjzQQBzz7EYMj1UHgCFVS0o7AG5L5xwY,12942
+together/lib/types/fine_tuning.py,sha256=gTB66x4jrmxyWuKJGcqX_4aYcI2iGgZvL3MzMLePDC8,13325
 together/lib/utils/__init__.py,sha256=F_CVqnvK-aEshMg-5FLFincPbhuVbsM6IKSCNyEByKs,545
 together/lib/utils/_log.py,sha256=mo5tDhyFTNqEj8MOcpy3bLmLBcC0OQ67orTw_nxFdcU,1930
-together/lib/utils/files.py,sha256=S6orZixBPeRtV_iq_IktuYHIm41irrHuOuexz_NYZJ0,24863
+together/lib/utils/files.py,sha256=CVTFwI7yMzpaQ-GsGr1tD4O2kXA-i369Pi0eMnlWMmI,31854
 together/lib/utils/serializer.py,sha256=wJwySGxAL0e1giZzFpl4hHH3s9lkoNN_yzu-P_jdRIo,287
 together/lib/utils/tools.py,sha256=rrpz3EXEVViou5GDPjVoCSt2zDPJYDzWYqTsVO1-OgI,2183
 together/resources/__init__.py,sha256=Cuiy4FcdrfUzb0N-jZbl8Phqjvlzt12Iq7BhI9tFsXw,7577
@@ -55,7 +55,7 @@ together/resources/embeddings.py,sha256=7EU6DZQd0Nm0Sh7x7v37QQOLNuLqNmcjdJAyOTck
 together/resources/endpoints.py,sha256=dYdLlAJ0P7HJNhzZGxlbzEQYpUWsh35cjAMVfdWiifw,27884
 together/resources/evals.py,sha256=FPjvkbsBY5rrzLyQ-X1G9fWt2QmivI9ol5GExGtqYVA,16216
 together/resources/files.py,sha256=0paHeVqNt3NQCXoztCgFS8PEIg_-mMVto-ulHTr7GzE,16854
-together/resources/fine_tuning.py,sha256=rpdUFb2p7_AGn8TIn7dPuhcFCCsuE-yuUGzo8S-epO8,54170
+together/resources/fine_tuning.py,sha256=BiCxQpdTjW5ArBufmWHNQoYY4z7Ulge8dI9GDCa5Dow,54795
 together/resources/hardware.py,sha256=xgfCmMrrwF5o1igax0JGec8RY7kkS0s4kKm62RdC3ME,6850
 together/resources/images.py,sha256=mVPQYpDHKBjLVO_Sv0uT62zYXdtWKN2PW3fCvfQLQCs,12612
 together/resources/jobs.py,sha256=TnzSnvJw4x5pqo1xzrkYH8f0viZrzyOqT-_w7xc0NzY,7797
@@ -117,7 +117,7 @@ together/types/fine_tuning_list_events_response.py,sha256=DeDJLF1IxQV47HOwfuVt8Z
 together/types/fine_tuning_list_response.py,sha256=vdVZBQ4V1GFzHUBiRR0drEvKg1ASTyG5g84P0oM1S_c,5992
 together/types/finetune_event.py,sha256=0apAXe6Anx2_ffse2pOBJDxngCeuSvuDMYczZ0DtzZg,787
 together/types/finetune_event_type.py,sha256=Bm4lkBhsLI_kaD5yabsvW6BpnjXzZO_lwDtiEeMNXnw,824
-together/types/finetune_response.py,sha256=7W2_HBLCleIAF_u-qwvh8f4QEMsVyYkwtSPO3BqFI2Q,4708
+together/types/finetune_response.py,sha256=klN6uuZt1tQ_PJ3rt7OSd_Cf7f0MuUnFZ4uJ6ISdrEU,4975
 together/types/hardware_list_params.py,sha256=BbfiigtdQE0QNGFGr6o-Twg912x_riH5mbUNpZWYWO4,435
 together/types/hardware_list_response.py,sha256=KfGhnEy7qEW2Bzt4Q8b-JSvxG-IKIIFfcpWEQHS9YdM,1732
 together/types/image_data_b64.py,sha256=pLY7JDBb1HF1T29ACbae_xn6JQfttpqQVeG_jJeenZU,284
@@ -159,8 +159,8 @@ together/types/chat/chat_completion_warning.py,sha256=_Dp7YKlxyY2HeZopTvT-Go7qqK
 together/types/chat/completion_create_params.py,sha256=Ia2dWuVxjderCRfpqV4Zpl-9rVng0p8deILUXVC3O0s,13687
 together/types/code_interpreter/__init__.py,sha256=dAXfb3ryLMtcBalCfxxNu2wJVswVP8G1xXryZnahPQY,201
 together/types/code_interpreter/session_list_response.py,sha256=TRxLGFTmIY-KLpStKjJtsrm4EI6BBvakpx43B6pkhnw,662
-together-2.0.0a11.dist-info/METADATA,sha256=YQHojaT_Y39KC8RAq-bktHXAKF3SUO4YEd2jrscrp9U,20247
-together-2.0.0a11.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
-together-2.0.0a11.dist-info/entry_points.txt,sha256=4f4RAX89wQkx3AnfHXiGrKyg2fCPnwMd2UdPX48OczA,55
-together-2.0.0a11.dist-info/licenses/LICENSE,sha256=5I5MO2DiiBFcD_p4ZF2T4GDb-WeBMD591ALtADdtXDc,11338
-together-2.0.0a11.dist-info/RECORD,,
+together-2.0.0a12.dist-info/METADATA,sha256=9QxYAaPhjBTRxpvWRO3CIFG_ebYHjTzRwgQ2f2QJwko,20249
+together-2.0.0a12.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
+together-2.0.0a12.dist-info/entry_points.txt,sha256=4f4RAX89wQkx3AnfHXiGrKyg2fCPnwMd2UdPX48OczA,55
+together-2.0.0a12.dist-info/licenses/LICENSE,sha256=oSs-kmJHhMue4vIIPIxQMvXou9PbxgNdIX-r_AwfO7c,11338
+together-2.0.0a12.dist-info/RECORD,,

{together-2.0.0a11.dist-info → together-2.0.0a12.dist-info}/licenses/LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2025 Together
+   Copyright 2026 Together
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.