together 1.5.21__py3-none-any.whl → 1.5.23__py3-none-any.whl

together/cli/api/evaluation.py

@@ -0,0 +1,379 @@
+import json
+from typing import Optional, Dict, Union, Any
+
+import click
+from tabulate import tabulate
+
+from together import Together
+from together.utils import convert_unix_timestamp
+
+
+@click.group()
+@click.pass_context
+def evaluation(ctx: click.Context) -> None:
+    """Evaluation API commands"""
+    pass
+
+
+@evaluation.command()
+@click.pass_context
+@click.option(
+    "--type",
+    type=click.Choice(["classify", "score", "compare"]),
+    required=True,
+    help="Type of evaluation to create.",
+)
+@click.option(
+    "--judge-model-name",
+    type=str,
+    required=True,
+    help="Name of the judge model to use for evaluation.",
+)
+@click.option(
+    "--judge-system-template",
+    type=str,
+    required=True,
+    help="System template for the judge model.",
+)
+@click.option(
+    "--input-data-file-path",
+    type=str,
+    required=True,
+    help="Path to the input data file.",
+)
+@click.option(
+    "--model-field",
+    type=str,
+    help="Name of the field in the input file contaning text generated by the model."
+    "Can not be used when model-a-name and other model config parameters are specified",
+)
+@click.option(
+    "--model-to-evaluate-name",
+    type=str,
+    help="Model name when using the detailed config",
+)
+@click.option(
+    "--model-to-evaluate-max-tokens",
+    type=int,
+    help="Max tokens for model-to-evaluate",
+)
+@click.option(
+    "--model-to-evaluate-temperature",
+    type=float,
+    help="Temperature for model-to-evaluate",
+)
+@click.option(
+    "--model-to-evaluate-system-template",
+    type=str,
+    help="System template for model-to-evaluate",
+)
+@click.option(
+    "--model-to-evaluate-input-template",
+    type=str,
+    help="Input template for model-to-evaluate",
+)
+@click.option(
+    "--labels",
+    type=str,
+    help="Classification labels - comma-separated list",
+)
+@click.option(
+    "--pass-labels",
+    type=str,
+    help="Labels considered as passing (required for classify type). A comma-separated list.",
+)
+@click.option(
+    "--min-score",
+    type=float,
+    help="Minimum score value (required for score type).",
+)
+@click.option(
+    "--max-score",
+    type=float,
+    help="Maximum score value (required for score type).",
+)
+@click.option(
+    "--pass-threshold",
+    type=float,
+    help="Threshold score for passing (required for score type).",
+)
+@click.option(
+    "--model-a-field",
+    type=str,
+    help="Name of the field in the input file containing text generated by Model A. \
+        Can not be used when model-a-name and other model config parameters are specified",
+)
+@click.option(
+    "--model-a-name",
+    type=str,
+    help="Model name for model A when using detailed config.",
+)
+@click.option(
+    "--model-a-max-tokens",
+    type=int,
+    help="Max tokens for model A.",
+)
+@click.option(
+    "--model-a-temperature",
+    type=float,
+    help="Temperature for model A.",
+)
+@click.option(
+    "--model-a-system-template",
+    type=str,
+    help="System template for model A.",
+)
+@click.option(
+    "--model-a-input-template",
+    type=str,
+    help="Input template for model A.",
+)
+@click.option(
+    "--model-b-field",
+    type=str,
+    help="Name of the field in the input file containing text generated by Model B.\
+          Can not be used when model-b-name and other model config parameters are specified",
+)
+@click.option(
+    "--model-b-name",
+    type=str,
+    help="Model name for model B when using detailed config.",
+)
+@click.option(
+    "--model-b-max-tokens",
+    type=int,
+    help="Max tokens for model B.",
+)
+@click.option(
+    "--model-b-temperature",
+    type=float,
+    help="Temperature for model B.",
+)
+@click.option(
+    "--model-b-system-template",
+    type=str,
+    help="System template for model B.",
+)
+@click.option(
+    "--model-b-input-template",
+    type=str,
+    help="Input template for model B.",
+)
+def create(
+    ctx: click.Context,
+    type: str,
+    judge_model_name: str,
+    judge_system_template: str,
+    input_data_file_path: str,
+    model_field: Optional[str],
+    model_to_evaluate_name: Optional[str],
+    model_to_evaluate_max_tokens: Optional[int],
+    model_to_evaluate_temperature: Optional[float],
+    model_to_evaluate_system_template: Optional[str],
+    model_to_evaluate_input_template: Optional[str],
+    labels: str,
+    pass_labels: str,
+    min_score: Optional[float],
+    max_score: Optional[float],
+    pass_threshold: Optional[float],
+    model_a_field: Optional[str],
+    model_a_name: Optional[str],
+    model_a_max_tokens: Optional[int],
+    model_a_temperature: Optional[float],
+    model_a_system_template: Optional[str],
+    model_a_input_template: Optional[str],
+    model_b_field: Optional[str],
+    model_b_name: Optional[str],
+    model_b_max_tokens: Optional[int],
+    model_b_temperature: Optional[float],
+    model_b_system_template: Optional[str],
+    model_b_input_template: Optional[str],
+) -> None:
+    """Create a new evaluation job"""
+
+    client: Together = ctx.obj
+
+    # Convert strings to lists for labels
+    labels_list = labels.split(",") if labels else None
+    pass_labels_list = pass_labels.split(",") if pass_labels else None
+
+    # Build model configurations
+    model_to_evaluate_final: Union[Dict[str, Any], None, str] = None
+
+    # Check if any config parameters are provided
+    config_params_provided = any(
+        [
+            model_to_evaluate_name,
+            model_to_evaluate_max_tokens,
+            model_to_evaluate_temperature,
+            model_to_evaluate_system_template,
+            model_to_evaluate_input_template,
+        ]
+    )
+
+    if model_field:
+        # Simple mode: model_field is provided
+        if config_params_provided:
+            raise click.BadParameter(
+                "Cannot specify both --model-field and --model-to-evaluate-* parameters. "
+                "Use either --model-field alone if your input file has pre-generated responses, "
+                "or config parameters if you want to generate it on our end"
+            )
+        model_to_evaluate_final = model_field
+    elif config_params_provided:
+        # Config mode: config parameters are provided
+        model_to_evaluate_final = {
+            "model_name": model_to_evaluate_name,
+            "max_tokens": model_to_evaluate_max_tokens,
+            "temperature": model_to_evaluate_temperature,
+            "system_template": model_to_evaluate_system_template,
+            "input_template": model_to_evaluate_input_template,
+        }
+
+    # Build model-a configuration
+    model_a_final: Union[Dict[str, Any], None, str] = None
+    model_a_config_params = [
+        model_a_name,
+        model_a_max_tokens,
+        model_a_temperature,
+        model_a_system_template,
+        model_a_input_template,
+    ]
+
+    if model_a_field is not None:
+        # Simple mode: model_a_field is provided
+        if any(model_a_config_params):
+            raise click.BadParameter(
+                "Cannot specify both --model-a-field and config parameters (--model-a-name, etc.). "
+                "Use either --model-a-field alone if your input file has pre-generated responses, "
+                "or config parameters if you want to generate it on our end"
+            )
+        model_a_final = model_a_field
+    elif any(model_a_config_params):
+        # Config mode: config parameters are provided
+        model_a_final = {
+            "model_name": model_a_name,
+            "max_tokens": model_a_max_tokens,
+            "temperature": model_a_temperature,
+            "system_template": model_a_system_template,
+            "input_template": model_a_input_template,
+        }
+
+    # Build model-b configuration
+    model_b_final: Union[Dict[str, Any], None, str] = None
+    model_b_config_params = [
+        model_b_name,
+        model_b_max_tokens,
+        model_b_temperature,
+        model_b_system_template,
+        model_b_input_template,
+    ]
+
+    if model_b_field is not None:
+        # Simple mode: model_b_field is provided
+        if any(model_b_config_params):
+            raise click.BadParameter(
+                "Cannot specify both --model-b-field and config parameters (--model-b-name, etc.). "
+                "Use either --model-b-field alone if your input file has pre-generated responses, "
+                "or config parameters if you want to generate it on our end"
+            )
+        model_b_final = model_b_field
+    elif any(model_b_config_params):
+        # Config mode: config parameters are provided
+        model_b_final = {
+            "model_name": model_b_name,
+            "max_tokens": model_b_max_tokens,
+            "temperature": model_b_temperature,
+            "system_template": model_b_system_template,
+            "input_template": model_b_input_template,
+        }
+
+    try:
+        response = client.evaluation.create(
+            type=type,
+            judge_model_name=judge_model_name,
+            judge_system_template=judge_system_template,
+            input_data_file_path=input_data_file_path,
+            model_to_evaluate=model_to_evaluate_final,
+            labels=labels_list,
+            pass_labels=pass_labels_list,
+            min_score=min_score,
+            max_score=max_score,
+            pass_threshold=pass_threshold,
+            model_a=model_a_final,
+            model_b=model_b_final,
+        )
+    except ValueError as e:
+        raise click.BadParameter(str(e))
+
+    click.echo(json.dumps(response.model_dump(exclude_none=True), indent=4))
+
+
+@evaluation.command()
+@click.pass_context
+@click.option(
+    "--status",
+    type=str,
+    help="Filter by job status.",
+)
+@click.option(
+    "--limit",
+    type=int,
+    help="Limit number of results (max 100).",
+)
+def list(ctx: click.Context, status: Optional[str], limit: Optional[int]) -> None:
+    """List evaluation jobs"""
+
+    client: Together = ctx.obj
+
+    response = client.evaluation.list(status=status, limit=limit)
+
+    display_list = []
+    for job in response:
+        if job.parameters:
+            model = job.parameters.get("model_to_evaluate", "")
+            model_a = job.parameters.get("model_a", "")
+            model_b = job.parameters.get("model_b", "")
+        else:
+            model = ""
+
+        display_list.append(
+            {
+                "Workflow ID": job.workflow_id or "",
+                "Type": job.type,
+                "Status": job.status,
+                "Created At": job.created_at or 0,
+                "Model": model,
+                "Model A": model_a,
+                "Model B": model_b,
+            }
+        )
+
+    table = tabulate(display_list, headers="keys", tablefmt="grid", showindex=True)
+    click.echo(table)
+
+
+@evaluation.command()
+@click.pass_context
+@click.argument("evaluation_id", type=str, required=True)
+def retrieve(ctx: click.Context, evaluation_id: str) -> None:
+    """Get details of a specific evaluation job"""
+
+    client: Together = ctx.obj
+
+    response = client.evaluation.retrieve(evaluation_id=evaluation_id)
+
+    click.echo(json.dumps(response.model_dump(exclude_none=True), indent=4))
+
+
+@evaluation.command()
+@click.pass_context
+@click.argument("evaluation_id", type=str, required=True)
+def status(ctx: click.Context, evaluation_id: str) -> None:
+    """Get the status and results of a specific evaluation job"""
+
+    client: Together = ctx.obj
+
+    response = client.evaluation.status(evaluation_id=evaluation_id)
+
+    click.echo(json.dumps(response.model_dump(exclude_none=True), indent=4))

together/cli/api/finetune.py

@@ -304,13 +304,8 @@ def create(
             raise click.BadParameter(
                 f"LoRA fine-tuning is not supported for the model `{model}`"
             )
-        if training_method == "dpo":
-            default_batch_size = model_limits.lora_training.max_batch_size_dpo
-        else:
-            default_batch_size = model_limits.lora_training.max_batch_size
         default_values = {
             "lora_r": model_limits.lora_training.max_rank,
-            "batch_size": default_batch_size,
             "learning_rate": 1e-3,
         }
 
@@ -335,15 +330,6 @@ def create(
                     f"Please change the job type with --lora or remove `{param}` from the arguments"
                 )
 
-        batch_size_source = ctx.get_parameter_source("batch_size")  # type: ignore[attr-defined]
-        if batch_size_source == ParameterSource.DEFAULT:
-            if training_method == "dpo":
-                training_args["batch_size"] = (
-                    model_limits.full_training.max_batch_size_dpo
-                )
-            else:
-                training_args["batch_size"] = model_limits.full_training.max_batch_size
-
     if n_evals <= 0 and validation_file:
         log_warn(
             "Warning: You have specified a validation file but the number of evaluation loops is set to 0. No evaluations will be performed."

together/cli/cli.py

@@ -9,6 +9,7 @@ import together
 from together.cli.api.chat import chat, interactive
 from together.cli.api.completions import completions
 from together.cli.api.endpoints import endpoints
+from together.cli.api.evaluation import evaluation
 from together.cli.api.files import files
 from together.cli.api.finetune import fine_tuning
 from together.cli.api.images import images
@@ -74,6 +75,7 @@ main.add_command(files)
 main.add_command(fine_tuning)
 main.add_command(models)
 main.add_command(endpoints)
+main.add_command(evaluation)
 
 if __name__ == "__main__":
     main()

together/client.py

@@ -25,6 +25,7 @@ class Together:
     audio: resources.Audio
     batches: resources.Batches
     code_interpreter: CodeInterpreter
+    evaluation: resources.Evaluation
 
     # client options
     client: TogetherClient
@@ -92,6 +93,7 @@ class Together:
         self.endpoints = resources.Endpoints(self.client)
         self.code_interpreter = CodeInterpreter(self.client)
         self.batches = resources.Batches(self.client)
+        self.evaluation = resources.Evaluation(self.client)
 
 
 class AsyncTogether:
@@ -106,6 +108,7 @@ class AsyncTogether:
     audio: resources.AsyncAudio
     code_interpreter: CodeInterpreter
     batches: resources.AsyncBatches
+    evaluation: resources.AsyncEvaluation
     # client options
     client: TogetherClient
 
@@ -171,6 +174,7 @@ class AsyncTogether:
         self.audio = resources.AsyncAudio(self.client)
         self.code_interpreter = CodeInterpreter(self.client)
         self.batches = resources.AsyncBatches(self.client)
+        self.evaluation = resources.AsyncEvaluation(self.client)
 
 
 Client = Together

together/filemanager.py

@@ -73,8 +73,6 @@ def _get_file_size(
         if len(range_parts) == 2:
             total_size_in_bytes = int(range_parts[1])
 
-    assert total_size_in_bytes != 0, "Unable to retrieve remote file."
-
     return total_size_in_bytes
 
 
@@ -213,8 +211,6 @@ class DownloadManager:
                 if not fetch_metadata:
                     file_size = int(response.headers.get("content-length", 0))
 
-                assert file_size != 0, "Unable to retrieve remote file."
-
                 with tqdm(
                     total=file_size,
                     unit="B",
@@ -334,6 +330,8 @@ class UploadManager:
                 filetype = FileType.jsonl
             elif file.suffix == ".parquet":
                 filetype = FileType.parquet
+            elif file.suffix == ".csv":
+                filetype = FileType.csv
             else:
                 raise FileTypeError(
                     f"Unknown extension of file {file}. "

together/legacy/finetune.py

@@ -16,7 +16,7 @@ class Finetune:
         model: str,
         n_epochs: int = 1,
         n_checkpoints: int | None = 1,
-        batch_size: int | None = 32,
+        batch_size: int | Literal["max"] = "max",
         learning_rate: float = 0.00001,
         suffix: (
             str | None
@@ -43,7 +43,7 @@ class Finetune:
             model=model,
             n_epochs=n_epochs,
             n_checkpoints=n_checkpoints,
-            batch_size=batch_size if isinstance(batch_size, int) else "max",
+            batch_size=batch_size,
             learning_rate=learning_rate,
             suffix=suffix,
             wandb_api_key=wandb_api_key,

together/resources/__init__.py

@@ -9,6 +9,7 @@ from together.resources.images import AsyncImages, Images
 from together.resources.models import AsyncModels, Models
 from together.resources.rerank import AsyncRerank, Rerank
 from together.resources.batch import Batches, AsyncBatches
+from together.resources.evaluation import Evaluation, AsyncEvaluation
 
 
 __all__ = [
@@ -34,4 +35,6 @@ __all__ = [
     "Endpoints",
     "Batches",
     "AsyncBatches",
+    "Evaluation",
+    "AsyncEvaluation",
 ]

together/resources/batch.py

@@ -16,7 +16,6 @@ class Batches:
         self._client = client
 
     def create_batch(self, file_id: str, endpoint: str) -> BatchJob:
-
         requestor = api_requestor.APIRequestor(
             client=self._client,
         )