together 1.0.1__py3-none-any.whl → 1.1.0__py3-none-any.whl

together/constants.py

@@ -26,3 +26,6 @@ NUM_BYTES_IN_GB = 2**30
 
 # maximum number of GB sized files we support finetuning for
 MAX_FILE_SIZE_GB = 4.9
+
+# expected columns for Parquet files
+PARQUET_EXPECTED_COLUMNS = ["input_ids", "attention_mask", "labels"]

together/filemanager.py

@@ -25,7 +25,13 @@ from together.error import (
     FileTypeError,
 )
 from together.together_response import TogetherResponse
-from together.types import FilePurpose, FileResponse, TogetherClient, TogetherRequest
+from together.types import (
+    FilePurpose,
+    FileResponse,
+    FileType,
+    TogetherClient,
+    TogetherRequest,
+)
 
 
 def chmod_and_replace(src: Path, dst: Path) -> None:
@@ -260,12 +266,17 @@ class UploadManager:
                 http_status=response.status_code,
             )
 
-    def redirect_policy(
-        self, url: str, file: Path, purpose: FilePurpose
+    def get_upload_url(
+        self,
+        url: str,
+        file: Path,
+        purpose: FilePurpose,
+        filetype: FileType,
     ) -> Tuple[str, str]:
         data = {
             "purpose": purpose.value,
             "file_name": file.name,
+            "file_type": filetype.value,
         }
 
         requestor = api_requestor.APIRequestor(
@@ -324,7 +335,16 @@ class UploadManager:
 
         redirect_url = None
         if redirect:
-            redirect_url, file_id = self.redirect_policy(url, file, purpose)
+            if file.suffix == ".jsonl":
+                filetype = FileType.jsonl
+            elif file.suffix == ".parquet":
+                filetype = FileType.parquet
+            else:
+                raise FileTypeError(
+                    f"Unknown extension of file {file}. "
+                    "Only files with extensions .jsonl and .parquet are supported."
+                )
+            redirect_url, file_id = self.get_upload_url(url, file, purpose, filetype)
 
         file_size = os.stat(file.as_posix()).st_size
 

together/types/__init__.py

@@ -18,6 +18,7 @@ from together.types.files import (
     FilePurpose,
     FileRequest,
     FileResponse,
+    FileType,
 )
 from together.types.finetune import (
     FinetuneDownloadResult,
@@ -55,6 +56,7 @@ __all__ = [
     "FileDeleteResponse",
     "FileObject",
     "FilePurpose",
+    "FileType",
     "ImageRequest",
     "ImageResponse",
     "ModelObject",

together/types/files.py

@@ -15,6 +15,11 @@ class FilePurpose(str, Enum):
     FineTune = "fine-tune"
 
 
+class FileType(str, Enum):
+    jsonl = "jsonl"
+    parquet = "parquet"
+
+
 class FileRequest(BaseModel):
     """
     Files request type
@@ -43,21 +48,17 @@ class FileResponse(BaseModel):
     Files API response type
     """
 
-    # file id
     id: str
-    # object type
     object: Literal[ObjectType.File]
     # created timestamp
     created_at: int | None = None
-    # file purpose
+    type: FileType | None = None
     purpose: FilePurpose | None = None
-    # file-name
     filename: str | None = None
     # file byte size
     bytes: int | None = None
     # JSONL line count
     line_count: int | None = Field(None, alias="LineCount")
-    # is processed
     processed: bool | None = Field(None, alias="Processed")
 
 

together/utils/files.py

@@ -3,9 +3,17 @@ from __future__ import annotations
 import json
 import os
 from pathlib import Path
+from traceback import format_exc
 from typing import Any, Dict
 
-from together.constants import MAX_FILE_SIZE_GB, MIN_SAMPLES, NUM_BYTES_IN_GB
+from pyarrow import ArrowInvalid, parquet
+
+from together.constants import (
+    MAX_FILE_SIZE_GB,
+    MIN_SAMPLES,
+    NUM_BYTES_IN_GB,
+    PARQUET_EXPECTED_COLUMNS,
+)
 
 
 def check_file(
@@ -50,6 +58,25 @@ def check_file(
     else:
         report_dict["file_size"] = file_size
 
+    if file.suffix == ".jsonl":
+        report_dict["filetype"] = "jsonl"
+        data_report_dict = _check_jsonl(file)
+    elif file.suffix == ".parquet":
+        report_dict["filetype"] = "parquet"
+        data_report_dict = _check_parquet(file)
+    else:
+        report_dict["filetype"] = (
+            f"Unknown extension of file {file}. "
+            "Only files with extensions .jsonl and .parquet are supported."
+        )
+        report_dict["is_check_passed"] = False
+
+    report_dict.update(data_report_dict)
+    return report_dict
+
+
+def _check_jsonl(file: Path) -> Dict[str, Any]:
+    report_dict: Dict[str, Any] = {}
     # Check that the file is UTF-8 encoded. If not report where the error occurs.
     try:
         with file.open(encoding="utf-8") as f:
@@ -71,7 +98,7 @@ def check_file(
                 if not isinstance(json_line, dict):
                     report_dict["line_type"] = False
                     report_dict["message"] = (
-                        f"Error parsing file. Invalid format on line {idx+1} of the input file. "
+                        f"Error parsing file. Invalid format on line {idx + 1} of the input file. "
                         'Example of valid json: {"text": "my sample string"}. '
                     )
 
@@ -80,7 +107,7 @@ def check_file(
                 if "text" not in json_line.keys():
                     report_dict["text_field"] = False
                     report_dict["message"] = (
-                        f"Missing 'text' field was found on line {idx+1} of the the input file. "
+                        f"Missing 'text' field was found on line {idx + 1} of the the input file. "
                         "Expected format: {'text': 'my sample string'}. "
                     )
                     report_dict["is_check_passed"] = False
@@ -89,7 +116,7 @@ def check_file(
                     if not isinstance(json_line["text"], str):
                         report_dict["key_value"] = False
                         report_dict["message"] = (
-                            f'Invalid value type for "text" key on line {idx+1}. '
+                            f'Invalid value type for "text" key on line {idx + 1}. '
                             f'Expected string. Found {type(json_line["text"])}.'
                         )
 
@@ -99,7 +126,7 @@ def check_file(
             if idx + 1 < MIN_SAMPLES:
                 report_dict["min_samples"] = False
                 report_dict["message"] = (
-                    f"Processing {file} resulted in only {idx+1} samples. "
+                    f"Processing {file} resulted in only {idx + 1} samples. "
                     f"Our minimum is {MIN_SAMPLES} samples. "
                 )
                 report_dict["is_check_passed"] = False
@@ -118,7 +145,7 @@ def check_file(
                 )
             else:
                 report_dict["message"] = (
-                    f"Error parsing json payload. Unexpected format on line {idx+1}."
+                    f"Error parsing json payload. Unexpected format on line {idx + 1}."
                 )
             report_dict["is_check_passed"] = False
 
@@ -128,5 +155,50 @@ def check_file(
         report_dict["line_type"] = True
     if report_dict["key_value"] is not False:
         report_dict["key_value"] = True
+    return report_dict
+
+
+def _check_parquet(file: Path) -> Dict[str, Any]:
+    report_dict: Dict[str, Any] = {}
+
+    try:
+        table = parquet.read_table(str(file), memory_map=True)
+    except ArrowInvalid:
+        report_dict["load_parquet"] = (
+            f"An exception has occurred when loading the Parquet file {file}. Please check the file for corruption. "
+            f"Exception trace:\n{format_exc()}"
+        )
+        report_dict["is_check_passed"] = False
+        return report_dict
+
+    column_names = table.schema.names
+    if "input_ids" not in column_names:
+        report_dict["load_parquet"] = (
+            f"Parquet file {file} does not contain the `input_ids` column."
+        )
+        report_dict["is_check_passed"] = False
+        return report_dict
+
+    for column_name in column_names:
+        if column_name not in PARQUET_EXPECTED_COLUMNS:
+            report_dict["load_parquet"] = (
+                f"Parquet file {file} contains an unexpected column {column_name}. "
+                f"Only columns {PARQUET_EXPECTED_COLUMNS} are supported."
+            )
+            report_dict["is_check_passed"] = False
+            return report_dict
+
+    num_samples = len(table)
+    if num_samples < MIN_SAMPLES:
+        report_dict["min_samples"] = (
+            f"Processing {file} resulted in only {num_samples} samples. "
+            f"Our minimum is {MIN_SAMPLES} samples. "
+        )
+        report_dict["is_check_passed"] = False
+        return report_dict
+    else:
+        report_dict["num_samples"] = num_samples
+
+    report_dict["is_check_passed"] = True
 
     return report_dict

{together-1.0.1.dist-info → together-1.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: together
-Version: 1.0.1
+Version: 1.1.0
 Summary: Python client for Together's Cloud Platform!
 Home-page: https://github.com/togethercomputer/together-python
 License: Apache-2.0
@@ -19,7 +19,10 @@ Requires-Dist: aiohttp (>=3.9.3,<4.0.0)
 Requires-Dist: click (>=8.1.7,<9.0.0)
 Requires-Dist: eval-type-backport (>=0.1.3,<0.2.0)
 Requires-Dist: filelock (>=3.13.1,<4.0.0)
+Requires-Dist: numpy (>=1.23.5) ; python_version < "3.12"
+Requires-Dist: numpy (>=1.26.0) ; python_version >= "3.12"
 Requires-Dist: pillow (>=10.3.0,<11.0.0)
+Requires-Dist: pyarrow (>=10.0.1)
 Requires-Dist: pydantic (>=2.6.3,<3.0.0)
 Requires-Dist: requests (>=2.31.0,<3.0.0)
 Requires-Dist: tabulate (>=0.9.0,<0.10.0)

{together-1.0.1.dist-info → together-1.1.0.dist-info}/RECORD

@@ -11,9 +11,9 @@ together/cli/api/images.py,sha256=01dFYa2sK1HqUwVCD9FlwcjqkYWLoNxFZkzok13EriE,23
 together/cli/api/models.py,sha256=xWEzu8ZpxM_Pz9KEjRPRVuv_v22RayYZ4QcgiezT5tE,1126
 together/cli/cli.py,sha256=RC0tgapkSOFjsRPg8p-8dx9D2LDzm8YmVCHUjk_aVyQ,1977
 together/client.py,sha256=7QT5lwn7-QGf0vtgbhslQujm4986CgfE2spyMYP4JyU,4774
-together/constants.py,sha256=YxaViLyvBwch6eKC4FlebIRniHl4gMfav9g6PG73Eyk,801
+together/constants.py,sha256=WHe6JA9TliwgErkCnovWPS9w9xXfA3X5PtKJv_y2JxQ,908
 together/error.py,sha256=cILzwDde18INNFYgbYdgvVfOhEjRZM9sg9I8Rl6nc_Y,5329
-together/filemanager.py,sha256=q518tL5lcqSqGmwa2GzpO6cGn1Yazyzp7h0y_TATaE0,10916
+together/filemanager.py,sha256=tBr9LOqiv6G63rH7HYePKeKhyylJXfZFHW5Aybk8UUw,11438
 together/legacy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 together/legacy/base.py,sha256=ehrX1SCfRbK5OA83wL1q7-tfF-yuZOUxzjxYfFtdvvQ,727
 together/legacy/complete.py,sha256=8itEMlyuora2EIO-b-tvjvpHfKhxckDEiqaN-z6ccQs,2343
@@ -32,25 +32,25 @@ together/resources/finetune.py,sha256=gxkVOHuBgv19K8EBxhzyrXuJvTVe5b5FHofM9RHHlK
 together/resources/images.py,sha256=gFzXy7gLzr20KsXJXHEsOZtJJpuR6pH1d8XHCd6Dl9E,4775
 together/resources/models.py,sha256=2dtHhXAqTDOOpwSbYLzWcKTC0-m2Szlb7LDYvp7Jr4w,1786
 together/together_response.py,sha256=MhczUCPem93cjX-A1TOAUrRj3sO-o3SLcEcTsZgVzQI,1319
-together/types/__init__.py,sha256=L5-JgDRcczrriRYjRb0HzhXPsIgXcMdyjTDyztbfclk,1402
+together/types/__init__.py,sha256=K7Gv6hLmobIfqfmijZbZwFrwxK_YKIDR_v94_ElFmVA,1432
 together/types/abstract.py,sha256=1lFQI_3WjsR_t1128AeKW0aTk6EiM6Gh1J3ZuyLLPao,642
 together/types/chat_completions.py,sha256=WoIzd21wXLwen7AIM0Ttq9KFl4GsPOaJs-UMu2Bs110,3630
 together/types/common.py,sha256=0evjGduXV_tStd0TkGNwvU4fDjfLGUOuUJz6Vl5KYbs,1491
 together/types/completions.py,sha256=_ll3jCY7OhEJP9md_x2FrmPNGE4P_Nx44klIQ2BrfEc,2173
 together/types/embeddings.py,sha256=J7grkYYn7xhqeKaBO2T-8XQRtHhkzYzymovtGdIUK5A,751
 together/types/error.py,sha256=OVlCs3cx_2WhZK4JzHT8SQyRIIqKOP1AZQ4y1PydjAE,370
-together/types/files.py,sha256=m2jm6aPPdQ6ieMk2PUKozvFeMSKtWQ9vZugYoIcCaac,1934
+together/types/files.py,sha256=-rEUfsV6f2vZB9NrFxT4_933ubsDIUNkPB-3OlOFk4A,1954
 together/types/finetune.py,sha256=Td7zEk7ePcR9FTUwiGKl01mT-uRB6R5xzqQM0qmblX4,5779
 together/types/images.py,sha256=zX4Vt38tFDKU6yGb_hBY_N5eSTn3KPdpP5Ce_qnRHXQ,915
 together/types/models.py,sha256=3zag9x2fus2McNmLkr3fKzQL-2RNIT1-tiabI-z21p8,1013
 together/utils/__init__.py,sha256=VpjeRTya1m5eEE-Qe1zYTFsNAvuEA-dy7M2eG9Xu4fc,662
 together/utils/_log.py,sha256=yzdOV6iBEsyqF8UVvKhZm-ATtRokm34V-dXjTv3WKdE,1665
 together/utils/api_helpers.py,sha256=RSF7SRhbjHzroMOSWAXscflByM1r1ta_1SpxkAT22iE,2407
-together/utils/files.py,sha256=Fn4wNbv2zJPV7AvtZSiCJ8sN7g6xs_PAugwR5Q_G_kY,4835
+together/utils/files.py,sha256=rH407SdONtdqQAUJAFwhTJiLgiLaU5MldvCqsjPfMQ4,7186
 together/utils/tools.py,sha256=3-lXWP3cBCzOVSZg9tr5zOT1jaVeKAKVWxO2fcXZTh8,1788
 together/version.py,sha256=p03ivHyE0SyWU4jAnRTBi_sOwywVWoZPU4g2gzRgG-Y,126
-together-1.0.1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-together-1.0.1.dist-info/METADATA,sha256=J13nSD5FoJQCZOujerIxYDxQD73uTbhuhoE-llSqf84,10963
-together-1.0.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-together-1.0.1.dist-info/entry_points.txt,sha256=G-b5NKW6lUUf1V1fH8IPTBb7jXnK7lhbX9H1zTEJXPs,50
-together-1.0.1.dist-info/RECORD,,
+together-1.1.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+together-1.1.0.dist-info/METADATA,sha256=Y497GBOVVNallmm3tmuhM6qxa2slJIIeQuBLoyCjyN4,11114
+together-1.1.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+together-1.1.0.dist-info/entry_points.txt,sha256=G-b5NKW6lUUf1V1fH8IPTBb7jXnK7lhbX9H1zTEJXPs,50
+together-1.1.0.dist-info/RECORD,,