clarifai 10.8.0__py3-none-any.whl → 10.8.2__py3-none-any.whl

clarifai/__init__.py

@@ -1 +1 @@
-__version__ = "10.8.0"
+__version__ = "10.8.2"

clarifai/client/model.py

@@ -18,8 +18,9 @@ from clarifai.client.base import BaseClient
 from clarifai.client.dataset import Dataset
 from clarifai.client.input import Inputs
 from clarifai.client.lister import Lister
-from clarifai.constants.model import (MAX_MODEL_PREDICT_INPUTS, MODEL_EXPORT_TIMEOUT,
-                                      TRAINABLE_MODEL_TYPES)
+from clarifai.constants.model import (CHUNK_SIZE, MAX_CHUNK_SIZE, MAX_MODEL_PREDICT_INPUTS,
+                                      MAX_RANGE_SIZE, MIN_CHUNK_SIZE, MIN_RANGE_SIZE,
+                                      MODEL_EXPORT_TIMEOUT, RANGE_SIZE, TRAINABLE_MODEL_TYPES)
 from clarifai.errors import UserError
 from clarifai.urls.helper import ClarifaiUrlHelper
 from clarifai.utils.logging import get_logger
@@ -1296,20 +1297,52 @@ class Model(Lister, BaseClient):
       model_export_url = get_model_export_response.export.url
       model_export_file_size = get_model_export_response.export.size
 
-      session = requests.Session()
-      retries = Retry(total=10, backoff_factor=1, status_forcelist=[500, 502, 503, 504])
-      session.mount('https://', HTTPAdapter(max_retries=retries))
-      session.headers.update({'Authorization': self.metadata[0][1]})
-      response = session.get(model_export_url, stream=True)
-      response.raise_for_status()
-
       with open(local_filepath, 'wb') as f:
         progress = tqdm(
             total=model_export_file_size, unit='B', unit_scale=True, desc="Exporting model")
-        for chunk in response.iter_content(chunk_size=8192):
-          f.write(chunk)
-          progress.update(len(chunk))
-        progress.close()
+        downloaded_size = 0
+        range_size = RANGE_SIZE
+        chunk_size = CHUNK_SIZE
+        retry = False
+        retry_count = 0
+        while downloaded_size < model_export_file_size:
+          if downloaded_size + range_size >= model_export_file_size:
+            range_header = f"bytes={downloaded_size}-"
+          else:
+            range_header = f"bytes={downloaded_size}-{(downloaded_size+range_size-1)}"
+          try:
+            session = requests.Session()
+            retries = Retry(total=5, backoff_factor=0.1, status_forcelist=[500, 502, 503, 504])
+            session.mount('https://', HTTPAdapter(max_retries=retries))
+            session.headers.update({'Authorization': self.metadata[0][1], 'Range': range_header})
+            response = session.get(model_export_url, stream=True)
+            response.raise_for_status()
+
+            for chunk in response.iter_content(chunk_size=chunk_size):
+              f.write(chunk)
+              progress.update(len(chunk))
+            f.flush()
+            os.fsync(f.fileno())
+            downloaded_size += range_size
+            if not retry:
+              range_size = (
+                  range_size * 2) if (range_size * 2) < MAX_RANGE_SIZE else MAX_RANGE_SIZE
+              chunk_size = (
+                  chunk_size * 2) if (chunk_size * 2) < MAX_CHUNK_SIZE else MAX_CHUNK_SIZE
+          except Exception as e:
+            self.logger.error(f"Error downloading model: {e}")
+            range_size = (
+                range_size // 2) if (range_size // 2) > MIN_RANGE_SIZE else MIN_RANGE_SIZE
+            chunk_size = (
+                chunk_size // 2) if (chunk_size // 2) > MIN_CHUNK_SIZE else MIN_CHUNK_SIZE
+            retry = True
+            retry_count += 1
+            f.seek(downloaded_size)
+            progress.reset(total=model_export_file_size)
+            progress.update(downloaded_size)
+            if retry_count > 5:
+              break
+      progress.close()
 
       self.logger.info(
           f"Model ID {self.id} with version {self.model_info.model_version.id} exported successfully to {export_dir}/model.tar"

clarifai/constants/model.py

@@ -4,3 +4,9 @@ TRAINABLE_MODEL_TYPES = [
 ]
 MAX_MODEL_PREDICT_INPUTS = 128
 MODEL_EXPORT_TIMEOUT = 1800
+MIN_RANGE_SIZE = 4194304  # 4MB
+MAX_RANGE_SIZE = 314572800  # 300MB
+MIN_CHUNK_SIZE = 131072  # 128KB
+MAX_CHUNK_SIZE = 10485760  # 10MB
+RANGE_SIZE = 31457280  # 30MB
+CHUNK_SIZE = 1048576  # 1MB

clarifai/datasets/export/inputs_annotations.py

@@ -196,7 +196,7 @@ class InputAnnotationDownloader:
 
     if data_dict.get("concepts") or data_dict.get("regions"):
       file_name = os.path.join(split, "annotations", input_.id + ".json")
-      annot_data = data_dict.get("regions", []) + data_dict.get("concepts")
+      annot_data = data_dict.get("regions", []) + data_dict.get("concepts", [])
 
       self._save_annotation_to_archive(new_archive, annot_data, file_name)
       self.num_annotations += 1

{clarifai-10.8.0.dist-info → clarifai-10.8.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: clarifai
-Version: 10.8.0
+Version: 10.8.2
 Summary: Clarifai Python SDK
 Home-page: https://github.com/Clarifai/clarifai-python
 Author: Clarifai
@@ -20,7 +20,7 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: clarifai-grpc >=10.8.0
+Requires-Dist: clarifai-grpc >=10.8.6
 Requires-Dist: numpy >=1.22.0
 Requires-Dist: tqdm >=4.65.0
 Requires-Dist: tritonclient >=2.34.0
@@ -30,6 +30,7 @@ Requires-Dist: schema ==0.7.5
 Requires-Dist: Pillow >=9.5.0
 Requires-Dist: inquirerpy ==0.3.4
 Requires-Dist: tabulate >=0.9.0
+Requires-Dist: protobuf ==5.27.3
 Provides-Extra: all
 Requires-Dist: pycocotools ==2.0.6 ; extra == 'all'
 
@@ -162,7 +163,7 @@ app = client.create_app(app_id="demo_app", base_workflow="Universal")
 dataset = app.create_dataset(dataset_id="demo_dataset")
 
 # execute data upload to Clarifai app dataset
-from clarifai.datasets.upload.laoders.coco_detection import COCODetectionDataLoader
+from clarifai.datasets.upload.loaders.coco_detection import COCODetectionDataLoader
 coco_dataloader = COCODetectionDataLoader("images_dir", "coco_annotation_filepath")
 dataset.upload_dataset(dataloader=coco_dataloader, get_upload_status=True)
 

{clarifai-10.8.0.dist-info → clarifai-10.8.2.dist-info}/RECORD

@@ -1,4 +1,4 @@
-clarifai/__init__.py,sha256=x_bDnpR_W5o1ETeLzxk1ahCDd3RSOmf0Fs_F69i8AEo,23
+clarifai/__init__.py,sha256=FG8GLi_82vIRb8Dfpf5CyID9RUNkh4jIYFUZwrkh_cI,23
 clarifai/cli.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 clarifai/errors.py,sha256=RwzTajwds51wLD0MVlMC5kcpBnzRpreDLlazPSBZxrg,2605
 clarifai/versions.py,sha256=jctnczzfGk_S3EnVqb2FjRKfSREkNmvNEwAAa_VoKiQ,222
@@ -8,7 +8,7 @@ clarifai/client/base.py,sha256=JXbbjg2CXo8rOdw-XgKWWtLVAhPv3OZua5LFT5w4U2Q,7380
 clarifai/client/dataset.py,sha256=XX-J-9Ict1CQrEycq-JbdxUTuucSgLeDSvnlHE1ucQY,29903
 clarifai/client/input.py,sha256=ZLqa1jGx4NgCbunOTpJxCq4lDQ5xAf4GQ0rsZY8AHCM,44456
 clarifai/client/lister.py,sha256=03KGMvs5RVyYqxLsSrWhNc34I8kiF1Ph0NeyEwu7nMU,2082
-clarifai/client/model.py,sha256=TsIiId2a1eqR8Ct83f8KbqBsxiDT9XNPxqLSQtr3i1A,72930
+clarifai/client/model.py,sha256=-jwkuXKgKDGJJPSLj_b-b_yyHcSlBy_GS-9D-eHNf6s,74507
 clarifai/client/module.py,sha256=360JaOasX0DZCNE_Trj0LNTr-T_tUDZLfGpz0CdIi78,4248
 clarifai/client/search.py,sha256=GaPWN6JmTQGZaCHr6U1yv0zqR6wKFl7i9IVLg2ul1CI,14254
 clarifai/client/user.py,sha256=Fr3vDEHieqD7HRRKnlp9h-AIX4AuYBirRYtG4KLwSe8,11836
@@ -19,13 +19,13 @@ clarifai/client/auth/register.py,sha256=2CMdBsoVLoTfjyksE6j7BM2tiEc73WKYvxnwDDgN
 clarifai/client/auth/stub.py,sha256=xy4-fV0W8keCgXld4eOVzFQEIKxOktNwtL5bLztReug,4940
 clarifai/constants/dataset.py,sha256=Puz6_FfTm30G5FVBb1GJsobMkNtbg0Y2Soy7eyHjvtI,587
 clarifai/constants/input.py,sha256=WcHwToUVIK9ItAhDefaSohQHCLNeR55PSjZ0BFnoZ3U,28
-clarifai/constants/model.py,sha256=oTad43ncskVHfQ9vEbL2yy0Fac666dXr7QuO8zZXHAE,245
+clarifai/constants/model.py,sha256=Um1hLfMFlh5R_vtP3Z6P-o6zon-tdbLcKVIl4PucrV4,438
 clarifai/constants/rag.py,sha256=WcHwToUVIK9ItAhDefaSohQHCLNeR55PSjZ0BFnoZ3U,28
 clarifai/constants/search.py,sha256=yYEqTaFg-KdnpJE_Ytp-EPVHIIC395iNtZrpVlLIf4o,101
 clarifai/constants/workflow.py,sha256=cECq1xdvf44MCdtK2AbkiuuwhyL-6OWZdQfYbsLKy_o,33
 clarifai/datasets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 clarifai/datasets/export/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-clarifai/datasets/export/inputs_annotations.py,sha256=X1Y1XjnJrPrlwqv2aM4a1mBPUpWgEpz5stq2Vq-mLaM,9740
+clarifai/datasets/export/inputs_annotations.py,sha256=7c6HWdATI4aPCRoCPZetUBNNEz9dBhbyYX1QqX-xYe4,9744
 clarifai/datasets/upload/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 clarifai/datasets/upload/base.py,sha256=IP4sdBRfThk2l0W1rDWciFrAJnKwVsM-gu4zEslJ2_E,2198
 clarifai/datasets/upload/features.py,sha256=oq0PGpAw8LEafiSkdMMl0yn-NJeZ7K_CKzpJ71b0H40,1731
@@ -107,9 +107,9 @@ clarifai/workflows/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
 clarifai/workflows/export.py,sha256=vICRhIreqDSShxLKjHNM2JwzKsf1B4fdXB0ciMcA70k,1945
 clarifai/workflows/utils.py,sha256=nGeB_yjVgUO9kOeKTg4OBBaBz-AwXI3m-huSVj-9W18,1924
 clarifai/workflows/validate.py,sha256=yJq03MaJqi5AK3alKGJJBR89xmmjAQ31sVufJUiOqY8,2556
-clarifai-10.8.0.dist-info/LICENSE,sha256=mUqF_d12-qE2n41g7C5_sq-BMLOcj6CNN-jevr15YHU,555
-clarifai-10.8.0.dist-info/METADATA,sha256=4oERpF1zh5aLZU6ZUh4jh6XqXyl-4n9pnle4KlbwBOM,19372
-clarifai-10.8.0.dist-info/WHEEL,sha256=uCRv0ZEik_232NlR4YDw4Pv3Ajt5bKvMH13NUU7hFuI,91
-clarifai-10.8.0.dist-info/entry_points.txt,sha256=qZOr_MIPG0dBBE1zringDJS_wXNGTAA_SQ-zcbmDHOw,82
-clarifai-10.8.0.dist-info/top_level.txt,sha256=wUMdCQGjkxaynZ6nZ9FAnvBUCgp5RJUVFSy2j-KYo0s,9
-clarifai-10.8.0.dist-info/RECORD,,
+clarifai-10.8.2.dist-info/LICENSE,sha256=mUqF_d12-qE2n41g7C5_sq-BMLOcj6CNN-jevr15YHU,555
+clarifai-10.8.2.dist-info/METADATA,sha256=ckJDbErEFSTxwyILxVH4Grom2qN6dVjwX_F9zt7hlf0,19405
+clarifai-10.8.2.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+clarifai-10.8.2.dist-info/entry_points.txt,sha256=qZOr_MIPG0dBBE1zringDJS_wXNGTAA_SQ-zcbmDHOw,82
+clarifai-10.8.2.dist-info/top_level.txt,sha256=wUMdCQGjkxaynZ6nZ9FAnvBUCgp5RJUVFSy2j-KYo0s,9
+clarifai-10.8.2.dist-info/RECORD,,

{clarifai-10.8.0.dist-info → clarifai-10.8.2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (74.1.1)
+Generator: setuptools (75.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any