anatools 5.1.24__py3-none-any.whl → 5.1.25__py3-none-any.whl

anatools/__init__.py

@@ -16,4 +16,4 @@ View the `Introduction to Rendered.ai Documentation`_ to learn more.
 from .anaclient.anaclient import client
 from .annotations.annotations import annotations
 
-__version__ = '5.1.24'
+__version__ = '5.1.25'

anatools/lib/transfer.py

@@ -17,11 +17,13 @@ from threading import Lock
 try:
     import boto3
     from botocore.exceptions import ClientError
+    from boto3.s3.transfer import TransferConfig
     BOTO3_AVAILABLE = True
 except ImportError:
     BOTO3_AVAILABLE = False
     boto3 = None
-    ClientError = Exception  # Placeholder
+    ClientError = Exception
+    TransferConfig = None  # Placeholder
 
 from anatools.lib.print import print_color
 
@@ -479,9 +481,16 @@ def upload_file_to_s3(s3_client, local_file, bucket, key, max_retries=3):
     Returns:
         bool: True if successful, False otherwise
     """
+    config = TransferConfig(
+        multipart_threshold=100 * 1024 * 1024,
+        max_concurrency=10,
+        multipart_chunksize=100 * 1024 * 1024,
+        use_threads=True
+    )
+
     for attempt in range(max_retries):
         try:
-            s3_client.upload_file(local_file, bucket, key)
+            s3_client.upload_file(local_file, bucket, key, Config=config)
             return True
         except ClientError as e:
             if attempt < max_retries - 1:
@@ -509,9 +518,16 @@ def download_file_from_s3(s3_client, bucket, key, local_file, max_retries=3):
     # Create directory if it doesn't exist
     os.makedirs(os.path.dirname(local_file), exist_ok=True)
 
+    config = TransferConfig(
+        multipart_threshold=100 * 1024 * 1024,
+        max_concurrency=10,
+        multipart_chunksize=100 * 1024 * 1024,
+        use_threads=True
+    )
+
     for attempt in range(max_retries):
         try:
-            s3_client.download_file(bucket, key, local_file)
+            s3_client.download_file(bucket, key, local_file, Config=config)
             return True
         except ClientError as e:
             if attempt < max_retries - 1:
@@ -525,6 +541,7 @@ def download_file_from_s3(s3_client, bucket, key, local_file, max_retries=3):
 def copy_s3_object(s3_client, source_bucket, source_key, dest_bucket, dest_key, max_retries=3):
     """
     Copy an object within S3 (server-side copy).
+    Uses multipart copy for files >5GB to handle large files efficiently.
 
     Args:
         s3_client: boto3 S3 client
@@ -539,6 +556,18 @@ def copy_s3_object(s3_client, source_bucket, source_key, dest_bucket, dest_key,
     """
     copy_source = {'Bucket': source_bucket, 'Key': source_key}
 
+    try:
+        head_response = s3_client.head_object(Bucket=source_bucket, Key=source_key)
+        file_size = head_response['ContentLength']
+    except ClientError as e:
+        print_color(f"Failed to get object metadata for s3://{source_bucket}/{source_key}: {e}", 'error')
+        return False
+
+    MULTIPART_THRESHOLD = 5 * 1024 * 1024 * 1024
+
+    if file_size >= MULTIPART_THRESHOLD:
+        return _multipart_copy_s3_object(s3_client, source_bucket, source_key, dest_bucket, dest_key, file_size, max_retries)
+
     for attempt in range(max_retries):
         try:
             s3_client.copy_object(
@@ -549,13 +578,101 @@ def copy_s3_object(s3_client, source_bucket, source_key, dest_bucket, dest_key,
             return True
         except ClientError as e:
             if attempt < max_retries - 1:
-                time.sleep(2 ** attempt)  # Exponential backoff
+                time.sleep(2 ** attempt)
             else:
                 print_color(f"Failed to copy s3://{source_bucket}/{source_key} to s3://{dest_bucket}/{dest_key}: {e}", 'error')
                 return False
     return False
 
 
+def _multipart_copy_s3_object(s3_client, source_bucket, source_key, dest_bucket, dest_key, file_size, max_retries=3):
+    """
+    Perform multipart copy for large S3 objects (>5GB).
+
+    Args:
+        s3_client: boto3 S3 client
+        source_bucket: Source S3 bucket
+        source_key: Source S3 key
+        dest_bucket: Destination S3 bucket
+        dest_key: Destination S3 key
+        file_size: Size of the source object in bytes
+        max_retries: Maximum number of retry attempts
+
+    Returns:
+        bool: True if successful, False otherwise
+    """
+    PART_SIZE = 500 * 1024 * 1024
+
+    try:
+        mpu = s3_client.create_multipart_upload(
+            Bucket=dest_bucket,
+            Key=dest_key
+        )
+        upload_id = mpu['UploadId']
+
+        parts = []
+        part_number = 1
+        bytes_copied = 0
+
+        while bytes_copied < file_size:
+            start_byte = bytes_copied
+            end_byte = min(bytes_copied + PART_SIZE - 1, file_size - 1)
+
+            for attempt in range(max_retries):
+                try:
+                    part_response = s3_client.upload_part_copy(
+                        Bucket=dest_bucket,
+                        Key=dest_key,
+                        PartNumber=part_number,
+                        UploadId=upload_id,
+                        CopySource={
+                            'Bucket': source_bucket,
+                            'Key': source_key
+                        },
+                        CopySourceRange=f'bytes={start_byte}-{end_byte}'
+                    )
+
+                    parts.append({
+                        'ETag': part_response['CopyPartResult']['ETag'],
+                        'PartNumber': part_number
+                    })
+                    break
+                except ClientError as e:
+                    if attempt < max_retries - 1:
+                        time.sleep(2 ** attempt)
+                    else:
+                        s3_client.abort_multipart_upload(
+                            Bucket=dest_bucket,
+                            Key=dest_key,
+                            UploadId=upload_id
+                        )
+                        print_color(f"Failed to copy part {part_number} of s3://{source_bucket}/{source_key}: {e}", 'error')
+                        return False
+
+            bytes_copied = end_byte + 1
+            part_number += 1
+
+        s3_client.complete_multipart_upload(
+            Bucket=dest_bucket,
+            Key=dest_key,
+            UploadId=upload_id,
+            MultipartUpload={'Parts': parts}
+        )
+        return True
+
+    except ClientError as e:
+        print_color(f"Failed multipart copy of s3://{source_bucket}/{source_key} to s3://{dest_bucket}/{dest_key}: {e}", 'error')
+        try:
+            s3_client.abort_multipart_upload(
+                Bucket=dest_bucket,
+                Key=dest_key,
+                UploadId=upload_id
+            )
+        except:
+            pass
+        return False
+
+
 def local_to_s3_transfer(client, source_path, dest_volume_id, dest_prefix='',
                          recursive=True, parallel=10, dry_run=False, verbose=False):
     """

{anatools-5.1.24.data → anatools-5.1.25.data}/scripts/anadeploy

@@ -418,7 +418,7 @@ try:
             if localservice.remotes and len(localservice.remotes) > 0:
                 for i, remote in enumerate(localservice.remotes):
                     try:
-                        if 'environment' not in remote: remote['environment'] == 'prod'
+                        if 'environment' not in remote: remote['environment'] = 'prod'
                         if client.environment == remote['environment']: 
                             s = client.get_services(serviceId=remote['serviceId'])[0]
                             organization = client.get_organizations(organizationId=s['organizationId'])[0]

{anatools-5.1.24.dist-info → anatools-5.1.25.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anatools
-Version: 5.1.24
+Version: 5.1.25
 Summary: Tools for development with the Rendered.ai Platform.
 Home-page: https://rendered.ai
 Author: Rendered AI, Inc

{anatools-5.1.24.dist-info → anatools-5.1.25.dist-info}/RECORD

@@ -1,4 +1,4 @@
-anatools/__init__.py,sha256=KVIC59BmwpX22GRK722YLUjO646VVk1kxz3YZBJrtYI,954
+anatools/__init__.py,sha256=NS0kdcxltaz_gtuNFFGIX74ZBHX6_4od1uo05xuipng,954
 anatools/anacreate.py,sha256=wK1GKxGSzFdeQtqBkGhpai1sFjSJaKZ2686zrVJPBd0,5622
 anatools/anaclient/__init__.py,sha256=tjCd-MMkWOjhjY1mlbGtHNhEdHLj-W687GiYJHsf_J0,50
 anatools/anaclient/_menu.py,sha256=WSD8h7_SPRq9F6I0ohzkA4L0aLUx7L9YtIihk9eiD_o,5965
@@ -120,7 +120,7 @@ anatools/lib/scene.py,sha256=ANTcP_UVNQQ_6WjgmqHF6u_9HpJsElKpqszSUiN75u4,8339
 anatools/lib/search_utils.py,sha256=DUu7fOIgTWPXItAjPlzfiMazrN9J8ytSjjKW2oWCLbw,3757
 anatools/lib/service.py,sha256=4HfNBKIj3cHqKxD2fJcxlP8MMD1FozHayAsuoRGWURI,2720
 anatools/lib/tools.py,sha256=C1vnbGU6yZJ9q1xsaUP6RDHTohfmm69KZRz7nIjrMLo,5847
-anatools/lib/transfer.py,sha256=YRCZ09w6vYL1b9abN4QOQzEAFgStTXsHyFfJdwWhMqQ,28776
+anatools/lib/transfer.py,sha256=QZ47zD5MsjhBY5jaQMNI84WELUm4QGwFvCjFDy9LOJU,32788
 anatools/nodes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 anatools/nodes/constants.py,sha256=D0nikvblG1XR5umQFqAgbHjUeh9m8eEDiaYOqfInegk,1115
 anatools/nodes/constants.yml,sha256=MZJjhUOFth5Mg8EPU-_wnQkJ9DqPJEzyiLsXyS9UVME,1285
@@ -140,17 +140,17 @@ anatools/nodes/volume_directory.py,sha256=oe721h7qOplRj6N6NpGuyY1HCM277NSYA-6Uo3
 anatools/nodes/volume_directory.yml,sha256=MbyuLUlcqWIlQadYcn4Rvf6roypqh5IiP3dP57TilbY,901
 anatools/nodes/volume_file.py,sha256=YA4zCyRvVzF_9mMefGx29JLE7o9i6-NPaC40BFGv4_c,557
 anatools/nodes/volume_file.yml,sha256=i8bo9QeQmTLeWjv9Rh4EDoDqwOBULNPV7SMLO5AK8DI,862
-anatools-5.1.24.data/scripts/ana,sha256=qe7LDHRgJRPyomzAumdcYy0D5SuaUVag0N8SVevpxcU,5739
-anatools-5.1.24.data/scripts/anadeploy,sha256=udgzz-NlSvJcnypfprFoS-wDad3QrPfZNIYeE1TsqRE,38217
-anatools-5.1.24.data/scripts/anamount,sha256=AiaUgaaVVREFY2FLYRmSA8xgSwbfiF9NJYi91SWRA1I,6186
-anatools-5.1.24.data/scripts/anaprofile,sha256=1YUUwHiSa4ORQsxVf2HaSakayNTwMTNeF2snEMSJQAM,9779
-anatools-5.1.24.data/scripts/anarules,sha256=vT2V-77DoL-o6mApSMVXjqeXRIWm44X3jPXdcNK29N0,8972
-anatools-5.1.24.data/scripts/anaserver,sha256=QB1k_vhXAFGMOi9SNIFwgzkzN5LzJeVLtIVkp1oHq4I,8343
-anatools-5.1.24.data/scripts/anatransfer,sha256=GbMLjgA3TP4Oo2mbUxWnkkSC4nKpw1DWta-WVfcNftw,14564
-anatools-5.1.24.data/scripts/anautils,sha256=fziapZuKuBO0VKRgb4C4Js8p9zxxh8OHQmmkNdo3t3E,9530
-anatools-5.1.24.dist-info/licenses/LICENSE,sha256=aw0uaPvFzrHLJxBvuRqUcE2_srfM32-1suya9HbZCY8,1072
-anatools-5.1.24.dist-info/METADATA,sha256=704ROOBSZ5RG7vb3wVfFjb1d9i86pNyleoeOqkYEoR8,8218
-anatools-5.1.24.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-anatools-5.1.24.dist-info/entry_points.txt,sha256=KsZUmvbH3HXC2CdVpE2GZNR2u_cJNVIbm6BnD658FgM,54
-anatools-5.1.24.dist-info/top_level.txt,sha256=p7xa5bG7NX8pSMJOvRunSz1d7rGPGsBd5-A4gzD4r6w,9
-anatools-5.1.24.dist-info/RECORD,,
+anatools-5.1.25.data/scripts/ana,sha256=qe7LDHRgJRPyomzAumdcYy0D5SuaUVag0N8SVevpxcU,5739
+anatools-5.1.25.data/scripts/anadeploy,sha256=GW0J6YzEvGY2TMbyzkfi-4Ov_gsw4qz2hHJwRx2o7nc,38216
+anatools-5.1.25.data/scripts/anamount,sha256=AiaUgaaVVREFY2FLYRmSA8xgSwbfiF9NJYi91SWRA1I,6186
+anatools-5.1.25.data/scripts/anaprofile,sha256=1YUUwHiSa4ORQsxVf2HaSakayNTwMTNeF2snEMSJQAM,9779
+anatools-5.1.25.data/scripts/anarules,sha256=vT2V-77DoL-o6mApSMVXjqeXRIWm44X3jPXdcNK29N0,8972
+anatools-5.1.25.data/scripts/anaserver,sha256=QB1k_vhXAFGMOi9SNIFwgzkzN5LzJeVLtIVkp1oHq4I,8343
+anatools-5.1.25.data/scripts/anatransfer,sha256=GbMLjgA3TP4Oo2mbUxWnkkSC4nKpw1DWta-WVfcNftw,14564
+anatools-5.1.25.data/scripts/anautils,sha256=fziapZuKuBO0VKRgb4C4Js8p9zxxh8OHQmmkNdo3t3E,9530
+anatools-5.1.25.dist-info/licenses/LICENSE,sha256=aw0uaPvFzrHLJxBvuRqUcE2_srfM32-1suya9HbZCY8,1072
+anatools-5.1.25.dist-info/METADATA,sha256=Pg2TjtE7TGhheLDoVYgyLS3qbZdcc5Iltr969jmhMC0,8218
+anatools-5.1.25.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+anatools-5.1.25.dist-info/entry_points.txt,sha256=KsZUmvbH3HXC2CdVpE2GZNR2u_cJNVIbm6BnD658FgM,54
+anatools-5.1.25.dist-info/top_level.txt,sha256=p7xa5bG7NX8pSMJOvRunSz1d7rGPGsBd5-A4gzD4r6w,9
+anatools-5.1.25.dist-info/RECORD,,