anatools 5.1.23__py3-none-any.whl → 5.1.25__py3-none-any.whl

anatools/__init__.py

@@ -16,4 +16,4 @@ View the `Introduction to Rendered.ai Documentation`_ to learn more.
 from .anaclient.anaclient import client
 from .annotations.annotations import annotations
 
-__version__ = '5.1.23'
+__version__ = '5.1.25'

anatools/anaclient/api/volumes.py

@@ -46,10 +46,16 @@ def getVolumeData(self, volumeId, keys=[], dir=None, recursive=False, cursor=Non
                 "cursor": cursor,
                 "filters": filters
             },
-            "query": f"""query 
+            "query": f"""query
                 getVolumeData($volumeId: String!, $keys: [String], $dir: String, $recursive: Boolean, $limit: Int, $cursor: String, $filters: VolumeDataFilter) {{
                     getVolumeData(volumeId: $volumeId, keys: $keys, dir: $dir, recursive: $recursive, limit: $limit, cursor: $cursor, filters: $filters) {{
                        keys {{{fields}}}
+                       pageInfo {{
+                           totalItems
+                           cursor
+                           offset
+                           limit
+                       }}
                     }}
                 }}"""})
     return self.errorhandler(response, "getVolumeData")

anatools/anaclient/volumes.py

@@ -144,17 +144,17 @@ def get_volume_data(self, volumeId, files=None, dir=None, recursive=False, curso
     self.check_logout()
     if volumeId is None: raise Exception('The volumeId parameter is required!')
     if limit is not None and limit <= 100: items = limit
+    else: items = 100
     if files is None: files = []
     if dir is None: dir = ''
-    if cursor is None: cursor = 0
-    else: items = 100
     volumedata = []
     while True:
         if limit and len(volumedata) + items > limit: items = limit - len(volumedata)
-        ret = self.ana_api.getVolumeData(volumeId=volumeId, keys=files, dir=dir, recursive=recursive, cursor=str(cursor), limit=limit)
-        volumedata.extend(ret)
-        if len(ret) < items or len(volumedata) == limit: break
-        cursor += items
+        ret = self.ana_api.getVolumeData(volumeId=volumeId, keys=files, dir=dir, recursive=recursive, cursor=cursor, limit=items)
+        volumedata.extend(ret['keys'])
+        if len(ret['keys']) < items or len(volumedata) == limit: break
+        cursor = ret['pageInfo']['cursor'] if ret.get('pageInfo') and ret['pageInfo'].get('cursor') else None
+        if cursor is None: break
     return volumedata
 
 

anatools/lib/transfer.py

@@ -17,11 +17,13 @@ from threading import Lock
 try:
     import boto3
     from botocore.exceptions import ClientError
+    from boto3.s3.transfer import TransferConfig
     BOTO3_AVAILABLE = True
 except ImportError:
     BOTO3_AVAILABLE = False
     boto3 = None
-    ClientError = Exception  # Placeholder
+    ClientError = Exception
+    TransferConfig = None  # Placeholder
 
 from anatools.lib.print import print_color
 
@@ -479,9 +481,16 @@ def upload_file_to_s3(s3_client, local_file, bucket, key, max_retries=3):
     Returns:
         bool: True if successful, False otherwise
     """
+    config = TransferConfig(
+        multipart_threshold=100 * 1024 * 1024,
+        max_concurrency=10,
+        multipart_chunksize=100 * 1024 * 1024,
+        use_threads=True
+    )
+
     for attempt in range(max_retries):
         try:
-            s3_client.upload_file(local_file, bucket, key)
+            s3_client.upload_file(local_file, bucket, key, Config=config)
             return True
         except ClientError as e:
             if attempt < max_retries - 1:
@@ -509,9 +518,16 @@ def download_file_from_s3(s3_client, bucket, key, local_file, max_retries=3):
     # Create directory if it doesn't exist
     os.makedirs(os.path.dirname(local_file), exist_ok=True)
 
+    config = TransferConfig(
+        multipart_threshold=100 * 1024 * 1024,
+        max_concurrency=10,
+        multipart_chunksize=100 * 1024 * 1024,
+        use_threads=True
+    )
+
     for attempt in range(max_retries):
         try:
-            s3_client.download_file(bucket, key, local_file)
+            s3_client.download_file(bucket, key, local_file, Config=config)
             return True
         except ClientError as e:
             if attempt < max_retries - 1:
@@ -525,6 +541,7 @@ def download_file_from_s3(s3_client, bucket, key, local_file, max_retries=3):
 def copy_s3_object(s3_client, source_bucket, source_key, dest_bucket, dest_key, max_retries=3):
     """
     Copy an object within S3 (server-side copy).
+    Uses multipart copy for files >5GB to handle large files efficiently.
 
     Args:
         s3_client: boto3 S3 client
@@ -539,6 +556,18 @@ def copy_s3_object(s3_client, source_bucket, source_key, dest_bucket, dest_key,
     """
     copy_source = {'Bucket': source_bucket, 'Key': source_key}
 
+    try:
+        head_response = s3_client.head_object(Bucket=source_bucket, Key=source_key)
+        file_size = head_response['ContentLength']
+    except ClientError as e:
+        print_color(f"Failed to get object metadata for s3://{source_bucket}/{source_key}: {e}", 'error')
+        return False
+
+    MULTIPART_THRESHOLD = 5 * 1024 * 1024 * 1024
+
+    if file_size >= MULTIPART_THRESHOLD:
+        return _multipart_copy_s3_object(s3_client, source_bucket, source_key, dest_bucket, dest_key, file_size, max_retries)
+
     for attempt in range(max_retries):
         try:
             s3_client.copy_object(
@@ -549,13 +578,101 @@ def copy_s3_object(s3_client, source_bucket, source_key, dest_bucket, dest_key,
             return True
         except ClientError as e:
             if attempt < max_retries - 1:
-                time.sleep(2 ** attempt)  # Exponential backoff
+                time.sleep(2 ** attempt)
             else:
                 print_color(f"Failed to copy s3://{source_bucket}/{source_key} to s3://{dest_bucket}/{dest_key}: {e}", 'error')
                 return False
     return False
 
 
+def _multipart_copy_s3_object(s3_client, source_bucket, source_key, dest_bucket, dest_key, file_size, max_retries=3):
+    """
+    Perform multipart copy for large S3 objects (>5GB).
+
+    Args:
+        s3_client: boto3 S3 client
+        source_bucket: Source S3 bucket
+        source_key: Source S3 key
+        dest_bucket: Destination S3 bucket
+        dest_key: Destination S3 key
+        file_size: Size of the source object in bytes
+        max_retries: Maximum number of retry attempts
+
+    Returns:
+        bool: True if successful, False otherwise
+    """
+    PART_SIZE = 500 * 1024 * 1024
+
+    try:
+        mpu = s3_client.create_multipart_upload(
+            Bucket=dest_bucket,
+            Key=dest_key
+        )
+        upload_id = mpu['UploadId']
+
+        parts = []
+        part_number = 1
+        bytes_copied = 0
+
+        while bytes_copied < file_size:
+            start_byte = bytes_copied
+            end_byte = min(bytes_copied + PART_SIZE - 1, file_size - 1)
+
+            for attempt in range(max_retries):
+                try:
+                    part_response = s3_client.upload_part_copy(
+                        Bucket=dest_bucket,
+                        Key=dest_key,
+                        PartNumber=part_number,
+                        UploadId=upload_id,
+                        CopySource={
+                            'Bucket': source_bucket,
+                            'Key': source_key
+                        },
+                        CopySourceRange=f'bytes={start_byte}-{end_byte}'
+                    )
+
+                    parts.append({
+                        'ETag': part_response['CopyPartResult']['ETag'],
+                        'PartNumber': part_number
+                    })
+                    break
+                except ClientError as e:
+                    if attempt < max_retries - 1:
+                        time.sleep(2 ** attempt)
+                    else:
+                        s3_client.abort_multipart_upload(
+                            Bucket=dest_bucket,
+                            Key=dest_key,
+                            UploadId=upload_id
+                        )
+                        print_color(f"Failed to copy part {part_number} of s3://{source_bucket}/{source_key}: {e}", 'error')
+                        return False
+
+            bytes_copied = end_byte + 1
+            part_number += 1
+
+        s3_client.complete_multipart_upload(
+            Bucket=dest_bucket,
+            Key=dest_key,
+            UploadId=upload_id,
+            MultipartUpload={'Parts': parts}
+        )
+        return True
+
+    except ClientError as e:
+        print_color(f"Failed multipart copy of s3://{source_bucket}/{source_key} to s3://{dest_bucket}/{dest_key}: {e}", 'error')
+        try:
+            s3_client.abort_multipart_upload(
+                Bucket=dest_bucket,
+                Key=dest_key,
+                UploadId=upload_id
+            )
+        except:
+            pass
+        return False
+
+
 def local_to_s3_transfer(client, source_path, dest_volume_id, dest_prefix='',
                          recursive=True, parallel=10, dry_run=False, verbose=False):
     """

{anatools-5.1.23.data → anatools-5.1.25.data}/scripts/anadeploy

@@ -321,6 +321,7 @@ try:
                 if selection == str(options-2): remotechannel = create_channel(client=client, organization=remoteorganization, name=args.channel.split('/')[-1].split('.')[0], volumes=volumes)
                 else:
                     channels = sorted(client.get_channels(organizationId=remoteorganization['organizationId']), key=lambda x: x['name'].lower())
+                    channels = [c for c in channels if c['organizationId'] == remoteorganization['organizationId']]
                     if len(channels) > 0:
                         print_color("Choose a channel to deploy to: ", color='brand')
                         for i, channel in enumerate(channels):
@@ -417,7 +418,7 @@ try:
             if localservice.remotes and len(localservice.remotes) > 0:
                 for i, remote in enumerate(localservice.remotes):
                     try:
-                        if 'environment' not in remote: remote['environment'] == 'prod'
+                        if 'environment' not in remote: remote['environment'] = 'prod'
                         if client.environment == remote['environment']: 
                             s = client.get_services(serviceId=remote['serviceId'])[0]
                             organization = client.get_organizations(organizationId=s['organizationId'])[0]
@@ -480,6 +481,7 @@ try:
                 if selection == str(options-2): remoteservice = create_service(client=client, organization=remoteorganization, name=localservice.name, volumes=volumes)
                 else:
                     services = sorted(client.get_services(organizationId=remoteorganization['organizationId']), key=lambda x: x['name'].lower())
+                    services = [s for s in services if s['organizationId'] == remoteorganization['organizationId']]
                     if len(services) > 0:
                         print_color("Choose a service to deploy to: ", color='brand')
                         for i, service in enumerate(services):

{anatools-5.1.23.dist-info → anatools-5.1.25.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anatools
-Version: 5.1.23
+Version: 5.1.25
 Summary: Tools for development with the Rendered.ai Platform.
 Home-page: https://rendered.ai
 Author: Rendered AI, Inc

{anatools-5.1.23.dist-info → anatools-5.1.25.dist-info}/RECORD

@@ -1,4 +1,4 @@
-anatools/__init__.py,sha256=oCvITQd3wdDEjbgafcqf2cV8z4u76klewKrwnoCrtUY,954
+anatools/__init__.py,sha256=NS0kdcxltaz_gtuNFFGIX74ZBHX6_4od1uo05xuipng,954
 anatools/anacreate.py,sha256=wK1GKxGSzFdeQtqBkGhpai1sFjSJaKZ2686zrVJPBd0,5622
 anatools/anaclient/__init__.py,sha256=tjCd-MMkWOjhjY1mlbGtHNhEdHLj-W687GiYJHsf_J0,50
 anatools/anaclient/_menu.py,sha256=WSD8h7_SPRq9F6I0ohzkA4L0aLUx7L9YtIihk9eiD_o,5965
@@ -22,7 +22,7 @@ anatools/anaclient/preview.py,sha256=524-If_otuRAuwJtVRcQxL8G0wB-1i_DbgYoi_yePXI
 anatools/anaclient/rules.py,sha256=BA5KHLfyh4ddQYRdGmRwX-hGITL3GolMVzDuBX-hZno,3903
 anatools/anaclient/services.py,sha256=sRRdxM7ZEUfTDKU4700C9KcKsWY8AUTaw210pulwML4,18367
 anatools/anaclient/umap.py,sha256=Dy37yqoNb8HQWPmCh8ZqEIBMijSTqv9tcRcfO0_IXWQ,4144
-anatools/anaclient/volumes.py,sha256=f_4y7Xa7jMRRW0a2QeVReTqyIHTLs2m319_NanXsc0U,21346
+anatools/anaclient/volumes.py,sha256=7NOBC1g_UV5miVLgaQ_LRgoyWiSWBvA-3381f0gVV6A,21442
 anatools/anaclient/workspaces.py,sha256=4aUBfqfW7jzlb6P55LcA0IHmZoNzIWej9tL0DY2pNog,7956
 anatools/anaclient/api/__init__.py,sha256=SrGDhIFobFBhOlE-hOtnJvdrnBAuypG-Im_oyzSPa3E,39
 anatools/anaclient/api/analytics.py,sha256=AFeJj_pl1h70oki5teFDGYRgPIOwHn7yJB-TYNouZsc,3666
@@ -46,7 +46,7 @@ anatools/anaclient/api/preview.py,sha256=OUN4MkYvC0HK6_3ZDxq7jayjPyJpxtgkHhEytmY
 anatools/anaclient/api/rules.py,sha256=yvxbk48m0ZZrWoHpmwS6kC4E2m7iSaY7iaLp0d6xmoc,4780
 anatools/anaclient/api/services.py,sha256=htSjiRz7-kiDivt-xl5wtCSBZPLjDrPjKMKVHcdAM_o,11459
 anatools/anaclient/api/umap.py,sha256=iMRE_z4Umg4-U_uEvdxho93QXOocF3-lN96qHfgbu64,3490
-anatools/anaclient/api/volumes.py,sha256=ZpNGIpO722HFyK2e2OkYO0di-D99-kgCduEYiE36r1Y,9356
+anatools/anaclient/api/volumes.py,sha256=gOc-RPg7I988uKWk38w-DJIPM1Ltgby_wkrpwL_f-fE,9555
 anatools/anaclient/api/workspaces.py,sha256=kIXQAkznBGab38v5aaYcGwP1AnOp4PgID33lPMseRvA,8711
 anatools/anaclient/tests/__init__.py,sha256=uEOcjK2hof0s7RItreGywkCIULWxgcdKbI95JjrEGzk,104
 anatools/anaclient/tests/agents_test.py,sha256=dUrVzb4WmfAE_0Uu5V1zSdW4MUA4I1t1IfNgXTdP7rA,361
@@ -120,7 +120,7 @@ anatools/lib/scene.py,sha256=ANTcP_UVNQQ_6WjgmqHF6u_9HpJsElKpqszSUiN75u4,8339
 anatools/lib/search_utils.py,sha256=DUu7fOIgTWPXItAjPlzfiMazrN9J8ytSjjKW2oWCLbw,3757
 anatools/lib/service.py,sha256=4HfNBKIj3cHqKxD2fJcxlP8MMD1FozHayAsuoRGWURI,2720
 anatools/lib/tools.py,sha256=C1vnbGU6yZJ9q1xsaUP6RDHTohfmm69KZRz7nIjrMLo,5847
-anatools/lib/transfer.py,sha256=YRCZ09w6vYL1b9abN4QOQzEAFgStTXsHyFfJdwWhMqQ,28776
+anatools/lib/transfer.py,sha256=QZ47zD5MsjhBY5jaQMNI84WELUm4QGwFvCjFDy9LOJU,32788
 anatools/nodes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 anatools/nodes/constants.py,sha256=D0nikvblG1XR5umQFqAgbHjUeh9m8eEDiaYOqfInegk,1115
 anatools/nodes/constants.yml,sha256=MZJjhUOFth5Mg8EPU-_wnQkJ9DqPJEzyiLsXyS9UVME,1285
@@ -140,17 +140,17 @@ anatools/nodes/volume_directory.py,sha256=oe721h7qOplRj6N6NpGuyY1HCM277NSYA-6Uo3
 anatools/nodes/volume_directory.yml,sha256=MbyuLUlcqWIlQadYcn4Rvf6roypqh5IiP3dP57TilbY,901
 anatools/nodes/volume_file.py,sha256=YA4zCyRvVzF_9mMefGx29JLE7o9i6-NPaC40BFGv4_c,557
 anatools/nodes/volume_file.yml,sha256=i8bo9QeQmTLeWjv9Rh4EDoDqwOBULNPV7SMLO5AK8DI,862
-anatools-5.1.23.data/scripts/ana,sha256=qe7LDHRgJRPyomzAumdcYy0D5SuaUVag0N8SVevpxcU,5739
-anatools-5.1.23.data/scripts/anadeploy,sha256=zmyxXblJBftJpyaCU2osRjFtNG0Lfc0C89f07I5baI0,37985
-anatools-5.1.23.data/scripts/anamount,sha256=AiaUgaaVVREFY2FLYRmSA8xgSwbfiF9NJYi91SWRA1I,6186
-anatools-5.1.23.data/scripts/anaprofile,sha256=1YUUwHiSa4ORQsxVf2HaSakayNTwMTNeF2snEMSJQAM,9779
-anatools-5.1.23.data/scripts/anarules,sha256=vT2V-77DoL-o6mApSMVXjqeXRIWm44X3jPXdcNK29N0,8972
-anatools-5.1.23.data/scripts/anaserver,sha256=QB1k_vhXAFGMOi9SNIFwgzkzN5LzJeVLtIVkp1oHq4I,8343
-anatools-5.1.23.data/scripts/anatransfer,sha256=GbMLjgA3TP4Oo2mbUxWnkkSC4nKpw1DWta-WVfcNftw,14564
-anatools-5.1.23.data/scripts/anautils,sha256=fziapZuKuBO0VKRgb4C4Js8p9zxxh8OHQmmkNdo3t3E,9530
-anatools-5.1.23.dist-info/licenses/LICENSE,sha256=aw0uaPvFzrHLJxBvuRqUcE2_srfM32-1suya9HbZCY8,1072
-anatools-5.1.23.dist-info/METADATA,sha256=hdvNRbuUCsV4PI-cX4v1wU8lDBGTLghohV8XJDFe4gw,8218
-anatools-5.1.23.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-anatools-5.1.23.dist-info/entry_points.txt,sha256=KsZUmvbH3HXC2CdVpE2GZNR2u_cJNVIbm6BnD658FgM,54
-anatools-5.1.23.dist-info/top_level.txt,sha256=p7xa5bG7NX8pSMJOvRunSz1d7rGPGsBd5-A4gzD4r6w,9
-anatools-5.1.23.dist-info/RECORD,,
+anatools-5.1.25.data/scripts/ana,sha256=qe7LDHRgJRPyomzAumdcYy0D5SuaUVag0N8SVevpxcU,5739
+anatools-5.1.25.data/scripts/anadeploy,sha256=GW0J6YzEvGY2TMbyzkfi-4Ov_gsw4qz2hHJwRx2o7nc,38216
+anatools-5.1.25.data/scripts/anamount,sha256=AiaUgaaVVREFY2FLYRmSA8xgSwbfiF9NJYi91SWRA1I,6186
+anatools-5.1.25.data/scripts/anaprofile,sha256=1YUUwHiSa4ORQsxVf2HaSakayNTwMTNeF2snEMSJQAM,9779
+anatools-5.1.25.data/scripts/anarules,sha256=vT2V-77DoL-o6mApSMVXjqeXRIWm44X3jPXdcNK29N0,8972
+anatools-5.1.25.data/scripts/anaserver,sha256=QB1k_vhXAFGMOi9SNIFwgzkzN5LzJeVLtIVkp1oHq4I,8343
+anatools-5.1.25.data/scripts/anatransfer,sha256=GbMLjgA3TP4Oo2mbUxWnkkSC4nKpw1DWta-WVfcNftw,14564
+anatools-5.1.25.data/scripts/anautils,sha256=fziapZuKuBO0VKRgb4C4Js8p9zxxh8OHQmmkNdo3t3E,9530
+anatools-5.1.25.dist-info/licenses/LICENSE,sha256=aw0uaPvFzrHLJxBvuRqUcE2_srfM32-1suya9HbZCY8,1072
+anatools-5.1.25.dist-info/METADATA,sha256=Pg2TjtE7TGhheLDoVYgyLS3qbZdcc5Iltr969jmhMC0,8218
+anatools-5.1.25.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+anatools-5.1.25.dist-info/entry_points.txt,sha256=KsZUmvbH3HXC2CdVpE2GZNR2u_cJNVIbm6BnD658FgM,54
+anatools-5.1.25.dist-info/top_level.txt,sha256=p7xa5bG7NX8pSMJOvRunSz1d7rGPGsBd5-A4gzD4r6w,9
+anatools-5.1.25.dist-info/RECORD,,