dxpy 0.400.1__tar.gz → 0.402.0__tar.gz

{dxpy-0.400.1 → dxpy-0.402.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dxpy
-Version: 0.400.1
+Version: 0.402.0
 Summary: DNAnexus Platform API bindings for Python
 Home-page: https://github.com/dnanexus/dx-toolkit
 Author: Aleksandra Zalcman, Andrey Kislyuk, Anurag Biyani, Geet Duggal, Katherine Lai, Kurt Jensen, Marek Hrvol, Ohad Rodeh, Phil Sung
@@ -26,6 +26,7 @@ Requires-Dist: urllib3<2.2,>=1.25
 Requires-Dist: pyreadline3==3.4.1; sys_platform == "win32"
 Requires-Dist: colorama<=0.4.6,>=0.4.4; sys_platform == "win32"
 Requires-Dist: crc32c>=2.7.1
+Requires-Dist: awscrt>=0.23.4
 Provides-Extra: pandas
 Requires-Dist: pandas==1.3.5; extra == "pandas"
 Requires-Dist: numpy<2.0.0; extra == "pandas"

{dxpy-0.400.1 → dxpy-0.402.0}/dxpy/bindings/dxfile_functions.py

@@ -31,6 +31,7 @@ import warnings
 from collections import defaultdict
 import multiprocessing
 import crc32c
+from awscrt import checksums
 import zlib
 import base64
 
@@ -227,70 +228,35 @@ def _download_symbolic_link(dxid, md5digest, project, dest_filename, symlink_max
     if md5digest is not None:
         _verify(dest_filename, md5digest)
 
-
-def _verify_per_part_checksum_on_downloaded_file(filename, dxfile_desc, show_progress=False):
-    parts = dxfile_desc["parts"]
-    parts_to_get = sorted(parts, key=int)
-    file_size = dxfile_desc.get("size")
-    per_part_checksum = dxfile_desc.get('perPartCheckSum')
-    _bytes = 0
-
-    if per_part_checksum is None:
+def _verify_checksum(parts, part_id, chunk_data, checksum_type, dxfile_id):
+    if checksum_type is None:
         return
-    
-    offset = 0
-    for part_id in parts_to_get:
-        parts[part_id]["start"] = offset
-        offset += parts[part_id]["size"]
-
-    def read_chunk(filename, start, size, part_id):
-        with open(filename, 'rb') as f:
-            f.seek(start)
-            chunk = f.read(size)
-            return (chunk, part_id)
-
-    def process_file_in_parallel(filename):
-        chunks = []
-        for part_id in parts_to_get:
-            part_info = parts[part_id]
-            start = part_info["start"]
-            size = part_info["size"]
-            chunks.append((start, size, part_id))
-
-        with concurrent.futures.ThreadPoolExecutor() as executor:
-            futures = [executor.submit(read_chunk, filename, start, size, part_id) for start, size, part_id in chunks]
-            for future in concurrent.futures.as_completed(futures):
-                yield future.result()
-
-    for (chunk, part_id) in process_file_in_parallel(filename):
-        _verify_per_part_checksum(parts, part_id, chunk, per_part_checksum, dxfile_desc['id'])
-        if show_progress:
-            _bytes += parts[part_id]["size"]
-            _print_progress(_bytes, file_size, filename, action="Verified")
 
+    part = parts.get(part_id)
+    if part is None:
+        raise DXFileError("Part {} not found in {}".format(part_id, dxfile_id))
 
-def _verify_per_part_checksum(parts, part_id, chunk_data, per_part_checksum, dxfile_id):
-    if per_part_checksum is None:
-        return
-
-    part = parts[part_id]
     expected_checksum = part.get('checksum')
     verifiers = {
         'CRC32': lambda data: zlib.crc32(data).to_bytes(4, 'big'),
         'CRC32C': lambda data: crc32c.crc32c(data).to_bytes(4, 'big'),
         'SHA1': lambda data: hashlib.sha1(data).digest(),
-        'SHA256': lambda data: hashlib.sha256(data).digest()
+        'SHA256': lambda data: hashlib.sha256(data).digest(),
+        'CRC64NVME': lambda data: checksums.crc64nvme(data).to_bytes(8, 'big'),
     }
 
-    if per_part_checksum not in verifiers:
-        raise DXFileError("Unsupported per-part checksum type: {}".format(per_part_checksum))
+    if checksum_type not in verifiers:
+        raise DXFileError("Unsupported checksum type: {}".format(checksum_type))
     if expected_checksum is None:
-        raise DXFileError("{} checksum not found in part {}".format(per_part_checksum, part_id))
+        raise DXFileError("{} checksum not found in part {}".format(checksum_type, part_id))
 
     expected_checksum = base64.b64decode(expected_checksum)
-    got_checksum = verifiers[per_part_checksum](chunk_data)
+    got_checksum = verifiers[checksum_type](chunk_data)
+
     if got_checksum != expected_checksum:
-        raise DXChecksumMismatchError("Checksum mismatch in {} part {} (expected {}, got {})".format(dxfile_id, part_id, expected_checksum, got_checksum))
+        raise DXChecksumMismatchError("{} checksum mismatch in {} in part {} (expected {}, got {})".format(checksum_type, dxfile_id, part_id, expected_checksum, got_checksum))
+
+    return True
 
 
 def _print_progress(bytes_downloaded, file_size, filename, action="Downloaded"):
@@ -352,12 +318,12 @@ def _download_dxfile(dxid, filename, part_retry_counter,
         else:
             md5 = None
         _download_symbolic_link(dxid, md5, project, filename, symlink_max_tries=symlink_max_tries)
-        _verify_per_part_checksum_on_downloaded_file(filename, dxfile_desc, show_progress)
         return True
 
     parts = dxfile_desc["parts"]
     parts_to_get = sorted(parts, key=int)
     file_size = dxfile_desc.get("size")
+    checksum_type = dxfile_desc.get("checksumType")
 
     offset = 0
     for part_id in parts_to_get:
@@ -375,8 +341,11 @@ def _download_dxfile(dxid, filename, part_retry_counter,
     if show_progress:
         _print_progress(0, None, filename)
 
-    def get_chunk(part_id_to_get, start, end):
+    def get_chunk(part_id_to_get, start, end, e_tag = None):
         url, headers = dxfile.get_download_url(project=project, **kwargs)
+        if e_tag is not None:
+            headers["If-Match"] = e_tag
+
         # If we're fetching the whole object in one shot, avoid setting the Range header to take advantage of gzip
         # transfer compression
         sub_range = False
@@ -385,14 +354,18 @@ def _download_dxfile(dxid, filename, part_retry_counter,
         data = dxpy._dxhttp_read_range(url, headers, start, end, FILE_REQUEST_TIMEOUT, sub_range)
         return part_id_to_get, data
 
-    def chunk_requests():
+    def chunk_requests(e_tag = None):
         for part_id_to_chunk in parts_to_get:
             part_info = parts[part_id_to_chunk]
             for chunk_start in range(part_info["start"], part_info["start"] + part_info["size"], chunksize):
                 chunk_end = min(chunk_start + chunksize, part_info["start"] + part_info["size"]) - 1
-                yield get_chunk, [part_id_to_chunk, chunk_start, chunk_end], {}
+                yield get_chunk, [part_id_to_chunk, chunk_start, chunk_end, e_tag], {}
+
+    def verify_part(_part_id, got_bytes, hasher, e_tag = None):
+        # If there's no md5 but we have an eTag, skip the client-side md5 check
+        if got_bytes is not None and "md5" not in parts[_part_id] and e_tag is not None:
+            return
 
-    def verify_part(_part_id, got_bytes, hasher):
         if got_bytes is not None and got_bytes != parts[_part_id]["size"]:
             msg = "Unexpected part data size in {} part {} (expected {}, got {})"
             msg = msg.format(dxfile.get_id(), _part_id, parts[_part_id]["size"], got_bytes)
@@ -400,10 +373,10 @@ def _download_dxfile(dxid, filename, part_retry_counter,
         if hasher is not None and "md5" not in parts[_part_id]:
             warnings.warn("Download of file {} is not being checked for integrity".format(dxfile.get_id()))
         elif hasher is not None and hasher.hexdigest() != parts[_part_id]["md5"]:
-            msg = "Checksum mismatch in {} part {} (expected {}, got {})"
+            msg = "md5sum mismatch in {} part {} (expected {}, got {})"
             msg = msg.format(dxfile.get_id(), _part_id, parts[_part_id]["md5"], hasher.hexdigest())
             raise DXChecksumMismatchError(msg)
-        
+
 
     with fh:
         last_verified_pos = 0
@@ -427,6 +400,8 @@ def _download_dxfile(dxid, filename, part_retry_counter,
                         bytes_to_read -= max_verify_chunk_size
                     if hasher.hexdigest() != part_info["md5"]:
                         raise DXFileError("Checksum mismatch when verifying downloaded part {}".format(part_id))
+                    if dxfile_desc.get('drive') is not None:
+                        _verify_checksum(parts, part_id, chunk, checksum_type, dxfile.get_id())
                     else:
                         last_verified_part = part_id
                         last_verified_pos = fh.tell()
@@ -447,19 +422,25 @@ def _download_dxfile(dxid, filename, part_retry_counter,
             # Main loop. In parallel: download chunks, verify them, and write them to disk.
             get_first_chunk_sequentially = (file_size > 128 * 1024 and last_verified_pos == 0 and dxpy.JOB_ID)
             cur_part, got_bytes, hasher = None, None, None
-            for chunk_part, chunk_data in response_iterator(chunk_requests(),
+            e_tag = None
+            if describe_output and describe_output.get("symlinkTargetIdentifier"):
+                e_tag = describe_output["symlinkTargetIdentifier"].get("ETag")
+
+            for chunk_part, chunk_data in response_iterator(chunk_requests(e_tag),
                                                             dxfile._http_threadpool,
                                                             do_first_task_sequentially=get_first_chunk_sequentially):
                 if chunk_part != cur_part:
-                    verify_part(cur_part, got_bytes, hasher)
+                    verify_part(cur_part, got_bytes, hasher, e_tag)
                     cur_part, got_bytes, hasher = chunk_part, 0, md5_hasher()
+                    if dxfile_desc.get('drive') is not None:
+                        _verify_checksum(parts, cur_part, chunk_data, checksum_type, dxfile.get_id())
                 got_bytes += len(chunk_data)
                 hasher.update(chunk_data)
                 fh.write(chunk_data)
                 if show_progress:
                     _bytes += len(chunk_data)
                     _print_progress(_bytes, file_size, filename)
-            verify_part(cur_part, got_bytes, hasher)
+            verify_part(cur_part, got_bytes, hasher, e_tag)
             if show_progress:
                 _print_progress(_bytes, file_size, filename, action="Completed")
         except DXFileError:

{dxpy-0.400.1 → dxpy-0.402.0}/dxpy/bindings/dxproject.py

@@ -285,7 +285,7 @@ class DXProject(DXContainer):
             restricted=None, download_restricted=None, contains_phi=None,
             tags=None, properties=None, bill_to=None, database_ui_view_only=None,
             external_upload_restricted=None, default_symlink=None,
-            database_results_restricted=None,
+            database_results_restricted=None, drive=None,
             **kwargs):
         """
         :param name: The name of the project
@@ -318,6 +318,8 @@ class DXProject(DXContainer):
         :type database_results_restricted: int
         :param default_symlink: If provided, the details needed to have writable symlinks in the project. Dict must include drive, container, and optional prefix.
         :type default_symlink: dict
+        :param drive: If provided, this drive will associated with the project.
+        :type drive: string
 
         Creates a new project. Initially only the user performing this action
         will be in the permissions/member list, with ADMINISTER access.
@@ -357,6 +359,8 @@ class DXProject(DXContainer):
             input_hash["properties"] = properties
         if default_symlink is not None:
             input_hash["defaultSymlink"] = default_symlink
+        if drive is not None:
+            input_hash["drive"] = drive
 
         self.set_id(dxpy.api.project_new(input_hash, **kwargs)["id"])
         self._desc = {}

{dxpy-0.400.1 → dxpy-0.402.0}/dxpy/cli/download.py

@@ -200,7 +200,7 @@ def download(args):
                                              "size": True,
                                              "drive": True,
                                              "md5": True,
-                                             "perPartCheckSum": True
+                                             "checksumType": True
                                              }})
 
         project, folderpath, matching_files = try_call(resolve_existing_path, path, **resolver_kwargs)

{dxpy-0.400.1 → dxpy-0.402.0}/dxpy/cli/parsers.py

@@ -244,7 +244,7 @@ class PrintInstanceTypeHelp(argparse.Action):
                     'instance type to be used by each entry point.'))
         print(fill('A single instance type can be requested to be used by all entry points by providing the instance type name.  Different instance types can also be requested for different entry points of an app or applet by providing a JSON string mapping from function names to instance types, e.g.'))
         print()
-        print('    {"main": "mem2_hdd2_v2_x2", "other_function": "mem1_ssd1_v2_x2"}')
+        print('    {"main": "mem2_ssd2_v3_x4", "other_function": "mem1_ssd2_v3_x2"}')
         if parser.prog == 'dx run':
             print()
             print(fill('When running a workflow, --instance-type lets you specify instance types for ' +
@@ -261,15 +261,15 @@ class PrintInstanceTypeHelp(argparse.Action):
                   '                                         "*":    "mem1_ssd1_v2_x4"}\'')
             print()
             print(fill('2. Runs all entry points of the first stage with ' +
-                       'mem2_hdd2_v2_x2, the main entry point of the second stage with mem1_ssd1_v2_x4, ' +
+                       'mem2_ssd2_v3_x2, the main entry point of the second stage with mem1_ssd1_v2_x4, ' +
                        'the stage named "BWA" with mem1_ssd1_v2_x2, and all other ' +
-                       'stages with mem2_hdd2_v2_x4'))
+                       'stages with mem2_ssd2_v3_x4'))
             print()
             print('    dx run workflow-xxxx \\\n' +
-                  '     --instance-type 0=mem2_hdd2_v2_x2 \\\n' +
+                  '     --instance-type 0=mem2_ssd2_v3_x2 \\\n' +
                   '     --instance-type 1=\'{"main": "mem1_ssd1_v2_x4"}\' \\\n' +
                   '     --instance-type BWA=mem1_ssd1_v2_x2 \\\n' +
-                  '     --instance-type mem2_hdd2_v2_x4')
+                  '     --instance-type mem2_ssd2_v3_x4')
             print()
             print(fill('--instance-type-by-executable argument is a JSON string with a double mapping that ' +
                        'specifies instance types by app or applet id, then by entry point within the executable.' +
@@ -283,30 +283,30 @@ class PrintInstanceTypeHelp(argparse.Action):
                 '    dx run workflow-xxxx --instance-type-by-executable \'{"*": {"*": "mem2_ssd1_v2_x2"}}\'')
             print()
             print(fill(
-                '4. Force every job in the execution tree executing applet-xyz1 to use mem2_ssd1_v2_x2'))
+                '4. Force every job in the execution tree executing applet-xyz1 to use mem3_ssd2_v3_x2'))
             print()
             print(
-                '    dx run workflow-xxxx --instance-type-by-executable \'{"applet-xyz1":{"*": "mem2_ssd1_v2_x2"}}\'')
+                '    dx run workflow-xxxx --instance-type-by-executable \'{"applet-xyz1":{"*": "mem3_ssd2_v3_x2"}}\'')
             print()
-            print(fill('5. Force every job executing applet-xyz1 to use mem2_ssd1_v2_x4 ' +
-                       'for the main entry point and mem2_ssd1_v2_x2 for all other entry points.' +
-                       'Also force the collect entry point of all executables other than applet-xyz1 to use mem2_ssd1_v2_x8.' +
+            print(fill('5. Force every job executing applet-xyz1 to use mem1_ssd2_v3_x4 ' +
+                       'for the main entry point and mem1_ssd2_v3_x2 for all other entry points.' +
+                       'Also force the collect entry point of all executables other than applet-xyz1 to use mem2_ssd2_v3_x8.' +
                        'Other entry points of executable other than applet-xyz1 may be overridden by ' +
                        'lower-priority mechanisms'))
             print()
             print('    dx run workflow-xxxx --instance-type-by-executable \\\n' +
-                  '           \'{"applet-xyz1":  {"main":    "mem2_ssd1_v2_x4", "*": "mem2_ssd1_v2_x2"},\n' +
-                  '             "*":            {"collect": "mem2_ssd1_v2_x8"}}\'')
+                  '           \'{"applet-xyz1":  {"main":    "mem1_ssd2_v3_x4", "*": "mem1_ssd2_v3_x2"},\n' +
+                  '             "*":            {"collect": "mem2_ssd2_v3_x8"}}\'')
             print()
-            print(fill('6. Force every job executing applet-xxxx to use mem2_ssd1_v2_x2 for all entry points ' +
+            print(fill('6. Force every job executing applet-xxxx to use mem1_ssd2_v3_x2 for all entry points ' +
                        'in the entire execution tree. ' +
-                       'Also force stage 0 executable to run on mem2_ssd1_v2_x4, unless stage 0 invokes ' +
-                       'applet-xxxx, in which case applet-xxxx\'s jobs will use mem2_ssd1_v2_x2 as specified by ' +
+                       'Also force stage 0 executable to run on mem1_ssd2_v3_x4, unless stage 0 invokes ' +
+                       'applet-xxxx, in which case applet-xxxx\'s jobs will use mem1_ssd2_v3_x2 as specified by ' +
                        '--instance-type-by-executable.'))
             print()
             print('    dx run workflow-xxxx \\\n' +
-                  '     --instance-type-by-executable  \'{"applet-xxxx": {"*": "mem2_ssd1_v2_x2"}}\' \\\n' +
-                  '     --instance-type 0=mem2_ssd1_v2_x4')
+                  '     --instance-type-by-executable  \'{"applet-xxxx": {"*": "mem1_ssd2_v3_x2"}}\' \\\n' +
+                  '     --instance-type 0=mem1_ssd2_v3_x4')
             print()
             print(fill(
                 'See "Requesting Instance Types" in DNAnexus documentation for more details.'))
@@ -332,7 +332,7 @@ instance_type_arg.add_argument('--instance-type',
                                metavar='INSTANCE_TYPE_OR_MAPPING',
                                help=fill('''When running an app or applet, the mapping lists executable's entry points or "*" as keys, and instance types to use for these entry points as values.  
 When running a workflow, the specified instance types can be prefixed by a stage name or stage index followed by "=" to apply to a specific stage, or apply to all workflow stages without such prefix. 
-The instance type corresponding to the "*" key is applied to all entry points not explicitly mentioned in the --instance-type mapping. Specifying a single instance type is equivalent to using it for all entry points, so "--instance-type mem1_ssd1_v2_x2" is same as "--instance-type '{"*":"mem1_ssd1_v2_x2"}'. 
+The instance type corresponding to the "*" key is applied to all entry points not explicitly mentioned in the --instance-type mapping. Specifying a single instance type is equivalent to using it for all entry points, so "--instance-type mem1_ssd2_v3_x2" is same as "--instance-type '{"*":"mem1_ssd2_v3_x2"}'. 
 Note that "dx run" calls within the execution subtree may override the values specified at the root of the execution tree.
 See dx run --instance-type-help for details.
 ''', width_adjustment=-24, replace_whitespace=False),

dxpy-0.402.0/dxpy/nextflow/nextaur_assets.json

@@ -0,0 +1,10 @@
+{
+    "aws:ap-southeast-2": "record-J4YYvjj5FVfkvq9259y58qGz",
+    "aws:eu-central-1": "record-J4YYqf048jzY7Pk1KV22YB42",
+    "aws:eu-west-2-g": "record-J4YYvgXKXGfY3b41gVzXPzQV",
+    "aws:me-south-1": "record-J4YYvbV38v58JVJzvG20FXQQ",
+    "aws:us-east-1": "record-J4YYp2j0v52gXXp42ZgJ4F6k",
+    "azure:westeurope": "record-J4YYy5QBJV21F607bX2qX58F",
+    "azure:westus": "record-J4YYyV892393KKj9vbj76KpY",
+    "oci:us-ashburn-1": "record-J4YYx6162FZB2kfvz6g263vP"
+}

dxpy-0.402.0/dxpy/nextflow/nextaur_assets.staging.json

@@ -0,0 +1,10 @@
+{
+    "aws:ap-southeast-2": "record-J4YX1005k7GGX6g9xgg8yb91",
+    "aws:eu-central-1": "record-J4YX0Vj4X3z8fvFB1Jq16bv0",
+    "aws:eu-west-2-g": "record-J4YX0PXKfYb57JpGKZZ38f3p",
+    "aws:me-south-1": "record-J4YX0K139gXkGPkKf3yGz915",
+    "aws:us-east-1": "record-J4YVy580zkbzBBvyqgb90bQF",
+    "azure:westeurope": "record-J4YX2z8BVVBpgkJqFfz9p7gb",
+    "azure:westus": "record-J4YX2X09fJFY2VQb1Z1Xxjpq",
+    "oci:us-ashburn-1": "record-J4fk6yV6X487YP2gQPV93YZb"
+}

{dxpy-0.400.1 → dxpy-0.402.0}/dxpy/scripts/dx.py

@@ -6451,8 +6451,8 @@ EXAMPLES:
     parents=[all_arg],
     prog='dx unarchive')
 
-parser_unarchive.add_argument('--rate', help=fill('The speed at which all files in this request are unarchived.', width_adjustment=-24) + '\n'+ fill('- Azure regions: {Expedited, Standard}', width_adjustment=-24,initial_indent='  ') + '\n'+
-fill('- AWS regions: {Expedited, Standard, Bulk}', width_adjustment=-24,initial_indent='  '), choices=["Expedited", "Standard", "Bulk"], default="Standard")
+parser_unarchive.add_argument('--rate', help=fill('The speed at which all files in this request are unarchived.', width_adjustment=-24) + '\n'+ fill('- Azure regions: {Standard, Expedited}', width_adjustment=-24,initial_indent='  ') + '\n'+
+fill('- AWS regions: {Standard, Bulk}', width_adjustment=-24,initial_indent='  '), choices=["Expedited", "Standard", "Bulk"], default="Standard")
 
 parser_unarchive.add_argument('-q', '--quiet', help='Do not print extra info messages', action='store_true')
 parser_unarchive.add_argument(

dxpy-0.402.0/dxpy/toolkit_version.py

@@ -0,0 +1 @@
+version = '0.402.0'

{dxpy-0.400.1 → dxpy-0.402.0}/dxpy/utils/completer.py

@@ -423,6 +423,16 @@ class InstanceTypesCompleter():
               InstanceTypeSpec('mem1_ssd1_v2_x36', 36, 72.0, 900),
               InstanceTypeSpec('mem1_ssd1_v2_x72', 72, 144.0, 1800),
 
+              InstanceTypeSpec('mem1_ssd2_v3_x2', 2, 4.0, 118),
+              InstanceTypeSpec('mem1_ssd2_v3_x4', 4, 8.0, 237),
+              InstanceTypeSpec('mem1_ssd2_v3_x8', 8, 16.0, 474),
+              InstanceTypeSpec('mem1_ssd2_v3_x16', 16, 32.0, 950),
+              InstanceTypeSpec('mem1_ssd2_v3_x32', 32, 64.0, 1900),
+              InstanceTypeSpec('mem1_ssd2_v3_x48', 48, 96.0, 2850),
+              InstanceTypeSpec('mem1_ssd2_v3_x64', 64, 128.0, 3800),
+              InstanceTypeSpec('mem1_ssd2_v3_x96', 96, 192.0, 5700),
+              InstanceTypeSpec('mem1_ssd2_v3_x128', 128, 256.0, 7600),
+
               InstanceTypeSpec('mem1_ssd2_v2_x2', 2, 4.0, 160),
               InstanceTypeSpec('mem1_ssd2_v2_x4', 4, 8.0, 320),
               InstanceTypeSpec('mem1_ssd2_v2_x8', 8, 16.0, 640),
@@ -439,6 +449,16 @@ class InstanceTypesCompleter():
               InstanceTypeSpec('mem2_ssd1_v2_x64', 64, 256.0, 2400),
               InstanceTypeSpec('mem2_ssd1_v2_x96', 96, 384.0, 3600),
 
+              InstanceTypeSpec('mem2_ssd2_v3_x2', 2, 8.0, 118),
+              InstanceTypeSpec('mem2_ssd2_v3_x4', 4, 16.0, 237),
+              InstanceTypeSpec('mem2_ssd2_v3_x8', 8, 32.0, 474),
+              InstanceTypeSpec('mem2_ssd2_v3_x16', 16, 64.0, 950),
+              InstanceTypeSpec('mem2_ssd2_v3_x32', 32, 128.0, 1900),
+              InstanceTypeSpec('mem2_ssd2_v3_x48', 48, 192.0, 2850),
+              InstanceTypeSpec('mem2_ssd2_v3_x64', 64, 256.0, 3800),
+              InstanceTypeSpec('mem2_ssd2_v3_x96', 96, 384.0, 5700),
+              InstanceTypeSpec('mem2_ssd2_v3_x128', 128, 512.0, 7600),
+
               InstanceTypeSpec('mem2_ssd2_v2_x2', 2, 8.0, 160),
               InstanceTypeSpec('mem2_ssd2_v2_x4', 4, 16.0, 320),
               InstanceTypeSpec('mem2_ssd2_v2_x8', 8, 32.0, 640),
@@ -457,6 +477,16 @@ class InstanceTypesCompleter():
               InstanceTypeSpec('mem3_ssd1_v2_x64', 64, 512.0, 3200),
               InstanceTypeSpec('mem3_ssd1_v2_x96', 96, 768.0, 3600),
 
+              InstanceTypeSpec('mem3_ssd2_v3_x2', 2, 16.0, 118),
+              InstanceTypeSpec('mem3_ssd2_v3_x4', 4, 32.0, 237),
+              InstanceTypeSpec('mem3_ssd2_v3_x8', 8, 64.0, 474),
+              InstanceTypeSpec('mem3_ssd2_v3_x16', 16, 128.0, 950),
+              InstanceTypeSpec('mem3_ssd2_v3_x32', 32, 256.0, 1900),
+              InstanceTypeSpec('mem3_ssd2_v3_x48', 48, 384.0, 2850),
+              InstanceTypeSpec('mem3_ssd2_v3_x64', 64, 512.0, 3800),
+              InstanceTypeSpec('mem3_ssd2_v3_x96', 96, 768.0, 5700),
+              InstanceTypeSpec('mem3_ssd2_v3_x128', 128, 1024.0, 7600),
+
               InstanceTypeSpec('mem3_ssd2_v2_x2', 2, 15.25, 475),
               InstanceTypeSpec('mem3_ssd2_v2_x4', 4, 30.5, 950),
               InstanceTypeSpec('mem3_ssd2_v2_x8', 8, 61.0, 1900),
@@ -541,8 +571,8 @@ class InstanceTypesCompleter():
 
     fpga_instance_types = OrderedDict()
     for i in (FpgaInstanceTypeSpec('mem3_ssd2_fpga1_x24', 24, 256.0, 940, 1),
-              FpgaInstanceTypeSpec('mem3_ssd2_fpga1_x48', 48, 512.0, 1880, 2),
-              FpgaInstanceTypeSpec('mem3_ssd2_fpga1_x192', 192, 2048.0, 7520, 8)):
+              FpgaInstanceTypeSpec('mem3_ssd2_fpga2_x48', 48, 512.0, 1880, 2),
+              FpgaInstanceTypeSpec('mem3_ssd2_fpga8_x192', 192, 2048.0, 7520, 8)):
         fpga_instance_types[i.Name] = i
 
     aws_other_instance_types = OrderedDict()
@@ -551,6 +581,7 @@ class InstanceTypesCompleter():
               InstanceTypeSpec('mem1_ssd1_x8', 8, 15.0, 160),
               InstanceTypeSpec('mem1_ssd1_x16', 16, 30.0, 320),
               InstanceTypeSpec('mem1_ssd1_x32', 32, 60.0, 640),
+              InstanceTypeSpec('mem1_ssd1_x36', 36, 72.0, 900),
 
               InstanceTypeSpec('mem1_ssd2_x2', 2, 3.8, 160),
               InstanceTypeSpec('mem1_ssd2_x4', 4, 7.5, 320),
@@ -605,6 +636,10 @@ class InstanceTypesCompleter():
               InstanceTypeSpec('mem2_hdd2_v2_x2', 2, 8.0, 1000),
               InstanceTypeSpec('mem2_hdd2_v2_x4', 4, 16.0, 2000),
 
+              InstanceTypeSpec('mem3_hdd2_x2', 2, 17.1, 420),
+              InstanceTypeSpec('mem3_hdd2_x4', 4, 34.2, 850),
+              InstanceTypeSpec('mem3_hdd2_x8', 8, 68.4, 1680),
+
               InstanceTypeSpec('mem3_hdd2_v2_x2', 2, 16.0, 500),
               InstanceTypeSpec('mem3_hdd2_v2_x4', 4, 32.0, 1000),
               InstanceTypeSpec('mem3_hdd2_v2_x8', 8, 64.0, 2000)):

{dxpy-0.400.1 → dxpy-0.402.0}/dxpy.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dxpy
-Version: 0.400.1
+Version: 0.402.0
 Summary: DNAnexus Platform API bindings for Python
 Home-page: https://github.com/dnanexus/dx-toolkit
 Author: Aleksandra Zalcman, Andrey Kislyuk, Anurag Biyani, Geet Duggal, Katherine Lai, Kurt Jensen, Marek Hrvol, Ohad Rodeh, Phil Sung
@@ -26,6 +26,7 @@ Requires-Dist: urllib3<2.2,>=1.25
 Requires-Dist: pyreadline3==3.4.1; sys_platform == "win32"
 Requires-Dist: colorama<=0.4.6,>=0.4.4; sys_platform == "win32"
 Requires-Dist: crc32c>=2.7.1
+Requires-Dist: awscrt>=0.23.4
 Provides-Extra: pandas
 Requires-Dist: pandas==1.3.5; extra == "pandas"
 Requires-Dist: numpy<2.0.0; extra == "pandas"

{dxpy-0.400.1 → dxpy-0.402.0}/dxpy.egg-info/SOURCES.txt

@@ -192,6 +192,7 @@ test/test_dx_symlink.py
 test/test_dxabs.py
 test/test_dxasset.py
 test/test_dxclient.py
+test/test_dxfile_functions.py
 test/test_dxpy.py
 test/test_dxpy_utils.py
 test/test_dxunpack.py

{dxpy-0.400.1 → dxpy-0.402.0}/dxpy.egg-info/requires.txt

@@ -4,6 +4,7 @@ psutil>=5.9.3
 certifi>=2024.7.4
 urllib3<2.2,>=1.25
 crc32c>=2.7.1
+awscrt>=0.23.4
 
 [:python_version < "3.10"]
 argcomplete<2.0.0,>=1.9.4

{dxpy-0.400.1 → dxpy-0.402.0}/requirements.txt

@@ -8,3 +8,4 @@ urllib3>=1.25,<2.2
 pyreadline3==3.4.1; sys_platform == "win32"
 colorama>=0.4.4,<=0.4.6; sys_platform == "win32"
 crc32c>=2.7.1
+awscrt>=0.23.4

dxpy-0.402.0/test/test_dxfile_functions.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013-2019 DNAnexus, Inc.
+#
+# This file is part of dx-toolkit (DNAnexus platform client libraries).
+#
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may not
+#   use this file except in compliance with the License. You may obtain a copy
+#   of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+
+from __future__ import print_function, unicode_literals, division, absolute_import
+
+import unittest
+from dxpy.bindings.dxfile_functions import _verify_checksum
+from dxpy.exceptions import DXFileError, DXChecksumMismatchError
+
+class TestVerifyPerPartChecksum(unittest.TestCase):
+    def setUp(self):
+      self.valid_crc32_parts = {'1': {'checksum': '0kY1xw=='}}
+      self.valid_crc32c_parts = {'1': {'checksum': 'Hu2dEw=='}}
+      self.valid_sha1_parts = {'1': {'checksum': 'yl8CzxRPiurWmuYjQ3ySrSeaCAE='}}
+      self.valid_sha256_parts = {'1': {'checksum': 'TuNtcBmfMMIRMpTEjVWVnAVSG56/K+nL2nF2rYi67y0='}}
+      self.valid_crc64nvme_parts = {'1': {'checksum': '688cIX1wosY='}}
+      self.dx_file_id = 'file-xxxx'
+      self.chunk_data = 'fizzbuzz'.encode('utf-8')
+
+    def test_per_part_checksum_is_none(self):
+        assert _verify_checksum(self.valid_crc32_parts, '1', self.chunk_data, None, self.dx_file_id) == None
+
+    def test_part_id_is_invalid(self):
+        with self.assertRaisesRegex(DXFileError, 'Part 5 not found in file-xxxx'):
+          _verify_checksum(self.valid_crc32_parts, '5', self.chunk_data, 'CRC32', self.dx_file_id)
+
+    def test_invalid_checksum(self):
+        with self.assertRaisesRegex(DXFileError, 'Unsupported checksum type: ABC'):
+          _verify_checksum(self.valid_crc32_parts, '1', self.chunk_data, 'ABC', self.dx_file_id)
+
+    def test_checksum_not_found(self):
+       with self.assertRaisesRegex(DXFileError, 'checksum not found in part 1'):
+          _verify_checksum({'1': {}}, '1', self.chunk_data, 'CRC32', self.dx_file_id)
+
+    def test_valid_crc32_checksum(self):
+        assert _verify_checksum(self.valid_crc32_parts, '1', self.chunk_data, 'CRC32', self.dx_file_id) == True
+
+    def test_valid_crc32c_checksum(self):
+        assert _verify_checksum(self.valid_crc32c_parts, '1', self.chunk_data, 'CRC32C', self.dx_file_id) == True
+
+    def test_valid_sha1_checksum(self):
+        assert _verify_checksum(self.valid_sha1_parts, '1', self.chunk_data, 'SHA1', self.dx_file_id) == True
+
+    def test_valid_sha256_checksum(self):
+        assert _verify_checksum(self.valid_sha256_parts, '1', self.chunk_data, 'SHA256', self.dx_file_id) == True
+
+    def test_valid_crc64nvme_checksum(self):
+        assert _verify_checksum(self.valid_crc64nvme_parts, '1', self.chunk_data, 'CRC64NVME', self.dx_file_id) == True
+
+    def test_checksum_mismatch(self):
+       with self.assertRaisesRegex(DXChecksumMismatchError, '^CRC32 checksum mismatch in file-xxxx in part 1 '):
+          _verify_checksum(self.valid_crc32_parts, '1', 'foobar'.encode('utf-8'), 'CRC32', self.dx_file_id)
+
+if __name__ == '__main__':
+    unittest.main()

dxpy-0.400.1/dxpy/nextflow/nextaur_assets.json

@@ -1,10 +0,0 @@
-{
-    "aws:ap-southeast-2": "record-GzyP89850x72260z2yYyZQfJ",
-    "aws:eu-central-1": "record-GzyP7584vB9Yp0j7qfv5F134",
-    "aws:eu-west-2-g": "record-GzyP80XKPQ6K81G8XV89fyvZ",
-    "aws:me-south-1": "record-GzyP85V306B9xkQ4Pv1yf7Kz",
-    "aws:us-east-1": "record-GzyP6580KGjqV4gp6QXFGBzX",
-    "azure:westeurope": "record-GzyP9z0BzJx8zjf6bpvY6Yqv",
-    "azure:westus": "record-GzyP9vQ9zggPyfkj5GKKYVG5",
-    "oci:us-ashburn-1": "record-J30Z6396B086xYKfqQj5gG1j"
-}

dxpy-0.400.1/dxpy/nextflow/nextaur_assets.staging.json

@@ -1,10 +0,0 @@
-{
-    "aws:ap-southeast-2": "record-GzyJ2F0506kK1651KKg3K1Xq",
-    "aws:eu-central-1": "record-GzyJ2Yj4f2xpzzypx8G484FB",
-    "aws:eu-west-2-g": "record-GzyJ28XKKGJ7758Yj62X97bV",
-    "aws:me-south-1": "record-GzyJ2GV311jBJYzx2XpgZXQ0",
-    "aws:us-east-1": "record-GzyJ08002j9B36YfkbQQxpkf",
-    "azure:westeurope": "record-GzyJ4B8B095jZ8F4BF5p2Ggz",
-    "azure:westus": "record-GzyJ3q89X1QBpvfvgx9bZzgP",
-    "oci:us-ashburn-1": "record-J30XbBV6j7q0jq6q1Jfz5650"
-}

dxpy-0.400.1/dxpy/toolkit_version.py

@@ -1 +0,0 @@
-version = '0.400.1'