dxpy 0.384.0__tar.gz → 0.386.0__tar.gz

{dxpy-0.384.0 → dxpy-0.386.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dxpy
-Version: 0.384.0
+Version: 0.386.0
 Summary: DNAnexus Platform API bindings for Python
 Home-page: https://github.com/dnanexus/dx-toolkit
 Author: Aleksandra Zalcman, Andrey Kislyuk, Anurag Biyani, Geet Duggal, Katherine Lai, Kurt Jensen, Marek Hrvol, Ohad Rodeh, Phil Sung
@@ -48,6 +48,14 @@ Example:
 $ _DX_DEBUG=1 dx ls
 ```
 
+### Debugging inside the IDE (PyCharm)
+To be able to debug dx-toolkit (dx commands) directly in the IDE, 'Run/Debug Configurations' needs to be changed.
+1. Go to Run → Edit Configurations...
+2. Add New Configuration (Python)
+3. Change script to module (dxpy.scripts.dx)
+4. To Script parameters field write dx command you want to run (eg 'ls' runs 'dx ls')
+5. Apply and OK (now it is possible to start debugging via main() function in dx.py)
+
 Python coding style
 -------------------
 

{dxpy-0.384.0 → dxpy-0.386.0}/Readme.md

@@ -27,6 +27,14 @@ Example:
 $ _DX_DEBUG=1 dx ls
 ```
 
+### Debugging inside the IDE (PyCharm)
+To be able to debug dx-toolkit (dx commands) directly in the IDE, 'Run/Debug Configurations' needs to be changed.
+1. Go to Run → Edit Configurations...
+2. Add New Configuration (Python)
+3. Change script to module (dxpy.scripts.dx)
+4. To Script parameters field write dx command you want to run (eg 'ls' runs 'dx ls')
+5. Apply and OK (now it is possible to start debugging via main() function in dx.py)
+
 Python coding style
 -------------------
 

{dxpy-0.384.0 → dxpy-0.386.0}/dxpy/bindings/dxapplet.py

@@ -58,11 +58,12 @@ class DXExecutable:
             if kwargs.get(arg) is not None:
                 run_input[arg] = kwargs[arg]
 
-        if kwargs.get('instance_type') is not None or kwargs.get('cluster_spec') is not None or kwargs.get('fpga_driver') is not None:
+        if any(kwargs.get(key) is not None for key in ['instance_type', 'cluster_spec', 'fpga_driver', 'nvidia_driver']):
             instance_type_srd = SystemRequirementsDict.from_instance_type(kwargs.get('instance_type'))
             cluster_spec_srd = SystemRequirementsDict(kwargs.get('cluster_spec'))
             fpga_driver_srd = SystemRequirementsDict(kwargs.get('fpga_driver'))
-            run_input["systemRequirements"] = (instance_type_srd + cluster_spec_srd + fpga_driver_srd).as_dict()
+            nvidia_driver_srd = SystemRequirementsDict(kwargs.get('nvidia_driver'))
+            run_input["systemRequirements"] = (instance_type_srd + cluster_spec_srd + fpga_driver_srd + nvidia_driver_srd).as_dict()
 
         if kwargs.get('system_requirements') is not None:
             run_input["systemRequirements"] = kwargs.get('system_requirements')
@@ -195,7 +196,7 @@ class DXExecutable:
             depends_on=None, allow_ssh=None, debug=None, delay_workspace_destruction=None, priority=None, head_job_on_demand=None,
             ignore_reuse=None, ignore_reuse_stages=None, detach=None, cost_limit=None, rank=None, max_tree_spot_wait_time=None,
             max_job_spot_wait_time=None, preserve_job_outputs=None, detailed_job_metrics=None, extra_args=None,
-            fpga_driver=None, system_requirements=None, system_requirements_by_executable=None, **kwargs):
+            fpga_driver=None, system_requirements=None, system_requirements_by_executable=None, nvidia_driver=None, **kwargs):
         '''
         :param executable_input: Hash of the executable's input arguments
         :type executable_input: dict
@@ -252,6 +253,8 @@ class DXExecutable:
         :type system_requirements: dict
         :param system_requirements_by_executable: System requirement by executable double mapping
         :type system_requirements_by_executable: dict
+        :param nvidia_driver: a dict mapping function names to nvidia driver requests
+        :type nvidia_driver: dict
         :rtype: :class:`~dxpy.bindings.dxjob.DXJob`
 
         Creates a new job that executes the function "main" of this executable with
@@ -292,7 +295,8 @@ class DXExecutable:
                                         extra_args=extra_args,
                                         fpga_driver=fpga_driver,
                                         system_requirements=system_requirements,
-                                        system_requirements_by_executable=system_requirements_by_executable)
+                                        system_requirements_by_executable=system_requirements_by_executable,
+                                        nvidia_driver=nvidia_driver)
         return self._run_impl(run_input, **kwargs)
 
 

{dxpy-0.384.0 → dxpy-0.386.0}/dxpy/bindings/dxfile.py

@@ -728,7 +728,7 @@ class DXFile(DXDataObject):
             report_progress_fn(self, len(data))
 
     def wait_until_parts_uploaded(self, **kwargs):
-        self._wait_until_parts_uploaded(self, **kwargs)
+        self._wait_until_parts_uploaded(**kwargs)
 
 
     def get_download_url(self, duration=None, preauthenticated=False, filename=None, project=None, **kwargs):

{dxpy-0.384.0 → dxpy-0.386.0}/dxpy/bindings/dxjob.py

@@ -38,14 +38,15 @@ from ..system_requirements import SystemRequirementsDict
 from ..utils.local_exec_utils import queue_entry_point
 from ..compat import basestring
 
+
 #########
 # DXJob #
 #########
 
-def new_dxjob(fn_input, fn_name, name=None, tags=None, properties=None, details=None,
-              instance_type=None, depends_on=None,
-              cluster_spec=None, fpga_driver=None, system_requirements=None, system_requirements_by_executable=None,
-              **kwargs):
+
+def new_dxjob(fn_input, fn_name, name=None, tags=None, properties=None, details=None, instance_type=None,
+              depends_on=None, cluster_spec=None, fpga_driver=None, system_requirements=None,
+              system_requirements_by_executable=None, nvidia_driver=None, **kwargs):
     '''
     :param fn_input: Function input
     :type fn_input: dict
@@ -71,6 +72,8 @@ def new_dxjob(fn_input, fn_name, name=None, tags=None, properties=None, details=
     :type system_requirements: dict
     :param system_requirements_by_executable: System requirement by executable double mapping
     :type system_requirements_by_executable: dict
+    :param nvidia_driver: a dict mapping function names to nvidia driver requests
+    :type nvidia_driver: dict
     :rtype: :class:`~dxpy.bindings.dxjob.DXJob`
 
     Creates and enqueues a new job that will execute a particular
@@ -94,12 +97,13 @@ def new_dxjob(fn_input, fn_name, name=None, tags=None, properties=None, details=
 
     '''
     dxjob = DXJob()
-    dxjob.new(fn_input, fn_name, name=name, tags=tags, properties=properties,
-              details=details, instance_type=instance_type, depends_on=depends_on,
-              cluster_spec=cluster_spec, fpga_driver=fpga_driver, 
-              system_requirements=system_requirements, system_requirements_by_executable=system_requirements_by_executable, **kwargs)
+    dxjob.new(fn_input, fn_name, name=name, tags=tags, properties=properties, details=details,
+              instance_type=instance_type, depends_on=depends_on, cluster_spec=cluster_spec, fpga_driver=fpga_driver,
+              system_requirements=system_requirements, system_requirements_by_executable=system_requirements_by_executable,
+              nvidia_driver=nvidia_driver, **kwargs)
     return dxjob
 
+
 class DXJob(DXObject):
     '''
     Remote job object handler.
@@ -112,10 +116,9 @@ class DXJob(DXObject):
         DXObject.__init__(self, dxid=dxid)
         self.set_id(dxid)
 
-    def new(self, fn_input, fn_name, name=None, tags=None, properties=None, details=None,
-            instance_type=None, depends_on=None,
-            cluster_spec=None, fpga_driver=None, system_requirements=None, system_requirements_by_executable=None, 
-            **kwargs):
+    def new(self, fn_input, fn_name, name=None, tags=None, properties=None, details=None, instance_type=None,
+            depends_on=None, cluster_spec=None, fpga_driver=None, system_requirements=None,
+            system_requirements_by_executable=None, nvidia_driver=None, **kwargs):
         '''
         :param fn_input: Function input
         :type fn_input: dict
@@ -141,6 +144,8 @@ class DXJob(DXObject):
         :type system_requirements: dict
         :param system_requirements_by_executable: System requirement by executable double mapping
         :type system_requirements_by_executable: dict
+        :param nvidia_driver: a dict mapping function names to nvidia driver requests
+        :type nvidia_driver: dict
 
         Creates and enqueues a new job that will execute a particular
         function (from the same app or applet as the one the current job
@@ -179,11 +184,12 @@ class DXJob(DXObject):
                 req_input["tags"] = tags
             if properties is not None:
                 req_input["properties"] = properties
-            if instance_type is not None or cluster_spec is not None or fpga_driver is not None:
+            if any(requirement is not None for requirement in [instance_type, cluster_spec, fpga_driver, nvidia_driver]):
                 instance_type_srd = SystemRequirementsDict.from_instance_type(instance_type, fn_name)
                 cluster_spec_srd = SystemRequirementsDict(cluster_spec)
                 fpga_driver_srd = SystemRequirementsDict(fpga_driver)
-                req_input["systemRequirements"] = (instance_type_srd + cluster_spec_srd + fpga_driver_srd).as_dict()
+                nvidia_driver_srd = SystemRequirementsDict(nvidia_driver)
+                req_input["systemRequirements"] = (instance_type_srd + cluster_spec_srd + fpga_driver_srd + nvidia_driver_srd).as_dict()
             if system_requirements is not None:
                 req_input["systemRequirements"] = system_requirements
             if system_requirements_by_executable is not None:

{dxpy-0.384.0 → dxpy-0.386.0}/dxpy/bindings/dxproject.py

@@ -285,6 +285,7 @@ class DXProject(DXContainer):
             restricted=None, download_restricted=None, contains_phi=None,
             tags=None, properties=None, bill_to=None, database_ui_view_only=None,
             external_upload_restricted=None, default_symlink=None,
+            database_results_restricted=None,
             **kwargs):
         """
         :param name: The name of the project
@@ -313,6 +314,8 @@ class DXProject(DXContainer):
         :type database_ui_view_only: boolean
         :param external_upload_restricted: If provided, whether project members can upload data to project from external sources, e.g. outside of job
         :type external_upload_restricted: boolean
+        :param database_results_restricted: If provided, minimum amount of data that project members with VIEW access can see from databases in the project
+        :type database_results_restricted: int
         :param default_symlink: If provided, the details needed to have writable symlinks in the project. Dict must include drive, container, and optional prefix.
         :type default_symlink: dict
 
@@ -346,6 +349,8 @@ class DXProject(DXContainer):
             input_hash["databaseUIViewOnly"] = database_ui_view_only
         if external_upload_restricted is not None:
             input_hash["externalUploadRestricted"] = external_upload_restricted
+        if database_results_restricted is not None:
+            input_hash["databaseResultsRestricted"] = database_results_restricted
         if tags is not None:
             input_hash["tags"] = tags
         if properties is not None:
@@ -360,7 +365,9 @@ class DXProject(DXContainer):
     def update(self, name=None, summary=None, description=None, protected=None,
                restricted=None, download_restricted=None, version=None,
                allowed_executables=None, unset_allowed_executables=None,
-               database_ui_view_only=None, external_upload_restricted=None, **kwargs):
+               database_ui_view_only=None, external_upload_restricted=None,
+               database_results_restricted=None, unset_database_results_restricted=None,
+               **kwargs):
         """
         :param name: If provided, the new project name
         :type name: string
@@ -376,10 +383,16 @@ class DXProject(DXContainer):
         :type download_restricted: boolean
         :param allowed_executables: If provided, these are the only executable ID(s) allowed to run as root executions in this project
         :type allowed_executables: list
+        :param unset_allowed_executables: If provided, removes any restrictions set by allowed_executables
+        :type unset_allowed_executables: boolean
         :param database_ui_view_only: If provided, whether the viewers on the project can access the database data directly
         :type database_ui_view_only: boolean
         :param external_upload_restricted: If provided, whether project members can upload data to project from external sources, e.g. outside of job
         :type external_upload_restricted: boolean
+        :param database_results_restricted: If provided, minimum amount of data that project members with VIEW access can see from databases in the project
+        :type database_results_restricted: int
+        :param unset_database_results_restricted: If provided, removes any restrictions set by database_results_restricted
+        :type unset_database_results_restricted: boolean
         :param version: If provided, the update will only occur if the value matches the current project's version number
         :type version: int
 
@@ -413,6 +426,10 @@ class DXProject(DXContainer):
             update_hash["databaseUIViewOnly"] = database_ui_view_only
         if external_upload_restricted is not None:
             update_hash["externalUploadRestricted"] = external_upload_restricted
+        if database_results_restricted is not None:
+            update_hash["databaseResultsRestricted"] = database_results_restricted
+        if unset_database_results_restricted is not None:
+            update_hash["databaseResultsRestricted"] = None
         dxpy.api.project_update(self._dxid, update_hash, **kwargs)
 
     def invite(self, invitee, level, send_email=True, **kwargs):

{dxpy-0.384.0 → dxpy-0.386.0}/dxpy/cli/exec_io.py

@@ -22,7 +22,7 @@ from __future__ import print_function, unicode_literals, division, absolute_impo
 
 # TODO: refactor all dx run helper functions here
 
-import os, sys, json, collections, pipes
+import os, sys, json, collections, shlex
 from ..bindings.dxworkflow import DXWorkflow
 
 import dxpy
@@ -327,7 +327,7 @@ def format_choices_or_suggestions(header, items, obj_class, initial_indent=' ' *
         # TODO: in interactive prompts the quotes here may be a bit
         # misleading. Perhaps it should be a separate mode to print
         # "interactive-ready" suggestions.
-        return fill(header + ' ' + ', '.join([pipes.quote(str(item)) for item in items]),
+        return fill(header + ' ' + ', '.join([shlex.quote(str(item)) for item in items]),
                     initial_indent=initial_indent,
                     subsequent_indent=subsequent_indent)
 

{dxpy-0.384.0 → dxpy-0.386.0}/dxpy/cli/parsers.py

@@ -446,6 +446,10 @@ def get_update_project_args(args):
         input_params['allowedExecutables'] = args.allowed_executables
     if args.unset_allowed_executables:
         input_params['allowedExecutables'] = None
+    if args.database_results_restricted is not None:
+        input_params['databaseResultsRestricted'] = args.database_results_restricted
+    if args.unset_database_results_restricted:
+        input_params['databaseResultsRestricted'] = None
     if args.external_upload_restricted is not None:
         input_params['externalUploadRestricted'] = args.external_upload_restricted == 'true'
     return input_params

{dxpy-0.384.0 → dxpy-0.386.0}/dxpy/scripts/dx.py

@@ -1432,6 +1432,8 @@ def new_project(args):
         inputs["containsPHI"] = True
     if args.database_ui_view_only:
         inputs["databaseUIViewOnly"] = True
+    if args.database_results_restricted is not None:
+        inputs["databaseResultsRestricted"] = args.database_results_restricted
     if args.monthly_compute_limit is not None:
         inputs["monthlyComputeLimit"] = args.monthly_compute_limit
     if args.monthly_egress_bytes_limit is not None:
@@ -1440,6 +1442,8 @@ def new_project(args):
         inputs["monthlyStorageLimit"] = args.monthly_storage_limit
     if args.default_symlink is not None:
         inputs["defaultSymlink"] = json.loads(args.default_symlink)
+    if args.drive is not None:
+        inputs["drive"] = args.drive
     try:
         resp = dxpy.api.project_new(inputs)
         if args.brief:
@@ -3195,10 +3199,12 @@ def run_body(args, executable, dest_proj, dest_path, preset_inputs=None, input_n
         cloned_instance_type = SystemRequirementsDict.from_sys_requirements(cloned_system_requirements, _type='instanceType')
         cloned_cluster_spec = SystemRequirementsDict.from_sys_requirements(cloned_system_requirements, _type='clusterSpec')
         cloned_fpga_driver = SystemRequirementsDict.from_sys_requirements(cloned_system_requirements, _type='fpgaDriver')
+        cloned_nvidia_driver = SystemRequirementsDict.from_sys_requirements(cloned_system_requirements, _type='nvidiaDriver')
         cloned_system_requirements_by_executable = args.cloned_job_desc.get("mergedSystemRequirementsByExecutable", {}) or {}
     else:
         cloned_system_requirements = {}
-        cloned_instance_type, cloned_cluster_spec, cloned_fpga_driver = SystemRequirementsDict({}), SystemRequirementsDict({}), SystemRequirementsDict({})
+        cloned_instance_type, cloned_cluster_spec, cloned_fpga_driver, cloned_nvidia_driver = (
+            SystemRequirementsDict({}), SystemRequirementsDict({}), SystemRequirementsDict({}), SystemRequirementsDict({}))
         cloned_system_requirements_by_executable = {}
 
     # convert runtime --instance-type into mapping {entrypoint:{'instanceType':xxx}}
@@ -3227,12 +3233,15 @@ def run_body(args, executable, dest_proj, dest_path, preset_inputs=None, input_n
     else:
         requested_cluster_spec = cloned_cluster_spec
 
-    # fpga driver now does not have corresponding dx run option, so it can only be requested using the cloned value
+    # fpga/nvidia driver now does not have corresponding dx run option,
+    # so it can only be requested using the cloned value
     requested_fpga_driver = cloned_fpga_driver
+    requested_nvidia_driver = cloned_nvidia_driver
 
-    # combine the requested instance type, full cluster spec, fpga spec
+    # combine the requested instance type, full cluster spec, fpga spec, nvidia spec
     # into the runtime systemRequirements
-    requested_system_requirements = (requested_instance_type + requested_cluster_spec + requested_fpga_driver).as_dict()
+    requested_system_requirements = (requested_instance_type + requested_cluster_spec + requested_fpga_driver +
+                                     requested_nvidia_driver).as_dict()
 
     if (args.instance_type and cloned_system_requirements_by_executable):
         warning = BOLD("WARNING") + ": --instance-type argument: {} may get overridden by".format(args.instance_type) 
@@ -3283,6 +3292,7 @@ def run_body(args, executable, dest_proj, dest_path, preset_inputs=None, input_n
         "instance_type": None,
         "cluster_spec": None,
         "fpga_driver": None,
+        "nvidia_driver": None,
         "stage_instance_types": args.stage_instance_types,
         "stage_folders": args.stage_folders,
         "rerun_stages": args.rerun_stages,
@@ -5419,6 +5429,9 @@ parser_update_project.add_argument('--bill-to', help="Update the user or org ID
 allowed_executables_group = parser_update_project.add_mutually_exclusive_group()
 allowed_executables_group.add_argument('--allowed-executables', help='Executable ID(s) this project is allowed to run.  This operation overrides any existing list of executables.', type=str, nargs="+")
 allowed_executables_group.add_argument('--unset-allowed-executables', help='Removes any restriction to run executables as set by --allowed-executables', action='store_true')
+database_results_restricted_group = parser_update_project.add_mutually_exclusive_group()
+database_results_restricted_group.add_argument('--database-results-restricted', help='Viewers on the project can access only more than specified size of visual data from databases', type=positive_integer)
+database_results_restricted_group.add_argument('--unset-database-results-restricted', help='Removes any restriction to return data from databases as set by --database-results-restricted', action='store_true')
 
 parser_update_project.set_defaults(func=update_project)
 register_parser(parser_update_project, subparsers_action=subparsers_update, categories="metadata")
@@ -5805,10 +5818,12 @@ parser_new_project.add_argument('--phi', help='Add PHI protection to project', d
                                 action='store_true')
 parser_new_project.add_argument('--database-ui-view-only', help='Viewers on the project cannot access database data directly', default=False,
                                 action='store_true')
+parser_new_project.add_argument('--database-results-restricted', help='Viewers on the project can access only more than specified size of visual data from databases', type=positive_integer)
 parser_new_project.add_argument('--monthly-compute-limit', type=positive_integer, help='Monthly project spending limit for compute')
 parser_new_project.add_argument('--monthly-egress-bytes-limit', type=positive_integer, help='Monthly project spending limit for egress (in Bytes)')
 parser_new_project.add_argument('--monthly-storage-limit', type=positive_number, help='Monthly project spending limit for storage')
-parser_new_project.add_argument('--default-symlink', help='Default symlink for external store account')
+parser_new_project.add_argument('--default-symlink', help='Default symlink for external storage account')
+parser_new_project.add_argument('--drive', help='Drive for external storage account')
 parser_new_project.set_defaults(func=new_project)
 register_parser(parser_new_project, subparsers_action=subparsers_new, categories='fs')
 

{dxpy-0.384.0 → dxpy-0.386.0}/dxpy/scripts/dx_build_app.py

@@ -357,11 +357,11 @@ def _verify_app_source_dir_impl(src_dir, temp_dir, mode, enforce=True):
         if "interpreter" not in manifest['runSpec']:
             raise dxpy.app_builder.AppBuilderException('runSpec.interpreter field was not present')
 
-        if "release" not in manifest['runSpec'] or "distribution" not in manifest['runSpec']:
-            warn_message = 'runSpec.distribution or runSpec.release was not present. These fields '
-            warn_message += 'will be required in a future version of the API. Recommended value '
-            warn_message += 'for distribution is \"Ubuntu\" and release - \"14.04\".'
-            logger.warn(warn_message)
+        if "distribution" not in manifest['runSpec']:
+            raise dxpy.app_builder.AppBuilderException('Required field runSpec.distribution is not present')
+
+        if "release" not in manifest['runSpec']:
+            raise dxpy.app_builder.AppBuilderException('Required field runSpec.release is not present')
 
         if manifest['runSpec']['interpreter'] in ["python2.7", "bash", "python3"]:
             if "file" in manifest['runSpec']:

{dxpy-0.384.0 → dxpy-0.386.0}/dxpy/system_requirements.py

@@ -85,7 +85,7 @@ class SystemRequirementsDict(object):
         It can extract only entrypoints with specific fields ('clusterSpec',
         'instanceType', etc), depending on the value of _type.
         """
-        allowed_types = ['all', 'clusterSpec', 'instanceType', 'fpgaDriver']
+        allowed_types = ['all', 'clusterSpec', 'instanceType', 'fpgaDriver', 'nvidiaDriver']
         if _type not in (allowed_types):
             raise DXError("Expected '_type' to be one of the following: {}".format(allowed_types))
 

dxpy-0.386.0/dxpy/toolkit_version.py

@@ -0,0 +1 @@
+version = '0.386.0'

{dxpy-0.384.0 → dxpy-0.386.0}/dxpy/utils/describe.py

@@ -415,7 +415,7 @@ def print_project_desc(desc, verbose=False):
         'containsPHI', 'databaseUIViewOnly', 'externalUploadRestricted', 'region', 'storageCost', 'pendingTransfer',
         'atSpendingLimit', 'currentMonthComputeAvailableBudget', 'currentMonthEgressBytesAvailableBudget',
         'currentMonthStorageAvailableBudget', 'currentMonthComputeUsage', 'currentMonthEgressBytesUsage', 
-        'currentMonthExpectedStorageUsage', 'defaultSymlink'
+        'currentMonthExpectedStorageUsage', 'defaultSymlink', 'databaseResultsRestricted',
         # Following are app container-specific
         'destroyAt', 'project', 'type', 'app', 'appName'
     ]
@@ -455,6 +455,8 @@ def print_project_desc(desc, verbose=False):
         print_json_field('External Upload Restricted', desc['externalUploadRestricted'])
     if 'defaultSymlink' in desc and verbose:
         print_json_field('Default Symlink', desc['defaultSymlink'])
+    if 'databaseResultsRestricted' in desc and desc['databaseResultsRestricted']:
+        print_json_field('Database Results Restricted', desc['databaseResultsRestricted'])
 
     # Usage
     print_field("Created", render_timestamp(desc['created']))

{dxpy-0.384.0 → dxpy-0.386.0}/dxpy/utils/exec_utils.py

@@ -23,7 +23,7 @@ from __future__ import print_function, unicode_literals, division, absolute_impo
 import os, sys, json, re, collections, logging, argparse, string, itertools, subprocess, tempfile
 from functools import wraps
 from collections import namedtuple
-import pipes
+import shlex
 
 import dxpy
 from ..compat import USING_PYTHON2, open, Mapping
@@ -435,7 +435,7 @@ class DXExecDependencyInstaller(object):
                 dxpy.download_dxfile(bundle["id"], bundle["name"], project=dxpy.WORKSPACE_ID)
             except dxpy.exceptions.ResourceNotFound:
                 dxpy.download_dxfile(bundle["id"], bundle["name"])
-            self.run("dx-unpack {}".format(pipes.quote(bundle["name"])))
+            self.run("dx-unpack {}".format(shlex.quote(bundle["name"])))
         else:
             self.log('Skipping bundled dependency "{name}" because it does not refer to a file'.format(**bundle))
 

{dxpy-0.384.0 → dxpy-0.386.0}/dxpy/utils/file_load_utils.py

@@ -83,7 +83,7 @@ will download into the execution environment:
 from __future__ import print_function, unicode_literals, division, absolute_import
 
 import json
-import pipes
+import shlex
 import os
 import fnmatch
 import sys
@@ -401,10 +401,6 @@ def analyze_bash_vars(job_input_file, job_homedir):
     return file_key_descs, rest_hash
 
 
-#
-# Note: pipes.quote() to be replaced with shlex.quote() in Python 3
-# (see http://docs.python.org/2/library/pipes.html#pipes.quote)
-#
 def gen_bash_vars(job_input_file, job_homedir=None, check_name_collision=True):
     """
     :param job_input_file: path to a JSON file describing the job inputs
@@ -427,7 +423,7 @@ def gen_bash_vars(job_input_file, job_homedir=None, check_name_collision=True):
             result = json.dumps(dxpy.dxlink(elem))
         else:
             result = json.dumps(elem)
-        return pipes.quote(result)
+        return shlex.quote(result)
 
     def string_of_value(val):
         if isinstance(val, list):

{dxpy-0.384.0 → dxpy-0.386.0}/dxpy/utils/local_exec_utils.py

@@ -16,7 +16,7 @@
 
 from __future__ import print_function, unicode_literals, division, absolute_import
 
-import os, sys, json, subprocess, pipes
+import os, sys, json, subprocess, shlex
 import collections, datetime
 
 import dxpy
@@ -351,9 +351,9 @@ def run_one_entry_point(job_id, function, input_hash, run_spec, depends_on, name
           if [[ $(type -t {function}) == "function" ]];
           then {function};
           else echo "$0: Global scope execution complete. Not invoking entry point function {function} because it was not found" 1>&2;
-          fi'''.format(homedir=pipes.quote(job_homedir),
-                       env_path=pipes.quote(os.path.join(job_env['HOME'], 'environment')),
-                       code_path=pipes.quote(environ['DX_TEST_CODE_PATH']),
+          fi'''.format(homedir=shlex.quote(job_homedir),
+                       env_path=shlex.quote(os.path.join(job_env['HOME'], 'environment')),
+                       code_path=shlex.quote(environ['DX_TEST_CODE_PATH']),
                        function=function)
         invocation_args = ['bash', '-c', '-e'] + (['-x'] if environ.get('DX_TEST_X_FLAG') else []) + [script]
     elif run_spec['interpreter'] == 'python2.7':

{dxpy-0.384.0 → dxpy-0.386.0}/dxpy.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dxpy
-Version: 0.384.0
+Version: 0.386.0
 Summary: DNAnexus Platform API bindings for Python
 Home-page: https://github.com/dnanexus/dx-toolkit
 Author: Aleksandra Zalcman, Andrey Kislyuk, Anurag Biyani, Geet Duggal, Katherine Lai, Kurt Jensen, Marek Hrvol, Ohad Rodeh, Phil Sung
@@ -48,6 +48,14 @@ Example:
 $ _DX_DEBUG=1 dx ls
 ```
 
+### Debugging inside the IDE (PyCharm)
+To be able to debug dx-toolkit (dx commands) directly in the IDE, 'Run/Debug Configurations' needs to be changed.
+1. Go to Run → Edit Configurations...
+2. Add New Configuration (Python)
+3. Change script to module (dxpy.scripts.dx)
+4. To Script parameters field write dx command you want to run (eg 'ls' runs 'dx ls')
+5. Apply and OK (now it is possible to start debugging via main() function in dx.py)
+
 Python coding style
 -------------------
 

{dxpy-0.384.0 → dxpy-0.386.0}/test/test_dx_app_wizard.py

@@ -19,7 +19,6 @@
 
 import os, sys, unittest, json, tempfile, subprocess
 import pexpect
-import pipes
 
 from dxpy_testutil import DXTestCase, check_output
 import dxpy_testutil as testutil

{dxpy-0.384.0 → dxpy-0.386.0}/test/test_dx_bash_helpers.py

@@ -23,7 +23,7 @@ import dxpy
 import dxpy_testutil as testutil
 import json
 import os
-import pipes
+import shlex
 import pytest
 import shutil
 import tempfile
@@ -42,7 +42,7 @@ from dxpy.bindings.download_all_inputs import _get_num_parallel_threads
 def run(command, **kwargs):
     try:
         if isinstance(command, list) or isinstance(command, tuple):
-            print("$ %s" % " ".join(pipes.quote(f) for f in command))
+            print("$ %s" % " ".join(shlex.quote(f) for f in command))
             output = check_output(command, **kwargs)
         else:
             print("$ %s" % (command,))
@@ -919,22 +919,38 @@ class TestDXJobutilNewJob(DXTestCase):
             ),
             # instance type: mapping
             ("--instance-type " +
-                pipes.quote(json.dumps({"main": "mem2_hdd2_x2" , "other_function": "mem2_hdd2_x1" })),
+                shlex.quote(json.dumps({"main": "mem2_hdd2_x2" , "other_function": "mem2_hdd2_x1" })),
                 {"systemRequirements": {"main": { "instanceType": "mem2_hdd2_x2" },
                                         "other_function": { "instanceType": "mem2_hdd2_x1" }}}),
             ("--instance-type-by-executable " +
-                pipes.quote(json.dumps({"my_applet": {"main": "mem2_hdd2_x2",
+                shlex.quote(json.dumps({"my_applet": {"main": "mem2_hdd2_x2",
                                         "other_function": "mem3_ssd2_fpga1_x8"}})),
                 {"systemRequirementsByExecutable": {"my_applet": {"main": {"instanceType": "mem2_hdd2_x2"},
                                                     "other_function": {"instanceType": "mem3_ssd2_fpga1_x8"}}}}),
             ("--instance-type-by-executable " +
-             pipes.quote(json.dumps({"my_applet": {"main": "mem1_ssd1_v2_x2",
+             shlex.quote(json.dumps({"my_applet": {"main": "mem1_ssd1_v2_x2",
                                                    "other_function": "mem3_ssd2_fpga1_x8"}})) +
              " --extra-args " +
-                pipes.quote(json.dumps({"systemRequirementsByExecutable": {"my_applet": {"main": {"instanceType": "mem2_hdd2_x2", "clusterSpec": {"initialInstanceCount": 3}},
+                shlex.quote(json.dumps({"systemRequirementsByExecutable": {"my_applet": {"main": {"instanceType": "mem2_hdd2_x2", "clusterSpec": {"initialInstanceCount": 3}},
                                         "other_function": {"fpgaDriver": "edico-1.4.5"}}}})),
                 {"systemRequirementsByExecutable": {"my_applet":{"main": { "instanceType": "mem2_hdd2_x2", "clusterSpec":{"initialInstanceCount": 3}},
                                         "other_function": { "instanceType": "mem3_ssd2_fpga1_x8", "fpgaDriver": "edico-1.4.5"} }}}),
+            # nvidia driver
+            ("--instance-type-by-executable " +
+             shlex.quote(json.dumps({
+                 "my_applet": {
+                     "main": "mem1_ssd1_v2_x2",
+                     "other_function": "mem2_ssd1_gpu_x16"}})) +
+             " --extra-args " +
+             shlex.quote(json.dumps({
+                 "systemRequirementsByExecutable": {
+                     "my_applet": {
+                         "main": {"instanceType": "mem2_hdd2_x2"},
+                         "other_function": {"nvidiaDriver": "R535"}}}})),
+             {"systemRequirementsByExecutable": {
+                 "my_applet": {"main": {"instanceType": "mem2_hdd2_x2"},
+                               "other_function": {"instanceType": "mem2_ssd1_gpu_x16",
+                                                  "nvidiaDriver": "R535"}}}}),
             # properties - mapping
             (
                 "--property foo=foo_value --property bar=bar_value",
@@ -947,14 +963,14 @@ class TestDXJobutilNewJob(DXTestCase):
             self.assertNewJobInputHash(cmd_snippet, arguments_hash)
 
     def test_extra_arguments(self):
-        cmd_snippet = "--extra-args " + pipes.quote(
+        cmd_snippet = "--extra-args " + shlex.quote(
             json.dumps({"details": {"d1": "detail1", "d2": 1234}, "foo": "foo_value"})
         )
         arguments_hash = {"details": {"d1": "detail1", "d2": 1234}, "foo": "foo_value"}
         self.assertNewJobInputHash(cmd_snippet, arguments_hash)
 
         # override previously specified args
-        cmd_snippet = "--name JobName --extra-args " + pipes.quote(
+        cmd_snippet = "--name JobName --extra-args " + shlex.quote(
             json.dumps({"name": "FinalName"})
         )
         arguments_hash = {"name": "FinalName"}