dxpy 0.374.0__tar.gz → 0.375.1__tar.gz

{dxpy-0.374.0 → dxpy-0.375.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dxpy
-Version: 0.374.0
+Version: 0.375.1
 Summary: DNAnexus Platform API bindings for Python
 Home-page: https://github.com/dnanexus/dx-toolkit
 Author: Aleksandra Zalcman, Andrey Kislyuk, Anurag Biyani, Geet Duggal, Katherine Lai, Kurt Jensen, Marek Hrvol, Ohad Rodeh, Phil Sung

{dxpy-0.374.0 → dxpy-0.375.1}/dxpy/bindings/dxapplet.py

@@ -242,7 +242,7 @@ class DXExecutable:
         :param preserve_job_outputs: Copy cloneable outputs of every non-reused job entering "done" state in this root execution to a folder in the project. If value is True it will place job outputs into the "intermediateJobOutputs" subfolder under the output folder for the root execution. If the value is dict, it may contains "folder" key with desired folder path. If the folder path starts with '/' it refers to an absolute path within the project, otherwise, it refers to a subfolder under root execution's output folder.
         :type preserve_job_outputs: boolean or dict
         :param detailed_job_metrics: Enable detailed job metrics for this root execution
-        :type preserve_job_outputs: boolean
+        :type detailed_job_metrics: boolean
         :param extra_args: If provided, a hash of options that will be merged into the underlying JSON given for the API call
         :type extra_args: dict
         :returns: Object handler of the newly created job

{dxpy-0.374.0 → dxpy-0.375.1}/dxpy/scripts/dx.py

@@ -3181,7 +3181,7 @@ def run_batch_all_steps(args, executable, dest_proj, dest_path, input_json, run_
 def run_body(args, executable, dest_proj, dest_path, preset_inputs=None, input_name_prefix=None):
     input_json = _get_input_for_run(args, executable, preset_inputs)
 
-    requested_instance_type, requested_cluster_spec = {}, {}
+    requested_instance_type, requested_cluster_spec, requested_system_requirements_by_executable = {}, {}, {}
     executable_describe = None
 
     if args.cloned_job_desc:
@@ -3192,7 +3192,7 @@ def run_body(args, executable, dest_proj, dest_path, preset_inputs=None, input_n
         cloned_instance_type = SystemRequirementsDict.from_sys_requirements(cloned_system_requirements, _type='instanceType')
         cloned_cluster_spec = SystemRequirementsDict.from_sys_requirements(cloned_system_requirements, _type='clusterSpec')
         cloned_fpga_driver = SystemRequirementsDict.from_sys_requirements(cloned_system_requirements, _type='fpgaDriver')
-        cloned_system_requirements_by_executable = args.cloned_job_desc.get("mergedSystemRequirementsByExecutable", {})
+        cloned_system_requirements_by_executable = args.cloned_job_desc.get("mergedSystemRequirementsByExecutable", {}) or {}
     else:
         cloned_system_requirements = {}
         cloned_instance_type, cloned_cluster_spec, cloned_fpga_driver = SystemRequirementsDict({}), SystemRequirementsDict({}), SystemRequirementsDict({})
@@ -3231,10 +3231,25 @@ def run_body(args, executable, dest_proj, dest_path, preset_inputs=None, input_n
     # into the runtime systemRequirements
     requested_system_requirements = (requested_instance_type + requested_cluster_spec + requested_fpga_driver).as_dict()
 
-    # store runtime --instance-type-by-executable {executable:{entrypoint:{'instanceType':xxx}}} as systemRequirementsByExecutable 
+    if (args.instance_type and cloned_system_requirements_by_executable):
+        warning = BOLD("WARNING") + ": --instance-type argument: {} may get overridden by".format(args.instance_type) 
+        warning += " {} mergedSystemRequirementsByExecutable: {}".format(args.cloned_job_desc.get('id'), json.dumps(cloned_system_requirements_by_executable))
+        if (args.instance_type_by_executable):
+            warning += " and runtime --instance-type-by-executable argument:{}\n".format(json.dumps(args.instance_type_by_executable))
+        print(fill(warning))
+        print()
+
+    # store runtime --instance-type-by-executable {executable:{entrypoint:xxx}} as systemRequirementsByExecutable
     # Note: currently we don't have -by-executable options for other fields, for example --instance-count-by-executable
     # so this runtime systemRequirementsByExecutable double mapping only contains instanceType under each executable.entrypoint
-    requested_system_requirements_by_executable = SystemRequirementsDict(args.instance_type_by_executable).as_dict() or cloned_system_requirements_by_executable
+    if args.instance_type_by_executable:
+        requested_system_requirements_by_executable = {exec: SystemRequirementsDict.from_instance_type(sys_req_by_exec).as_dict(
+        ) for exec, sys_req_by_exec in args.instance_type_by_executable.items()}
+        requested_system_requirements_by_executable = SystemRequirementsDict(merge(
+            cloned_system_requirements_by_executable, requested_system_requirements_by_executable)).as_dict()
+    else:
+        requested_system_requirements_by_executable = cloned_system_requirements_by_executable
+
 
     if args.debug_on:
         if 'All' in args.debug_on:
@@ -3603,7 +3618,11 @@ def run(args):
     if args.clone is not None:
         # Resolve job ID or name; both job-id and analysis-id can be described using job_describe()
         if is_job_id(args.clone) or is_analysis_id(args.clone):
-            clone_desc = dxpy.api.job_describe(args.clone)
+            clone_desc = dxpy.api.job_describe(args.clone, {"defaultFields": True, 
+                                                            "fields": {"runSystemRequirements": True, 
+                                                                       "runSystemRequirementsByExecutable": True, 
+                                                                       "mergedSystemRequirementsByExecutable": True}})
+
         else:
             iterators = []
             if ":" in args.clone:

dxpy-0.375.1/dxpy/toolkit_version.py

@@ -0,0 +1 @@
+version = '0.375.1'

{dxpy-0.374.0 → dxpy-0.375.1}/dxpy.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dxpy
-Version: 0.374.0
+Version: 0.375.1
 Summary: DNAnexus Platform API bindings for Python
 Home-page: https://github.com/dnanexus/dx-toolkit
 Author: Aleksandra Zalcman, Andrey Kislyuk, Anurag Biyani, Geet Duggal, Katherine Lai, Kurt Jensen, Marek Hrvol, Ohad Rodeh, Phil Sung

{dxpy-0.374.0 → dxpy-0.375.1}/scripts/dx-jobutil-new-job

@@ -54,11 +54,14 @@ def get_job_new_input(args):
     if args.instance_type is not None:
         try_call(process_instance_type_arg, args, False)
         job_new_input["systemRequirements"] = SystemRequirementsDict.from_instance_type(args.instance_type, args.function).as_dict()
-    
+
     if args.instance_type_by_executable is not None:
         try_call(process_instance_type_by_executable_arg, args)
-        job_new_input["systemRequirementsByExecutable"] = SystemRequirementsDict(args.instance_type_by_executable).as_dict()
-    
+        job_new_input["systemRequirementsByExecutable"] = {
+            exec: SystemRequirementsDict.from_instance_type(sys_req_by_exec).as_dict()
+            for exec, sys_req_by_exec in args.instance_type_by_executable.items()
+        }
+
     if args.head_job_on_demand is not None:
         job_new_input['headJobOnDemand'] = args.head_job_on_demand
 

{dxpy-0.374.0 → dxpy-0.375.1}/test/test_dx_bash_helpers.py

@@ -918,53 +918,23 @@ class TestDXJobutilNewJob(DXTestCase):
                 },
             ),
             # instance type: mapping
-            (
-                "--instance-type "
-                + pipes.quote(
-                    json.dumps(
-                        {"main": "mem2_hdd2_x2", "other_function": "mem2_hdd2_x1"}
-                    )
-                ),
-                {
-                    "systemRequirements": {
-                        "main": {"instanceType": "mem2_hdd2_x2"},
-                        "other_function": {"instanceType": "mem2_hdd2_x1"},
-                    }
-                },
-            ),
-            (
-                "--instance-type-by-executable "
-                + pipes.quote(
-                    json.dumps(
-                        {
-                            "my_applet": {
-                                "main": {
-                                    "instanceType": "mem2_hdd2_x2",
-                                    "clusterSpec": {"initialInstanceCount": 3},
-                                },
-                                "other_function": {
-                                    "instanceType": "mem3_ssd2_fpga1_x8",
-                                    "fpgaDriver": "edico-1.4.5",
-                                },
-                            }
-                        }
-                    )
-                ),
-                {
-                    "systemRequirementsByExecutable": {
-                        "my_applet": {
-                            "main": {
-                                "instanceType": "mem2_hdd2_x2",
-                                "clusterSpec": {"initialInstanceCount": 3},
-                            },
-                            "other_function": {
-                                "instanceType": "mem3_ssd2_fpga1_x8",
-                                "fpgaDriver": "edico-1.4.5",
-                            },
-                        }
-                    }
-                },
-            ),
+            ("--instance-type " +
+                pipes.quote(json.dumps({"main": "mem2_hdd2_x2" , "other_function": "mem2_hdd2_x1" })),
+                {"systemRequirements": {"main": { "instanceType": "mem2_hdd2_x2" },
+                                        "other_function": { "instanceType": "mem2_hdd2_x1" }}}),
+            ("--instance-type-by-executable " +
+                pipes.quote(json.dumps({"my_applet": {"main": "mem2_hdd2_x2",
+                                        "other_function": "mem3_ssd2_fpga1_x8"}})),
+                {"systemRequirementsByExecutable": {"my_applet": {"main": {"instanceType": "mem2_hdd2_x2"},
+                                                    "other_function": {"instanceType": "mem3_ssd2_fpga1_x8"}}}}),
+            ("--instance-type-by-executable " +
+             pipes.quote(json.dumps({"my_applet": {"main": "mem1_ssd1_v2_x2",
+                                                   "other_function": "mem3_ssd2_fpga1_x8"}})) +
+             " --extra-args " +
+                pipes.quote(json.dumps({"systemRequirementsByExecutable": {"my_applet": {"main": {"instanceType": "mem2_hdd2_x2", "clusterSpec": {"initialInstanceCount": 3}},
+                                        "other_function": {"fpgaDriver": "edico-1.4.5"}}}})),
+                {"systemRequirementsByExecutable": {"my_applet":{"main": { "instanceType": "mem2_hdd2_x2", "clusterSpec":{"initialInstanceCount": 3}},
+                                        "other_function": { "instanceType": "mem3_ssd2_fpga1_x8", "fpgaDriver": "edico-1.4.5"} }}}),
             # properties - mapping
             (
                 "--property foo=foo_value --property bar=bar_value",

{dxpy-0.374.0 → dxpy-0.375.1}/test/test_dxclient.py

@@ -3310,9 +3310,8 @@ dx-jobutil-add-output record_array $second_record --array
         applet_id = dxpy.api.applet_new(app_spec)['id']
         requested_inst_type_by_exec = {
             applet_id: {
-                "main": {
-                    "instanceType": "mem2_ssd1_v2_x2",
-                    "clusterSpec": {"initialInstanceCount": 3}}}}
+                "main": "mem2_ssd1_v2_x2",
+                "other": "mem2_hdd2_x1"}}
 
         (stdout, stderr) = run('_DX_DEBUG=2 dx run ' + applet_id + ' ' +
                                '--instance-type mem2_hdd2_x2 ' +
@@ -3322,7 +3321,10 @@ dx-jobutil-add-output record_array $second_record --array
                                json.dumps(requested_inst_type_by_exec) + '\'',
                                also_return_stderr=True)
         expected_sys_reqs_by_exec = '"systemRequirementsByExecutable": ' + \
-            json.dumps(requested_inst_type_by_exec)
+            json.dumps({
+                applet_id: {
+                    "main": {"instanceType": "mem2_ssd1_v2_x2"},
+                    "other": {"instanceType": "mem2_hdd2_x1"}}})
         self.assertIn(expected_sys_reqs_by_exec, stderr)
 
         # parsing error
@@ -3399,12 +3401,14 @@ dx-jobutil-add-output record_array $second_record --array
         orig_job_id = run("dx run " + applet_id +
                           ' -inumber=32 --name jobname --folder /output ' +
                           '--instance-type mem2_hdd2_x2 ' +
-                          '--instance-type-by-executable \'{"' + applet_id + '": {"*": {"instanceType": "mem1_ssd1_v2_x2"}}}\' '   
+                          '--instance-type-by-executable \'{"' + applet_id + '": {"*": "mem1_ssd1_v2_x2"}}\' '   
                           '--tag Ψ --tag $hello.world ' +
                           '--property Σ_1^n=n --property $hello.=world ' +
                           '--priority normal ' +
-                          '--brief -y').strip()
-        orig_job_desc = dxpy.api.job_describe(orig_job_id, {"defaultFields": True, "fields":{"runSystemRequirements":True, "runSystemRequirementsByExecutable":True, "mergedSystemRequirementsByExecutable":True}} )
+                          '--brief -y').strip().split('\n')[-1]
+        orig_job_desc = dxpy.api.job_describe(orig_job_id, {"defaultFields": True, 
+                                                            "fields":{
+                                                                "runSystemRequirements":True, "runSystemRequirementsByExecutable":True, "mergedSystemRequirementsByExecutable":True}} )
         # control
         self.assertEqual(orig_job_desc['name'], 'jobname')
         self.assertEqual(orig_job_desc['project'], self.project)
@@ -3423,8 +3427,10 @@ dx-jobutil-add-output record_array $second_record --array
         check_new_job_metadata(new_job_desc, orig_job_desc)
 
         def get_new_job_desc(cmd_suffix):
-            new_job_id = run("dx run --clone " + orig_job_id + " --brief -y " + cmd_suffix).strip()
-            return dxpy.api.job_describe(new_job_id)
+            new_job_id = run("dx run --clone " + orig_job_id + " --brief -y " + cmd_suffix).strip().split('\n')[-1]
+            return dxpy.api.job_describe(new_job_id, {"defaultFields": True, 
+                                                      "fields":{
+                                                          "runSystemRequirements":True, "runSystemRequirementsByExecutable":True, "mergedSystemRequirementsByExecutable":True}})
 
         # override applet
         new_job_desc = get_new_job_desc(other_applet_id)
@@ -3483,8 +3489,14 @@ dx-jobutil-add-output record_array $second_record --array
         self.assertEqual(new_job_desc['input'], {"number2": 42})
         check_new_job_metadata(new_job_desc, orig_job_desc, overridden_fields=['input'])
 
-        # --instance-type override: original job with universal instance type 
+        # --instance-type override: original job with universal instance type
         # override the blanket instance type
+        orig_job_id = run("dx run " + applet_id +
+                          ' --instance-type mem1_ssd1_v2_x2 ' +
+                          '--brief -y').strip().split('\n')[-1]
+        orig_job_desc = dxpy.api.job_describe(orig_job_id, {"defaultFields": True, 
+                                                            "fields": {"runSystemRequirements": True,
+                                                                       "runSystemRequirementsByExecutable": True, "mergedSystemRequirementsByExecutable": True}})
         new_job_desc = get_new_job_desc("--instance-type mem2_hdd2_x1")
         self.assertEqual(new_job_desc['systemRequirements'],
                          {'*': {'instanceType': 'mem2_hdd2_x1'}})
@@ -3640,6 +3652,28 @@ dx-jobutil-add-output record_array $second_record --array
         self.assertEqual(new_job_desc['systemRequirements']['some_ep']['fpgaDriver'], 'edico-1.4.5')
         self.assertEqual(new_job_desc['systemRequirements']['some_ep']['clusterSpec']['bootstrapScript'], 'z.sh')
 
+        # --instance-type and --instance-type-by-executable override
+        orig_job_id = run("dx run " + other_applet_id +
+                          " --instance-type mem2_hdd2_x2" +
+                          " --instance-type-by-executable \'" + 
+                          json.dumps({other_applet_id: {"some_ep": "mem1_ssd1_v2_x2", "some_other_ep": "mem2_hdd2_x4"}}) + "\'" +
+                          " --brief -y").strip().split('\n')[-1]
+        orig_job_desc = dxpy.api.job_describe(orig_job_id, {"defaultFields": True, "fields": {
+                                              "runSystemRequirements": True, "runSystemRequirementsByExecutable": True, "mergedSystemRequirementsByExecutable": True}})
+
+        new_job_desc = get_new_job_desc("--instance-type-by-executable \'" + 
+                          json.dumps({other_applet_id: {"some_ep": "mem1_ssd1_v2_x8", "*": "mem2_hdd2_x1"}}) + "\'")
+
+        # cloned from original job systemRequirements
+        self.assertEqual(new_job_desc["runSystemRequirements"], orig_job_desc["systemRequirements"])
+        self.assertEqual(
+            new_job_desc['runSystemRequirementsByExecutable'][other_applet_id]['some_ep']['instanceType'], 'mem1_ssd1_v2_x8')
+        self.assertEqual(
+            new_job_desc['runSystemRequirementsByExecutable'][other_applet_id]['*']['instanceType'], 'mem2_hdd2_x1')
+        # cloned from original job mergedSystemRequirements
+        self.assertEqual(
+            new_job_desc['runSystemRequirementsByExecutable'][other_applet_id]['some_other_ep']['instanceType'], 'mem2_hdd2_x4')
+
     @unittest.skipUnless(testutil.TEST_RUN_JOBS,
                          'skipping tests that would run jobs')
     def test_dx_describe_job_with_resolved_jbors(self):
@@ -3882,9 +3916,9 @@ class TestDXClientWorkflow(DXTestCaseBuildWorkflows):
         change_inst_type_analysis_id = run("dx run --clone " + analysis_id +
                                            " --instance-type mem2_hdd2_x2 --brief -y").strip()
         change_inst_type_by_exec_analysis_id = run("dx run --clone " + analysis_id +
-                                           " --instance-type-by-executable \'" + 
-                                           json.dumps({applet_id:{"*": {"instanceType": "mem2_ssd1_v2_x2"}}}) +
-                                           "\' --brief -y").strip()
+                                                   " --instance-type-by-executable \'" +
+                                                   json.dumps({applet_id: {"*": "mem2_ssd1_v2_x2"}}) +
+                                                   "\' --brief -y").strip()
 
         time.sleep(25) # May need to wait for any new jobs to be created in the system
 
@@ -4004,9 +4038,9 @@ class TestDXClientWorkflow(DXTestCaseBuildWorkflows):
                          '--instance-type second=mem2_hdd2_x1 -y --brief').strip()
 
         # request for an executable
-        exec_req_id = run("dx run myworkflow" + 
-                         " --instance-type-by-executable \'" + 
-                         json.dumps({applet_id:{"*": {"instanceType": "mem2_ssd1_v2_x2"}}}) +
+        exec_req_id = run("dx run myworkflow" +
+                          " --instance-type-by-executable \'" +
+                          json.dumps({applet_id: {"*": "mem2_ssd1_v2_x2"}}) +
                          "\' --brief -y").strip()
         
         time.sleep(10) # give time for all jobs to be populated

dxpy-0.374.0/dxpy/toolkit_version.py

@@ -1 +0,0 @@
-version = '0.374.0'