dxpy 0.394.0__tar.gz → 0.396.0__tar.gz

{dxpy-0.394.0 → dxpy-0.396.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dxpy
-Version: 0.394.0
+Version: 0.396.0
 Summary: DNAnexus Platform API bindings for Python
 Home-page: https://github.com/dnanexus/dx-toolkit
 Author: Aleksandra Zalcman, Andrey Kislyuk, Anurag Biyani, Geet Duggal, Katherine Lai, Kurt Jensen, Marek Hrvol, Ohad Rodeh, Phil Sung

{dxpy-0.394.0 → dxpy-0.396.0}/dxpy/__init__.py

@@ -136,9 +136,11 @@ import socket
 import threading
 import certifi
 from collections import namedtuple
+from http.client import BadStatusLine
+from io import StringIO
+from reprlib import Repr
 
 from . import exceptions
-from .compat import BadStatusLine, StringIO, bytes, Repr
 from .utils.printing import BOLD, BLUE, YELLOW, GREEN, RED, WHITE
 
 from random import randint
@@ -353,7 +355,14 @@ def _calculate_retry_delay(response, num_attempts):
     '''
     if response is not None and response.status in (503, 429) and 'retry-after' in response.headers:
         try:
-            return int(response.headers['retry-after'])
+            suggested_delay = int(response.headers['retry-after'])
+
+            # By default, apiserver doesn't track attempts and doesn't provide increased timeout over attempts.
+            # So, increasing backoff for throttled requests up to x5 times from the original one.
+            # The current implementation of apiserver returns a retry-after header ranging from 20 to 30 seconds.
+            # Thus, after the 20th attempt the delay will always be between 100 and 150 seconds.
+            return suggested_delay if suggested_delay >= 60 \
+                else suggested_delay + int(0.25 * min(num_attempts - 1, 20) * suggested_delay)
         except ValueError:
             # In RFC 2616, retry-after can be formatted as absolute time
             # instead of seconds to wait. We don't bother to parse that,
@@ -361,7 +370,8 @@ def _calculate_retry_delay(response, num_attempts):
             pass
     if num_attempts <= 1:
         return 1
-    num_attempts = min(num_attempts, 7)
+    num_attempts = min(num_attempts, 8)
+    # After the 8th attempt the delay will always be between 64 and 128 seconds
     return randint(2 ** (num_attempts - 2), 2 ** (num_attempts - 1))
 
 
@@ -514,6 +524,9 @@ def DXHTTPRequest(resource, data, method='POST', headers=None, auth=True,
        through to :func:`DXHTTPRequest`.
 
     '''
+    # option wasn't named correctly, so to not break existing clients rename it locally for clarity
+    safe_to_retry = always_retry
+
     if headers is None:
         headers = {}
 
@@ -719,7 +732,7 @@ def DXHTTPRequest(resource, data, method='POST', headers=None, auth=True,
                 # up to (max_retries) subsequent retries.
                 total_allowed_tries = max_retries + 1
                 ok_to_retry = False
-                is_retryable = always_retry or (method == 'GET') or _is_retryable_exception(e)
+                is_retryable = safe_to_retry or (method == 'GET') or _is_retryable_exception(e)
                 # Because try_index is not incremented until we escape
                 # this iteration of the loop, try_index is equal to the
                 # number of tries that have failed so far, minus one.
@@ -800,7 +813,7 @@ def DXHTTPRequest(resource, data, method='POST', headers=None, auth=True,
                              want_full_response=want_full_response,
                              decode_response_body=decode_response_body, prepend_srv=prepend_srv,
                              session_handler=session_handler,
-                             max_retries=max_retries, always_retry=always_retry, **kwargs)
+                             max_retries=max_retries, always_retry=safe_to_retry, **kwargs)
     raise AssertionError('Should never reach this line: should never break out of loop')
 
 

{dxpy-0.394.0 → dxpy-0.396.0}/dxpy/app_builder.py

@@ -49,7 +49,6 @@ import dxpy.executable_builder
 from . import logger
 from .utils import merge
 from .utils.printing import fill
-from .compat import input
 from .cli import INTERACTIVE_CLI
 
 NUM_CORES = multiprocessing.cpu_count()

{dxpy-0.394.0 → dxpy-0.396.0}/dxpy/asset_builder.py

@@ -23,7 +23,6 @@ import tempfile
 import shutil
 import json
 
-from .compat import open
 from .exceptions import err_exit
 from .utils import json_load_raise_on_duplicates
 from .utils.resolver import is_container_id, resolve_path

{dxpy-0.394.0 → dxpy-0.396.0}/dxpy/bindings/dxdatabase.py

@@ -23,19 +23,16 @@ This remote database handler is a Python database-like object.
 
 from __future__ import print_function, unicode_literals, division, absolute_import
 
-import os, sys, logging, traceback, hashlib, copy, time
-import math
-import mmap
+import os, copy, time
+from io import BytesIO
 from threading import Lock
 from multiprocessing import cpu_count
 
 import dxpy
 from . import DXDataObject
-from ..exceptions import DXFileError, DXIncompleteReadsError
-from ..utils import warn
 from ..utils.resolver import object_exists_in_project
-from ..compat import BytesIO, basestring, USING_PYTHON2
 from .. import logger
+from ..compat import basestring
 
 
 DXFILE_HTTP_THREADS = min(cpu_count(), 8)

{dxpy-0.394.0 → dxpy-0.396.0}/dxpy/bindings/dxdatabase_functions.py

@@ -36,7 +36,7 @@ from .. import logger
 from . import dxfile, DXFile
 from . import dxdatabase, DXDatabase
 from .dxfile import FILE_REQUEST_TIMEOUT
-from ..compat import open, USING_PYTHON2, md5_hasher
+from ..compat import md5_hasher
 from ..exceptions import DXFileError, DXChecksumMismatchError, DXIncompleteReadsError, err_exit
 from ..utils import response_iterator
 import subprocess

{dxpy-0.394.0 → dxpy-0.396.0}/dxpy/bindings/dxfile.py

@@ -23,7 +23,8 @@ This remote file handler is a Python file-like object.
 
 from __future__ import print_function, unicode_literals, division, absolute_import
 
-import os, sys, logging, traceback, hashlib, copy, time
+import os, sys, traceback, copy, time
+from io import BytesIO
 import math
 import mmap
 from threading import Lock
@@ -34,7 +35,7 @@ from . import DXDataObject
 from ..exceptions import DXFileError, DXIncompleteReadsError
 from ..utils import warn
 from ..utils.resolver import object_exists_in_project
-from ..compat import BytesIO, basestring, USING_PYTHON2, md5_hasher
+from ..compat import md5_hasher, basestring
 
 
 DXFILE_HTTP_THREADS = min(cpu_count(), 8)
@@ -488,8 +489,7 @@ class DXFile(DXDataObject):
             does not affect where the next :meth:`write` will occur.
 
         '''
-        if not USING_PYTHON2:
-            assert(isinstance(data, bytes))
+        assert(isinstance(data, bytes))
 
         self._ensure_write_bufsize(**kwargs)
 
@@ -546,31 +546,28 @@ class DXFile(DXDataObject):
             does not affect where the next :meth:`write` will occur.
 
         '''
-        if USING_PYTHON2:
-            self._write2(data, multithread=multithread, **kwargs)
+        # In python3, the underlying system methods use the 'bytes' type, not 'string'
+        #
+        # This is, hopefully, a temporary hack. It is not a good idea for two reasons:
+        # 1) Performance, we need to make a pass on the data, and need to allocate
+        #    another buffer of similar size
+        # 2) The types are wrong. The "bytes" type should be visible to the caller
+        #    of the write method, instead of being hidden.
+
+        # Should we throw an exception if the file is opened in binary mode,
+        # and the data is unicode/text?
+        if isinstance(data, str):
+            bt = data.encode("utf-8")
+        elif isinstance(data, bytearray):
+            bt = bytes(data)
+        elif isinstance(data, bytes):
+            bt = data
+        elif isinstance(data, mmap.mmap):
+            bt = bytes(data)
         else:
-            # In python3, the underlying system methods use the 'bytes' type, not 'string'
-            #
-            # This is, hopefully, a temporary hack. It is not a good idea for two reasons:
-            # 1) Performance, we need to make a pass on the data, and need to allocate
-            #    another buffer of similar size
-            # 2) The types are wrong. The "bytes" type should be visible to the caller
-            #    of the write method, instead of being hidden.
-
-            # Should we throw an exception if the file is opened in binary mode,
-            # and the data is unicode/text?
-            if isinstance(data, str):
-                bt = data.encode("utf-8")
-            elif isinstance(data, bytearray):
-                bt = bytes(data)
-            elif isinstance(data, bytes):
-                bt = data
-            elif isinstance(data, mmap.mmap):
-                bt = bytes(data)
-            else:
-                raise DXFileError("Invalid type {} for write data argument".format(type(data)))
-            assert(isinstance(bt, bytes))
-            self._write2(bt, multithread=multithread, **kwargs)
+            raise DXFileError("Invalid type {} for write data argument".format(type(data)))
+        assert(isinstance(bt, bytes))
+        self._write2(bt, multithread=multithread, **kwargs)
 
     def closed(self, **kwargs):
         '''
@@ -606,10 +603,7 @@ class DXFile(DXDataObject):
             # settings allow last empty part upload, try to upload
             # an empty part (otherwise files with 0 parts cannot be closed).
             try:
-                if USING_PYTHON2:
-                    self.upload_part('', 1, **kwargs)
-                else:
-                    self.upload_part(b'', 1, **kwargs)
+                self.upload_part(b'', 1, **kwargs)
             except dxpy.exceptions.InvalidState:
                 pass
 
@@ -648,9 +642,7 @@ class DXFile(DXDataObject):
         defaults to 1. This probably only makes sense if this is the
         only part to be uploaded.
         """
-        if not USING_PYTHON2:
-            # In python3, the underlying system methods use the 'bytes' type, not 'string'
-            assert(isinstance(data, bytes))
+        assert(isinstance(data, bytes))
 
         req_input = {}
         if index is not None:

{dxpy-0.394.0 → dxpy-0.396.0}/dxpy/bindings/dxfile_functions.py

@@ -30,8 +30,6 @@ import traceback
 import warnings
 from collections import defaultdict
 import multiprocessing
-from random import randint
-from time import sleep
 import crc32c
 import zlib
 import base64
@@ -41,7 +39,7 @@ from .. import logger
 from . import dxfile, DXFile
 from .dxfile import FILE_REQUEST_TIMEOUT
 from ..exceptions import DXError, DXFileError, DXPartLengthMismatchError, DXChecksumMismatchError, DXIncompleteReadsError, err_exit
-from ..compat import open, md5_hasher, USING_PYTHON2
+from ..compat import md5_hasher
 from ..utils import response_iterator
 import subprocess
 import concurrent.futures

{dxpy-0.394.0 → dxpy-0.396.0}/dxpy/bindings/dxproject.py

@@ -367,7 +367,7 @@ class DXProject(DXContainer):
                allowed_executables=None, unset_allowed_executables=None,
                database_ui_view_only=None, external_upload_restricted=None,
                database_results_restricted=None, unset_database_results_restricted=None,
-               **kwargs):
+               https_app_isolated_browsing=None, **kwargs):
         """
         :param name: If provided, the new project name
         :type name: string
@@ -395,6 +395,9 @@ class DXProject(DXContainer):
         :type unset_database_results_restricted: boolean
         :param version: If provided, the update will only occur if the value matches the current project's version number
         :type version: int
+        :param https_app_isolated_browsing: Whether all https access to jobs in this project should be wrapped in Isolated Browsing.
+                                            If true, httpsApp-enabled executables must have httpsApp.shared_access set to 'NONE' to run in this project.
+        :type https_app_isolated_browsing: boolean
 
         Updates the project with the new fields. All fields are
         optional. Fields that are not provided are not changed.
@@ -430,6 +433,8 @@ class DXProject(DXContainer):
             update_hash["databaseResultsRestricted"] = database_results_restricted
         if unset_database_results_restricted is not None:
             update_hash["databaseResultsRestricted"] = None
+        if https_app_isolated_browsing is not None:
+            update_hash["httpsAppIsolatedBrowsing"] = https_app_isolated_browsing
         dxpy.api.project_update(self._dxid, update_hash, **kwargs)
 
     def invite(self, invitee, level, send_email=True, **kwargs):

{dxpy-0.394.0 → dxpy-0.396.0}/dxpy/cli/__init__.py

@@ -26,7 +26,6 @@ import sys
 INTERACTIVE_CLI = sys.stdin and sys.stdin.isatty() and sys.stdout and sys.stdout.isatty()
 
 from ..exceptions import err_exit, default_expected_exceptions, DXError
-from ..compat import input
 
 def try_call_err_exit():
     err_exit(expected_exceptions=default_expected_exceptions + (DXError,))

{dxpy-0.394.0 → dxpy-0.396.0}/dxpy/cli/dataset_utilities.py

@@ -36,13 +36,11 @@ from ..bindings.dxfile import DXFile
 from ..utils.resolver import resolve_existing_path, is_hashid, ResolutionError, resolve_path, check_folder_exists
 from ..utils.file_handle import as_handle
 from ..utils.describe import print_desc
-from ..compat import USING_PYTHON2
 from ..exceptions import (
     err_exit,
     PermissionDenied,
     InvalidInput,
     InvalidState,
-    ResourceNotFound,
     default_expected_exceptions,
 )
 
@@ -1633,10 +1631,7 @@ def validate_cohort_ids(descriptor, project, resp, ids):
     gpk_type = descriptor.model["entities"][entity_name]["fields"][field_name]["mapping"]["column_sql_type"]
     # Prepare a payload to find entries matching the input ids in the dataset
     if gpk_type in ["integer", "bigint"]:
-        if USING_PYTHON2:
-            lambda_for_list_conv = lambda a, b: a+[long(b)]
-        else:
-            lambda_for_list_conv = lambda a, b: a+[int(b)]
+        lambda_for_list_conv = lambda a, b: a+[int(b)]
     elif gpk_type in ["float", "double"]:
         lambda_for_list_conv = lambda a, b: a+[float(b)]
     elif gpk_type in ["string"]:

{dxpy-0.394.0 → dxpy-0.396.0}/dxpy/cli/exec_io.py

@@ -33,7 +33,7 @@ from ..utils.describe import (get_find_executions_string, get_ls_l_desc, get_ls_
 from ..utils.resolver import (parse_input_keyval, is_hashid, is_job_id, is_localjob_id, paginate_and_pick, pick,
                               resolve_existing_path, resolve_multiple_existing_paths, split_unescaped, is_analysis_id)
 from ..utils import OrderedDefaultdict
-from ..compat import input, str, shlex, basestring, USING_PYTHON2
+from ..compat import basestring
 try:
     # Import gnureadline if installed for macOS
     import gnureadline as readline
@@ -604,8 +604,6 @@ class ExecutableInputs(object):
                 try:
                     parsed_input_value = json.loads(input_value, object_pairs_hook=collections.OrderedDict)
                     immediate_types = {collections.OrderedDict, list, int, float}
-                    if USING_PYTHON2:
-                        immediate_types.add(long) # noqa
                     if type(parsed_input_value) not in immediate_types:
                         raise Exception()
                 except:

{dxpy-0.394.0 → dxpy-0.396.0}/dxpy/cli/org.py

@@ -20,7 +20,6 @@ the org-based commands of the dx command-line client.
 '''
 from __future__ import print_function, unicode_literals, division, absolute_import
 
-from ..compat import input
 import dxpy
 from . import try_call, prompt_for_yn, INTERACTIVE_CLI
 from .parsers import process_find_by_property_args, process_phi_param

{dxpy-0.394.0 → dxpy-0.396.0}/dxpy/cli/parsers.py

@@ -452,6 +452,8 @@ def get_update_project_args(args):
         input_params['databaseResultsRestricted'] = None
     if args.external_upload_restricted is not None:
         input_params['externalUploadRestricted'] = args.external_upload_restricted == 'true'
+    if args.https_app_isolated_browsing is not None:
+        input_params['httpsAppIsolatedBrowsing'] = args.https_app_isolated_browsing == 'true'
     return input_params
 
 def process_phi_param(args):

dxpy-0.396.0/dxpy/compat.py

@@ -0,0 +1,38 @@
+# Copyright (C) 2013-2016 DNAnexus, Inc.
+#
+# This file is part of dx-toolkit (DNAnexus platform client libraries).
+#
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may not
+#   use this file except in compliance with the License. You may obtain a copy
+#   of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+
+from __future__ import print_function, unicode_literals, division, absolute_import
+
+import sys, locale, threading, hashlib
+
+try:
+    sys_encoding = locale.getdefaultlocale()[1] or "UTF-8"
+except Exception:
+    sys_encoding = "UTF-8"
+
+USING_PYTHON2 = True if sys.version_info < (3, 0) else False
+
+basestring = (str, bytes)
+
+THREAD_TIMEOUT_MAX = threading.TIMEOUT_MAX
+
+# Support FIPS enabled Python
+def md5_hasher():
+    try:
+        md5_hasher = hashlib.new('md5', usedforsecurity=False)
+    except:
+        md5_hasher = hashlib.new('md5')
+    return md5_hasher

{dxpy-0.394.0 → dxpy-0.396.0}/dxpy/dxlog.py

@@ -21,7 +21,6 @@ import socket, json, time, os, logging
 from logging.handlers import SysLogHandler
 
 from dxpy.exceptions import DXError
-from dxpy.compat import USING_PYTHON2
 
 class DXLogHandler(SysLogHandler):
     '''
@@ -70,7 +69,7 @@ class DXLogHandler(SysLogHandler):
         return self.priority_names[self.priority_map.get(record.levelname, "warning")]
 
     def truncate_message(self, message):
-        msg_bytes = message if USING_PYTHON2 else message.encode('utf-8')
+        msg_bytes = message.encode('utf-8')
 
         if len(json.dumps(message)) <= 8015:
             return message
@@ -80,12 +79,9 @@ class DXLogHandler(SysLogHandler):
             msg_bytes = msg_bytes[:-1]
 
         message = _bytes2utf8(msg_bytes)
-        message = message.encode('utf-8') if USING_PYTHON2 else message
         return message + "... [truncated]"
 
     def is_resource_log(self, message):
-        if USING_PYTHON2:
-            return message.startswith(b"CPU: ")
         return message.startswith("CPU: ")
 
     def emit(self, record):
@@ -95,9 +91,6 @@ class DXLogHandler(SysLogHandler):
         # with the extra padding introduced by the log function, the
         # incoming message needs to be smaller - we truncate it to
         # at most 8015 bytes here.
-        # Note: we use Python 2 semantics here (byte strings). This
-        # script is not Python 3 ready. If *line* was a unicode string
-        # with wide chars, its byte length would exceed the limit.
         message = self.truncate_message(message)
 
         data = json.dumps({"source": self.source, "timestamp": int(round(time.time() * 1000)),

{dxpy-0.394.0 → dxpy-0.396.0}/dxpy/exceptions.py

@@ -28,7 +28,6 @@ import socket
 from urllib3.exceptions import HTTPError
 
 import dxpy
-from .compat import USING_PYTHON2
 import urllib3
 import ssl
 
@@ -246,10 +245,8 @@ network_exceptions = (urllib3.exceptions.ProtocolError,
                       urllib3.exceptions.SSLError,
                       ssl.SSLError,
                       HTTPError,
-                      socket.error)
-if not USING_PYTHON2:
-    network_exceptions += (ConnectionResetError,)
-
+                      socket.error,
+                      ConnectionResetError,)
 
 try:
     json_exceptions = (json.decoder.JSONDecodeError,)

dxpy-0.396.0/dxpy/nextflow/app_asset_projects_ids_prod.json

@@ -0,0 +1,9 @@
+{
+    "aws:ap-southeast-2": "project-F4gVXbj5qkxXq3v038V7qjxq",
+    "aws:eu-central-1": "project-F3zqG304yvb1GVkZ6987jP9F",
+    "aws:eu-west-2-g": "project-G4B9FBpKKgK6g9v13q423FXk",
+    "aws:me-south-1": "project-GvGXBYV3pK4v9X7F2jv9zz94",
+    "aws:us-east-1": "project-B6JG897KGbkGb6Z7pQ9Q02jG",
+    "azure:westeurope": "project-FGZZFkjBPJqBXq1X84pq4VJf",
+    "azure:westus": "project-F3vk1q09FX8Jxxp20pq6z8P9"
+}

dxpy-0.396.0/dxpy/nextflow/app_asset_projects_ids_staging.json

@@ -0,0 +1,9 @@
+{
+    "aws:ap-southeast-2": "project-G0zG49j5K4gjfv5V4FfqFK9X",
+    "aws:eu-central-1": "project-G0zG4F04KqpFYVz84FQkvF1P",
+    "aws:eu-west-2-g": "project-G4gyXkXK8BX9B0jp1V6029vJ",
+    "aws:me-south-1": "project-Gv8PVXk32gZ83kfp0pZ1v3V2",
+    "aws:us-east-1": "project-G0zG4B808VFFYVz84FQkvF1K",
+    "azure:westeurope": "project-G0zG4BQBq1XQby2F4FqQpPXv",
+    "azure:westus": "project-G0zG4B09ZJ5gx1Vz4Fb8Q9XX"
+}

dxpy-0.396.0/dxpy/nextflow/nextaur_assets.json

@@ -0,0 +1,9 @@
+{
+    "aws:ap-southeast-2": "record-GzyP89850x72260z2yYyZQfJ",
+    "aws:eu-central-1": "record-GzyP7584vB9Yp0j7qfv5F134",
+    "aws:eu-west-2-g": "record-GzyP80XKPQ6K81G8XV89fyvZ",
+    "aws:me-south-1": "record-GzyP85V306B9xkQ4Pv1yf7Kz",
+    "aws:us-east-1": "record-GzyP6580KGjqV4gp6QXFGBzX",
+    "azure:westeurope": "record-GzyP9z0BzJx8zjf6bpvY6Yqv",
+    "azure:westus": "record-GzyP9vQ9zggPyfkj5GKKYVG5"
+}

dxpy-0.396.0/dxpy/nextflow/nextaur_assets.staging.json

@@ -0,0 +1,9 @@
+{
+    "aws:ap-southeast-2": "record-GzyJ2F0506kK1651KKg3K1Xq",
+    "aws:eu-central-1": "record-GzyJ2Yj4f2xpzzypx8G484FB",
+    "aws:eu-west-2-g": "record-GzyJ28XKKGJ7758Yj62X97bV",
+    "aws:me-south-1": "record-GzyJ2GV311jBJYzx2XpgZXQ0",
+    "aws:us-east-1": "record-GzyJ08002j9B36YfkbQQxpkf",
+    "azure:westeurope": "record-GzyJ4B8B095jZ8F4BF5p2Ggz",
+    "azure:westus": "record-GzyJ3q89X1QBpvfvgx9bZzgP"
+}

dxpy-0.396.0/dxpy/nextflow/nextflow_assets.json

@@ -0,0 +1,9 @@
+{
+    "aws:ap-southeast-2": "record-GzyP8B05q2Y748ZfZXQ4139j",
+    "aws:eu-central-1": "record-GzyP7FQ4p6pX7q2Qbx06zX9v",
+    "aws:eu-west-2-g": "record-GzyP8FXKkjkqfjBjjZzFv68F",
+    "aws:me-south-1": "record-GzyP8593PgfZjYXPJP9FFqV4",
+    "aws:us-east-1": "record-GzyP6X00F3gq8p88gP7PQFkx",
+    "azure:westeurope": "record-GzyPB98BB2b867v2KK1BXx8p",
+    "azure:westus": "record-GzyPB6j9b5Yy81G8XV89jkZX"
+}

dxpy-0.396.0/dxpy/nextflow/nextflow_assets.staging.json

@@ -0,0 +1,9 @@
+{
+    "aws:ap-southeast-2": "record-GzyJ2Y05x444p61z5XKFP54Q",
+    "aws:eu-central-1": "record-GzyJ2xQ4Pffppvfvgx9bZzg6",
+    "aws:eu-west-2-g": "record-GzyJ2Z2KppXG1ZXvQ2kp08YQ",
+    "aws:me-south-1": "record-GzyJ2Y93z3b2Q4202Jpx1j78",
+    "aws:us-east-1": "record-GzyJ19j08Ffppvfvgx9bZzfz",
+    "azure:westeurope": "record-GzyJ4QjByJz29GJg3K3Gv2yB",
+    "azure:westus": "record-GzyJ45Q9xQPB36YfkbQQxpp8"
+}

{dxpy-0.394.0 → dxpy-0.396.0}/dxpy/nextflow/nextflow_templates.py

@@ -5,7 +5,6 @@ from .nextflow_utils import (get_template_dir, get_source_file_name, get_resourc
 import json
 import os
 from dxpy import TOOLKIT_VERSION
-from dxpy.compat import USING_PYTHON2, sys_encoding
 
 
 def get_nextflow_dxapp(
@@ -94,11 +93,7 @@ def get_nextflow_src(custom_inputs=None, profile=None, resources_dir=None):
     src = src.replace("@@PROFILE_ARG@@", profile_arg)
     src = src.replace("@@EXCLUDE_INPUT_DOWNLOAD@@", exclude_input_download)
     src = src.replace("@@DXPY_BUILD_VERSION@@", TOOLKIT_VERSION)
-    if USING_PYTHON2:
-        src = src.replace("@@RESOURCES_SUBPATH@@",
-                          get_resources_subpath(resources_dir).encode(sys_encoding))
-    else:
-        src = src.replace("@@RESOURCES_SUBPATH@@",
+    src = src.replace("@@RESOURCES_SUBPATH@@",
                           get_resources_subpath(resources_dir))
 
     return src

{dxpy-0.394.0 → dxpy-0.396.0}/dxpy/nextflow/nextflow_utils.py

@@ -5,6 +5,7 @@ import errno
 import dxpy
 import json
 import shutil
+import logging
 from dxpy.exceptions import ResourceNotFound
 from dxpy.nextflow.collect_images import run_nextaur_collect, bundle_docker_images
 
@@ -37,6 +38,22 @@ def get_template_dir():
     return path.join(path.dirname(dxpy.__file__), 'templating', 'templates', 'nextflow')
 
 
+def get_project_with_assets(region):
+    nextflow_basepath = path.join(path.dirname(dxpy.__file__), 'nextflow')
+    projects_path = path.join(nextflow_basepath, "app_asset_projects_ids_prod.json")
+
+    try:
+        with open(projects_path, 'r') as projects_f:
+            project = json.load(projects_f)[region]
+            dxpy.describe(project, fields={})  # existence check
+    except ResourceNotFound:
+        projects_path = path.join(nextflow_basepath, "app_asset_projects_ids_staging.json")
+        with open(projects_path, 'r') as projects_f:
+            project = json.load(projects_f)[region]
+
+    return project
+
+
 def is_importer_job():
     try:
         with open("/home/dnanexus/dnanexus-job.json", "r") as f:
@@ -102,6 +119,8 @@ def get_regional_options(region, resources_dir, profile, cache_docker, nextflow_
         image_bundled = bundle_docker_images(image_refs)
     else:
         image_bundled = {}
+
+    project_with_assets = get_project_with_assets(region)
     regional_options = {
         region: {
             "systemRequirements": {
@@ -110,9 +129,18 @@ def get_regional_options(region, resources_dir, profile, cache_docker, nextflow_
                 }
             },
             "assetDepends": [
-                {"id": nextaur_asset},
-                {"id": nextflow_asset},
-                {"id": awscli_asset}
+                {"id": {"$dnanexus_link": {
+                    "id": nextaur_asset,
+                    "project": project_with_assets
+                }}},
+                {"id": {"$dnanexus_link": {
+                    "id": nextflow_asset,
+                    "project": project_with_assets
+                }}},
+                {"id": {"$dnanexus_link": {
+                    "id": awscli_asset,
+                    "project": project_with_assets
+                }}}
             ],
             "bundledDepends": image_bundled
         }