dxpy 0.364.0__tar.gz → 0.366.0__tar.gz

{dxpy-0.364.0 → dxpy-0.366.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dxpy
-Version: 0.364.0
+Version: 0.366.0
 Summary: DNAnexus Platform API bindings for Python
 Home-page: https://github.com/dnanexus/dx-toolkit
 Author: Aleksandra Zalcman, Andrey Kislyuk, Anurag Biyani, Geet Duggal, Katherine Lai, Kurt Jensen, Marek Hrvol, Ohad Rodeh, Phil Sung
@@ -14,7 +14,7 @@ Classifier: Operating System :: POSIX
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Unix Shell
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
-Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*
+Requires-Python: >=3.6
 Description-Content-Type: text/markdown
 Provides-Extra: pandas
 Provides-Extra: xattr
@@ -77,7 +77,7 @@ Other useful resources:
 
 Python version compatibility
 ----------------------------
-dxpy is supported on Python 3 (3.5+)
+dxpy is supported on Python 3 (3.6+)
 
 Convention for Python scripts that are also modules
 ---------------------------------------------------

{dxpy-0.364.0 → dxpy-0.366.0}/Readme.md

@@ -56,7 +56,7 @@ Other useful resources:
 
 Python version compatibility
 ----------------------------
-dxpy is supported on Python 3 (3.5+)
+dxpy is supported on Python 3 (3.6+)
 
 Convention for Python scripts that are also modules
 ---------------------------------------------------

{dxpy-0.364.0 → dxpy-0.366.0}/dxpy/__init__.py

@@ -20,8 +20,7 @@ the following sources in order of decreasing priority:
 
 1. Environment variables
 2. Values stored in ``~/.dnanexus_config/environment``
-3. Values stored in ``/opt/dnanexus/environment``
-4. Hardcoded defaults
+3. Hardcoded defaults
 
 The bindings are configured by the following environment variables:
 
@@ -133,30 +132,19 @@ logger.addHandler(logging.NullHandler())
 import os, sys, json, time, platform, ssl, traceback
 import errno
 import math
-import mmap
-import requests
 import socket
 import threading
-import subprocess
-
+import certifi
 from collections import namedtuple
 
 from . import exceptions
-from .compat import USING_PYTHON2, BadStatusLine, StringIO, bytes, Repr
+from .compat import BadStatusLine, StringIO, bytes, Repr
 from .utils.printing import BOLD, BLUE, YELLOW, GREEN, RED, WHITE
 
 from random import randint
-from requests.auth import AuthBase
-from requests.packages import urllib3
+import urllib3
 from threading import Lock
-from . import ssh_tunnel_app_support
-
-try:
-    # python-3
-    from urllib.parse import urlsplit
-except ImportError:
-    # python-2
-    from urlparse import urlsplit
+from urllib.parse import urlsplit
 
 sequence_number_mutex = threading.Lock()
 counter = 0
@@ -172,7 +160,7 @@ def _get_sequence_number():
 def configure_urllib3():
     # Disable verbose urllib3 warnings and log messages
     urllib3.disable_warnings(category=urllib3.exceptions.InsecurePlatformWarning)
-    logging.getLogger('dxpy.packages.requests.packages.urllib3.connectionpool').setLevel(logging.ERROR)
+    logging.getLogger('urllib3.connectionpool').setLevel(logging.ERROR)
 
 configure_urllib3()
 
@@ -203,8 +191,13 @@ USER_AGENT = "{name}/{version} ({platform}) Python/{python_version}".format(name
                                                     version=TOOLKIT_VERSION,
                                                     platform=platform.platform(),
                                                     python_version=platform.python_version())
-_default_certs = requests.certs.where()
-_default_headers = requests.utils.default_headers()
+_default_certs = certifi.where()
+_default_headers = {
+            "User-Agent": USER_AGENT,
+            "Accept-Encoding": "gzip, deflate",
+            "Accept": "*/*",
+            "Connection": "keep-alive",
+        }
 _default_timeout = urllib3.util.timeout.Timeout(connect=DEFAULT_TIMEOUT, read=DEFAULT_TIMEOUT)
 _RequestForAuth = namedtuple('_RequestForAuth', 'method url headers')
 _expected_exceptions = (exceptions.network_exceptions, exceptions.DXAPIError, BadStatusLine, exceptions.BadJSONInReply,
@@ -277,7 +270,7 @@ def _get_pool_manager(verify, cert_file, key_file, ssl_context=None):
                          cert_file=cert_file,
                          key_file=key_file,
                          ssl_context=ssl_context,
-                         ca_certs=verify or os.environ.get('DX_CA_CERT') or requests.certs.where())
+                         ca_certs=verify or os.environ.get('DX_CA_CERT') or certifi.where())
         if verify is False or os.environ.get('DX_CA_CERT') == 'NOVERIFY':
             pool_args.update(cert_reqs=ssl.CERT_NONE, ca_certs=None)
             urllib3.disable_warnings()
@@ -295,15 +288,7 @@ def _process_method_url_headers(method, url, headers):
         _headers.update(headers)
     else:
         _url, _headers = url, headers
-    # When *data* is bytes but *headers* contains Unicode text, httplib tries to concatenate them and decode
-    # *data*, which should not be done. Also, per HTTP/1.1 headers must be encoded with MIME, but we'll
-    # disregard that here, and just encode them with the Python default (ascii) and fail for any non-ascii
-    # content. See http://tools.ietf.org/html/rfc3987 for a discussion of encoding URLs.
-    # TODO: ascertain whether this is a problem in Python 3/make test
-    if USING_PYTHON2:
-        return method.encode(), _url.encode('utf-8'), {k.encode(): v.encode() for k, v in _headers.items()}
-    else:
-        return method, _url, _headers
+    return method, _url, _headers
 
 
 # When any of the following errors are indicated, we are sure that the
@@ -316,9 +301,7 @@ _RETRYABLE_SOCKET_ERRORS = {
 }
 
 _RETRYABLE_WITH_RESPONSE = (exceptions.ContentLengthError, BadStatusLine, exceptions.BadJSONInReply,
-                                         urllib3.exceptions.ProtocolError, exceptions.UrllibInternalError)
-if not USING_PYTHON2:
-    _RETRYABLE_WITH_RESPONSE += (ConnectionResetError,)
+                            ConnectionResetError, urllib3.exceptions.ProtocolError, exceptions.UrllibInternalError)
 
 def _is_retryable_exception(e):
     """Returns True if the exception is always safe to retry.
@@ -333,7 +316,7 @@ def _is_retryable_exception(e):
     """
     if isinstance(e, urllib3.exceptions.ProtocolError):
         return True
-    if not USING_PYTHON2 and isinstance(e, ConnectionResetError):
+    if isinstance(e, ConnectionResetError):
         return True
     if isinstance(e, (socket.gaierror, socket.herror)):
         return True
@@ -341,8 +324,6 @@ def _is_retryable_exception(e):
         return True
     if isinstance(e, urllib3.exceptions.NewConnectionError):
         return True
-    if isinstance(e, requests.exceptions.SSLError):
-        return True
     if isinstance(e, urllib3.exceptions.SSLError):
         return True
     if isinstance(e, ssl.SSLError):
@@ -496,13 +477,11 @@ def DXHTTPRequest(resource, data, method='POST', headers=None, auth=True,
     :type auth: tuple, object, True (default), or None
     :param timeout: HTTP request timeout, in seconds
     :type timeout: float
-    :param config: *config* value to pass through to :meth:`requests.request`
-    :type config: dict
     :param use_compression: Deprecated
     :type use_compression: string or None
     :param jsonify_data: If True, *data* is converted from a Python list or dict to a JSON string
     :type jsonify_data: boolean
-    :param want_full_response: If True, the full :class:`requests.Response` object is returned (otherwise, only the content of the response body is returned)
+    :param want_full_response: If True, the full :class:`urllib3.response.HTTPResponse` object is returned (otherwise, only the content of the response body is returned)
     :type want_full_response: boolean
     :param decode_response_body: If True (and *want_full_response* is False), the response body is decoded and, if it is a JSON string, deserialized. Otherwise, the response body is uncompressed if transport compression is on, and returned raw.
     :type decode_response_body: boolean
@@ -523,9 +502,9 @@ def DXHTTPRequest(resource, data, method='POST', headers=None, auth=True,
 
     :type always_retry: boolean
     :returns: Response from API server in the format indicated by *want_full_response* and *decode_response_body*.
-    :raises: :exc:`exceptions.DXAPIError` or a subclass if the server returned a non-200 status code; :exc:`requests.exceptions.HTTPError` if an invalid response was received from the server; or :exc:`requests.exceptions.ConnectionError` if a connection cannot be established.
+    :raises: :exc:`exceptions.DXAPIError` or a subclass if the server returned a non-200 status code; :exc:`urllib3.exceptions.HTTPError` if an invalid response was received from the server; or :exc:`urllib3.exceptions.ConnectionError` if a connection cannot be established.
 
-    Wrapper around :meth:`requests.request()` that makes an HTTP
+    Wrapper around :meth:`urllib3.request()` that makes an HTTP
     request, inserting authentication headers and (by default)
     converting *data* to JSON.
 
@@ -607,28 +586,26 @@ def DXHTTPRequest(resource, data, method='POST', headers=None, auth=True,
                     return i
 
                 _headers = {ensure_ascii(k): ensure_ascii(v) for k, v in _headers.items()}
-                if USING_PYTHON2:
+
+                # This is needed for python 3 urllib
+                _headers.pop(b'host', None)
+                _headers.pop(b'content-length', None)
+                _headers.pop(b'Content-Length', None)
+
+                # The libraries downstream (http client) require elimination of non-ascii
+                # chars from URL.
+                # We check if the URL contains non-ascii characters to see if we need to
+                # quote it. It is important not to always quote the path (here: parts[2])
+                # since it might contain elements (e.g. HMAC for api proxy) containing
+                # special characters that should not be quoted.
+                try:
+                    ensure_ascii(_url)
                     encoded_url = _url
-                else:
-                    # This is needed for python 3 urllib
-                    _headers.pop(b'host', None)
-                    _headers.pop(b'content-length', None)
-                    _headers.pop(b'Content-Length', None)
-
-                    # The libraries downstream (http client) require elimination of non-ascii
-                    # chars from URL.
-                    # We check if the URL contains non-ascii characters to see if we need to
-                    # quote it. It is important not to always quote the path (here: parts[2])
-                    # since it might contain elements (e.g. HMAC for api proxy) containing
-                    # special characters that should not be quoted.
-                    try:
-                        ensure_ascii(_url)
-                        encoded_url = _url
-                    except UnicodeEncodeError:
-                        import urllib.parse
-                        parts = list(urllib.parse.urlparse(_url))
-                        parts[2] = urllib.parse.quote(parts[2])
-                        encoded_url = urllib.parse.urlunparse(parts)
+                except UnicodeEncodeError:
+                    import urllib.parse
+                    parts = list(urllib.parse.urlparse(_url))
+                    parts[2] = urllib.parse.quote(parts[2])
+                    encoded_url = urllib.parse.urlunparse(parts)
 
                 response = pool_manager.request(_method, encoded_url, headers=_headers, body=body,
                                                 timeout=timeout, retries=False, **kwargs)
@@ -750,7 +727,7 @@ def DXHTTPRequest(resource, data, method='POST', headers=None, auth=True,
                     # The server has closed the connection prematurely
                     if (response is not None
                        and response.status == 400 and is_retryable and method == 'PUT'
-                       and isinstance(e, requests.exceptions.HTTPError)):
+                       and isinstance(e, urllib3.exceptions.HTTPError)):
                         request_timeout_str = '<Code>RequestTimeout</Code>'
                         if (request_timeout_str in exception_msg
                             or (isinstance(e, exceptions.HTTPErrorWithContent) and request_timeout_str in e.content)):
@@ -810,7 +787,7 @@ def DXHTTPRequest(resource, data, method='POST', headers=None, auth=True,
     raise AssertionError('Should never reach this line: should never break out of loop')
 
 
-class DXHTTPOAuth2(AuthBase):
+class DXHTTPOAuth2():
     def __init__(self, security_context):
         self.security_context = security_context
 

{dxpy-0.364.0 → dxpy-0.366.0}/dxpy/bindings/dxfile.py

@@ -658,7 +658,7 @@ class DXFile(DXDataObject):
         :type display_progress: boolean
         :param report_progress_fn: Optional: a function to call that takes in two arguments (self, # bytes transmitted)
         :type report_progress_fn: function or None
-        :raises: :exc:`dxpy.exceptions.DXFileError` if *index* is given and is not in the correct range, :exc:`requests.exceptions.HTTPError` if upload fails
+        :raises: :exc:`dxpy.exceptions.DXFileError` if *index* is given and is not in the correct range, :exc:`urllib3.exceptions.HTTPError` if upload fails
 
         Uploads the data in *data* as part number *index* for the
         associated file. If no value for *index* is given, *index*

{dxpy-0.364.0 → dxpy-0.366.0}/dxpy/bindings/search.py

@@ -163,7 +163,7 @@ def find_data_objects(classname=None, state=None, visibility=None,
     :type level: string
     :param region: Filter on result set by the given region(s).
     :type region: string or list of strings
-    :param archival_state: Filter by the given archival state (one of "archived", "live", "archival", “unarchiving”, or "any"). Requires classname="file", project, and folder arguments to be provided.
+    :param archival_state: Filter by the given archival state (one of "archived", "live", "archival", "unarchiving", or "any"). Requires classname="file", project, and folder arguments to be provided.
     :type archival_state: string 
     :param limit: The maximum number of results to be returned (if not specified, the number of results is unlimited)
     :type limit: int

{dxpy-0.364.0 → dxpy-0.366.0}/dxpy/cli/cp.py

@@ -24,9 +24,8 @@ command-line client.
 from __future__ import print_function, unicode_literals, division, absolute_import
 
 import dxpy
-import requests
 from ..utils.resolver import (resolve_existing_path, resolve_path, is_hashid, get_last_pos_of_char)
-from ..exceptions import (err_exit, DXCLIError)
+from ..exceptions import (err_exit, DXCLIError, ResourceNotFound)
 from . import try_call
 from dxpy.utils.printing import (fill)
 
@@ -47,8 +46,8 @@ def cp_to_noexistent_destination(args, dest_path, dx_dest, dest_proj):
     dest_name = dest_path[last_slash_pos + 1:].replace('\/', '/')
     try:
         dx_dest.list_folder(folder=dest_folder, only='folders')
-    except dxpy.DXAPIError as details:
-        if details.code == requests.codes['not_found']:
+    except dxpy.DXAPIError as e:
+        if isinstance(e, ResourceNotFound):
             raise DXCLIError('The destination folder does not exist')
         else:
             raise

{dxpy-0.364.0 → dxpy-0.366.0}/dxpy/cli/dataset_utilities.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2022 DNAnexus, Inc.

{dxpy-0.364.0 → dxpy-0.366.0}/dxpy/exceptions.py

@@ -25,8 +25,7 @@ import json
 import traceback
 import errno
 import socket
-import requests
-from requests.exceptions import HTTPError
+from urllib3.exceptions import HTTPError
 
 import dxpy
 from .compat import USING_PYTHON2
@@ -238,12 +237,12 @@ def exit_with_exc_info(code=1, message='', print_tb=False, exception=None):
         sys.stderr.write('\n')
     sys.exit(code)
 
-network_exceptions = (requests.packages.urllib3.exceptions.ProtocolError,
-                      requests.packages.urllib3.exceptions.NewConnectionError,
-                      requests.packages.urllib3.exceptions.DecodeError,
-                      requests.packages.urllib3.exceptions.ConnectTimeoutError,
-                      requests.packages.urllib3.exceptions.ReadTimeoutError,
-                      requests.packages.urllib3.connectionpool.HTTPException,
+network_exceptions = (urllib3.exceptions.ProtocolError,
+                      urllib3.exceptions.NewConnectionError,
+                      urllib3.exceptions.DecodeError,
+                      urllib3.exceptions.ConnectTimeoutError,
+                      urllib3.exceptions.ReadTimeoutError,
+                      urllib3.connectionpool.HTTPException,
                       urllib3.exceptions.SSLError,
                       ssl.SSLError,
                       HTTPError,

{dxpy-0.364.0 → dxpy-0.366.0}/dxpy/executable_builder.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright (C) 2013-2016 DNAnexus, Inc.
 #

dxpy-0.366.0/dxpy/nextflow/ImageRef.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python
+
+import os
+import subprocess
+
+from dxpy import DXFile, config, upload_local_file
+from dxpy.exceptions import err_exit
+
+
+class ImageRef(object):
+    def __init__(
+            self,
+            process,
+            digest,
+            dx_file_id=None,
+            repository=None,
+            image_name=None,
+            tag=None
+    ):
+        """
+        A class to handle an image reference from nextflow pipeline.
+        :param process: An NPA proces name (aka task name) which uses a given image
+        :type process: String
+        :param digest: An image digest
+        :type digest: String
+        :param dx_file_id: dx file id on the platform
+        :type dx_file_id: Optional[String]
+        :param repository: Image repository
+        :type repository: Optional[String]
+        :param image_name: Image name (usually a basename of the image referenced with repository)
+        :type image_name: Optional[String]
+        :param tag: A version tag
+        :type tag: Optional[String]
+        """
+        self._caching_dir = os.path.join("/.cached_docker_images/", image_name or "")
+        self._dx_file_id = dx_file_id
+        self._bundled_depends = None
+        self._repository = repository
+        self._image_name = image_name
+        self._tag = tag
+        self._digest = digest
+        self._process = process
+
+    @property
+    def bundled_depends(self):
+        if not self._bundled_depends:
+            self._bundled_depends = self._package_bundle()
+        return self._bundled_depends
+
+    @property
+    def identifier(self):
+        return self._join_if_exists("_", [self._repository, self._image_name, self._tag, self._digest])
+
+    def _cache(self, file_name):
+        """
+        Function to store an image on the platform as a dx file object. Should be implemented in subclasses.
+        :param file_name: A file name under which the image will be saved on the platform
+        :type file_name: String
+        :returns: Tuple[String, String] dx file id, file name (basename)
+        """
+        raise NotImplementedError("Abstract class. Method not implemented. Use the concrete implementations.")
+
+    def _reconstruct_image_ref(self):
+        raise NotImplementedError("Abstract class. Method not implemented. Use the concrete implementations.")
+
+    def _construct_cache_file_name(self):
+        return self._join_if_exists("_", [self._image_name, self._tag])
+
+    @staticmethod
+    def _join_if_exists(delimiter, parts):
+        return delimiter.join([x for x in parts if x])
+
+    def _dx_file_get_name(self):
+        dx_file_handle = DXFile(self._dx_file_id, config["DX_PROJECT_CONTEXT_ID"])
+        return dx_file_handle.describe().get("name")
+
+    def _package_bundle(self):
+        """
+        Function to include a container image stored on the platform into NPA
+        :returns: Dict in the format of {"name": "bundle.tar.gz", "id": {"$dnanexus_link": "file-xxxx"}}
+        """
+        if not self._dx_file_id:
+            cache_file_name = self._construct_cache_file_name()
+            self._dx_file_id = self._cache(cache_file_name)
+        else:
+            cache_file_name = self._dx_file_get_name()
+        return {
+            "name": cache_file_name,
+            "id": {"$dnanexus_link": self._dx_file_id}
+        }
+
+
+class DockerImageRef(ImageRef):
+    def __init__(
+            self,
+            process,
+            digest,
+            dx_file_id=None,
+            repository=None,
+            image_name=None,
+            tag=None
+    ):
+        super().__init__(
+            process,
+            digest,
+            dx_file_id,
+            repository,
+            image_name,
+            tag)
+
+    def _cache(self, file_name):
+        full_image_ref = self._reconstruct_image_ref()
+        docker_pull_cmd = "docker pull {}".format(full_image_ref)
+        docker_save_cmd = "docker save {} | gzip > {}".format(full_image_ref, file_name)
+        for cmd in [docker_pull_cmd, docker_save_cmd]:
+            try:
+                _ = subprocess.check_output(cmd, shell=True)
+            except subprocess.CalledProcessError:
+                err_exit("Failed to run a subprocess command: {}".format(cmd))
+        # may need wait_on_close = True??
+        uploaded_dx_file = upload_local_file(
+            filename=file_name,
+            project=config["DX_PROJECT_CONTEXT_ID"],
+            folder=self._caching_dir,
+            name=file_name,
+            parents=True,
+            properties={"image_digest": self._digest}
+        )
+        return uploaded_dx_file.get_id()
+
+    def _reconstruct_image_ref(self):
+        """
+        Docker image reference has the form of <REPOSITORY_NAME>/<IMAGE_NAME>:<VERSION_TAG> or
+        <REPOSITORY_NAME>/<IMAGE_NAME>@<DIGEST>
+        """
+        repo_and_image_name = self._join_if_exists("/", [self._repository, self._image_name])
+        if self._digest:
+            full_ref = self._join_if_exists("@", [repo_and_image_name, self._digest])
+        else:
+            full_ref = self._join_if_exists(":", [repo_and_image_name, self._tag])
+        return full_ref

dxpy-0.366.0/dxpy/nextflow/ImageRefFactory.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+
+from dxpy.nextflow.ImageRef import DockerImageRef
+
+
+class ImageRefFactoryError(Exception):
+    """
+    Class to handle errors with instantiation of ImageRef subclasses
+    """
+
+
+class ImageRefFactory(object):
+    def __init__(
+            self,
+            image_ref
+    ):
+        """
+        A class to instantiate subclasses of ImageRef based on the container engine. Ususally instantiated after using the
+        nextaur:collect function to collect images for docker and other container engines.
+        :param image_ref: Image ref details
+        :type image_ref: Dict
+        """
+        self._image_ref = image_ref
+        self._engine = image_ref.get("engine", None)
+        if not self._engine:
+            raise ImageRefFactoryError("Provide the container engine")
+        self._imageRef_switch = {
+            "docker": DockerImageRef
+        }
+
+    def get_image(self):
+        image = self._imageRef_switch.get(self._engine, None)
+        if not image:
+            raise ImageRefFactoryError("Unsupported container engine: {}".format(self._engine))
+        return image(
+            process=self._image_ref["process"],
+            digest=self._image_ref["digest"],
+            dx_file_id=self._image_ref.get("dx_file_id", None),
+            repository=self._image_ref.get("repository", None),
+            image_name=self._image_ref.get("image_name", None),
+            tag=self._image_ref.get("tag", None)
+        )
+

dxpy-0.366.0/dxpy/nextflow/collect_images.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013-2016 DNAnexus, Inc.
+#
+# This file is part of dx-toolkit (DNAnexus platform client libraries).
+#
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may not
+#   use this file except in compliance with the License. You may obtain a copy
+#   of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+
+import json
+import subprocess
+from dxpy.nextflow.ImageRefFactory import ImageRefFactory, ImageRefFactoryError
+
+CONTAINERS_JSON = "containers.json"
+
+
+def bundle_docker_images(image_refs):
+    """
+    :param image_refs: Image references extracted from run_nextaur_collect().
+    :type image_refs: Dict
+    :returns: Array of dicts for bundledDepends attribute of the applet resources. Also saves images on the platform
+    if not done that before.
+    """
+    image_factories = [ImageRefFactory(x) for x in image_refs]
+    images = [x.get_image() for x in image_factories]
+    seen_images = set()
+    bundled_depends = []
+    for image in images:
+        if image.identifier in seen_images:
+            continue
+        else:
+            bundled_depends.append(image.bundled_depends.copy())
+            seen_images.add(image.identifier)
+    return bundled_depends
+
+
+def run_nextaur_collect(resources_dir):
+    """
+        :param resources_dir: URL to the local(ized) NF pipeline in the app(let) resources.
+        :type resources_dir: String
+        :returns: Dict. Image references in the form of
+            "process": String. Name of the process/task
+            "repository": String. Repository (host) prefix
+            "image_name": String. Image base name
+            "tag": String. Version tag
+            "digest": String. Image digest
+            "file_id": String. File ID if found on the platform
+            "engine": String. Container engine.
+        Runs nextaur:collect
+        """
+    collect_cmd = "nextflow plugin nextaur:collect docker {}".format(resources_dir)
+    _ = subprocess.check_output(collect_cmd, shell=True)
+    with open(CONTAINERS_JSON, "r") as json_file:
+        image_refs = json.load(json_file).get("processes", None)
+        if not image_refs:
+            raise ImageRefFactoryError("Could not extract processes from nextaur:collect")
+    return image_refs
+

dxpy-0.366.0/dxpy/nextflow/nextaur_assets.staging.json

@@ -0,0 +1,8 @@
+{
+    "aws:ap-southeast-2": "record-Gb53JYj5Gxk7z6xggK6qxq1K",
+    "aws:eu-central-1": "record-Gb53PJj4Q67q34JzbxK3G51g",
+    "aws:eu-west-2-g": "record-Gb53Qx2KjP0JVbQ0xVZxZYq2",
+    "aws:us-east-1": "record-Gb53Fk00VYYyFBpQjxy65jYY",
+    "azure:westeurope": "record-Gb53YF0B54bb886BZj9jv16v",
+    "azure:westus": "record-Gb53bkj9Yk85kQjfJZJKJPQ6"
+}

dxpy-0.366.0/dxpy/nextflow/nextflow_assets.staging.json

@@ -0,0 +1,8 @@
+{
+    "aws:ap-southeast-2": "record-GZvYK005pk0QG4FxZXqqgjKP",
+    "aws:eu-central-1": "record-GZvYPX04y9B66jXxk28qG7F0",
+    "aws:eu-west-2-g": "record-GZvYQgpKB7Y8pQJY8KbZF7kq",
+    "aws:us-east-1": "record-GZvY8bQ0057qKVy7G5G071jB",
+    "azure:westeurope": "record-GZvYYPjBF86q711V5VjGzFQ0",
+    "azure:westus": "record-GZvYfB09gx8gZpf049jPJQYg"
+}

{dxpy-0.364.0 → dxpy-0.366.0}/dxpy/nextflow/nextflow_builder.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 import os
 import dxpy
 import json
@@ -7,8 +7,7 @@ from glob import glob
 import tempfile
 
 from dxpy.nextflow.nextflow_templates import (get_nextflow_dxapp, get_nextflow_src)
-from dxpy.nextflow.nextflow_utils import (get_template_dir, write_exec, write_dxapp, get_importer_name,
-                                          get_resources_dir_name, create_readme)
+from dxpy.nextflow.nextflow_utils import (get_template_dir, write_exec, write_dxapp, get_importer_name, create_readme)
 from dxpy.cli.exec_io import parse_obj
 from dxpy.cli import try_call
 from dxpy.utils.resolver import resolve_existing_path
@@ -99,7 +98,7 @@ def prepare_nextflow(resources_dir, profile, region):
     dxapp_dir = tempfile.mkdtemp(prefix=".dx.nextflow")
 
     custom_inputs = prepare_custom_inputs(schema_file=os.path.join(resources_dir, "nextflow_schema.json"))
-    dxapp_content = get_nextflow_dxapp(custom_inputs=custom_inputs, name=get_resources_dir_name(resources_dir),
+    dxapp_content = get_nextflow_dxapp(custom_inputs=custom_inputs, resources_dir=resources_dir,
                                        region=region)
     exec_content = get_nextflow_src(custom_inputs=custom_inputs, profile=profile, resources_dir=resources_dir)
     copy_tree(get_template_dir(), dxapp_dir)