toil 8.1.0b1__py3-none-any.whl → 8.2.0__py3-none-any.whl

toil/lib/docker.py

@@ -11,26 +11,40 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from collections.abc import Generator
 import base64
 import logging
 import os
 import re
 import struct
 from shlex import quote
-from typing import Optional
+from typing import (
+    cast,
+    overload,
+    Any,
+    Literal,
+    NoReturn,
+    Optional,
+    Union,
+    TYPE_CHECKING,
+)
 
 import requests
 
 import docker
-from docker.errors import (
+from docker.errors import (  # type: ignore[import-not-found]
     ContainerError,
     ImageNotFound,
     NotFound,
     create_api_error_from_http_exception,
 )
-from docker.utils.socket import consume_socket_output, demux_adaptor
+from docker.utils.socket import consume_socket_output, demux_adaptor  # type: ignore[import-not-found]
 from toil.lib.accelerators import get_host_accelerator_numbers
 
+if TYPE_CHECKING:
+    from docker.models.containers import Container  # type: ignore[import-not-found]
+    from toil.job import Job
+
 logger = logging.getLogger(__name__)
 
 FORGO = 0
@@ -38,50 +52,193 @@ STOP = 1
 RM = 2
 
 
-def dockerCheckOutput(*args, **kwargs):
+def dockerCheckOutput(*args: Any, **kwargs: Any) -> NoReturn:
     raise RuntimeError(
         "dockerCheckOutput() using subprocess.check_output() has been removed, "
         "please switch to apiDockerCall()."
     )
 
 
-def dockerCall(*args, **kwargs):
+def dockerCall(*args: Any, **kwargs: Any) -> NoReturn:
     raise RuntimeError(
         "dockerCall() using subprocess.check_output() has been removed, "
         "please switch to apiDockerCall()."
     )
 
 
-def subprocessDockerCall(*args, **kwargs):
+def subprocessDockerCall(*args: Any, **kwargs: Any) -> NoReturn:
     raise RuntimeError(
         "subprocessDockerCall() has been removed, " "please switch to apiDockerCall()."
     )
 
 
+@overload
+def apiDockerCall(
+    job: "Job",
+    image: str,
+    parameters: Optional[Union[list[str], list[list[str]]]] = None,
+    deferParam: Optional[int] = None,
+    volumes: Optional[dict[str, dict[str, str]]] = None,
+    working_dir: Optional[str] = None,
+    containerName: Optional[str] = None,
+    entrypoint: Optional[Union[str, list[str]]] = None,
+    detach: Literal[False] = False,
+    log_config: Optional[dict[str, str]] = None,
+    auto_remove: Optional[bool] = None,
+    remove: Optional[bool] = False,
+    user: Optional[str] = None,
+    environment: Optional[dict[str, str]] = None,
+    stdout: Optional[bool] = None,
+    stderr: bool = False,
+    stream: bool = False,
+    demux: Literal[False] = False,
+    streamfile: Optional[str] = None,
+    accelerators: Optional[list[int]] = None,
+    timeout: int = 365 * 24 * 60 * 60,
+    **kwargs: Any,
+) -> str:
+    """detach=False + demux=False → str"""
+
+
+@overload
+def apiDockerCall(
+    job: "Job",
+    image: str,
+    parameters: Optional[Union[list[str], list[list[str]]]] = None,
+    deferParam: Optional[int] = None,
+    volumes: Optional[dict[str, dict[str, str]]] = None,
+    working_dir: Optional[str] = None,
+    containerName: Optional[str] = None,
+    entrypoint: Optional[Union[str, list[str]]] = None,
+    detach: Literal[False] = False,
+    log_config: Optional[dict[str, str]] = None,
+    auto_remove: Optional[bool] = None,
+    remove: Optional[bool] = False,
+    user: Optional[str] = None,
+    environment: Optional[dict[str, str]] = None,
+    stdout: Optional[bool] = None,
+    stderr: bool = False,
+    stream: Literal[True] = True,
+    demux: Literal[True] = True,
+    streamfile: Optional[str] = None,
+    accelerators: Optional[list[int]] = None,
+    timeout: int = 365 * 24 * 60 * 60,
+    **kwargs: Any,
+) -> Generator[tuple[str, str]]:
+    """detach=False + demux=True + stream=True → log generator"""
+
+
+@overload
+def apiDockerCall(
+    job: "Job",
+    image: str,
+    parameters: Optional[Union[list[str], list[list[str]]]] = None,
+    deferParam: Optional[int] = None,
+    volumes: Optional[dict[str, dict[str, str]]] = None,
+    working_dir: Optional[str] = None,
+    containerName: Optional[str] = None,
+    entrypoint: Optional[Union[str, list[str]]] = None,
+    detach: Literal[False] = False,
+    log_config: Optional[dict[str, str]] = None,
+    auto_remove: Optional[bool] = None,
+    remove: Optional[bool] = False,
+    user: Optional[str] = None,
+    environment: Optional[dict[str, str]] = None,
+    stdout: Optional[bool] = None,
+    stderr: bool = False,
+    stream: Literal[False] = False,
+    demux: Literal[True] = True,
+    streamfile: Optional[str] = None,
+    accelerators: Optional[list[int]] = None,
+    timeout: int = 365 * 24 * 60 * 60,
+    **kwargs: Any,
+) -> tuple[str, str]:
+    """detach=False + demux=True + stream=False → string"""
+
+
+@overload
 def apiDockerCall(
-    job,
-    image,
-    parameters=None,
-    deferParam=None,
-    volumes=None,
-    working_dir=None,
-    containerName=None,
-    entrypoint=None,
-    detach=False,
-    log_config=None,
-    auto_remove=None,
-    remove=False,
-    user=None,
-    environment=None,
-    stdout=None,
-    stderr=False,
-    stream=False,
-    demux=False,
-    streamfile=None,
+    job: "Job",
+    image: str,
+    parameters: Optional[Union[list[str], list[list[str]]]] = None,
+    deferParam: Optional[int] = None,
+    volumes: Optional[dict[str, dict[str, str]]] = None,
+    working_dir: Optional[str] = None,
+    containerName: Optional[str] = None,
+    entrypoint: Optional[Union[str, list[str]]] = None,
+    detach: Literal[True] = True,
+    log_config: Optional[dict[str, str]] = None,
+    auto_remove: Optional[bool] = None,
+    remove: Optional[bool] = False,
+    user: Optional[str] = None,
+    environment: Optional[dict[str, str]] = None,
+    stdout: Optional[bool] = None,
+    stderr: bool = False,
+    stream: bool = False,
+    demux: bool = False,
+    streamfile: Optional[str] = None,
     accelerators: Optional[list[int]] = None,
-    timeout=365 * 24 * 60 * 60,
-    **kwargs,
-):
+    timeout: int = 365 * 24 * 60 * 60,
+    **kwargs: Any,
+) -> "Container":
+    """detach=True → Container"""
+
+
+@overload
+def apiDockerCall(
+    job: "Job",
+    image: str,
+    parameters: Optional[Union[list[str], list[list[str]]]] = None,
+    deferParam: Optional[int] = None,
+    volumes: Optional[dict[str, dict[str, str]]] = None,
+    working_dir: Optional[str] = None,
+    containerName: Optional[str] = None,
+    entrypoint: Optional[Union[str, list[str]]] = None,
+    detach: bool = False,
+    log_config: Optional[dict[str, str]] = None,
+    auto_remove: Optional[bool] = None,
+    remove: Optional[bool] = False,
+    user: Optional[str] = None,
+    environment: Optional[dict[str, str]] = None,
+    stdout: Optional[bool] = None,
+    stderr: bool = False,
+    stream: bool = False,
+    demux: bool = False,
+    streamfile: Optional[str] = None,
+    accelerators: Optional[list[int]] = None,
+    timeout: int = 365 * 24 * 60 * 60,
+    **kwargs: Any,
+) -> Union[
+    str, "Container", Generator[tuple[str, str]], Generator[str], tuple[str, str]
+]: ...
+
+
+def apiDockerCall(
+    job: "Job",
+    image: str,
+    parameters: Optional[Union[list[str], list[list[str]]]] = None,
+    deferParam: Optional[int] = None,
+    volumes: Optional[dict[str, dict[str, str]]] = None,
+    working_dir: Optional[str] = None,
+    containerName: Optional[str] = None,
+    entrypoint: Optional[Union[str, list[str]]] = None,
+    detach: bool = False,
+    log_config: Optional[dict[str, str]] = None,
+    auto_remove: Optional[bool] = None,
+    remove: Optional[bool] = False,
+    user: Optional[str] = None,
+    environment: Optional[dict[str, str]] = None,
+    stdout: Optional[bool] = None,
+    stderr: bool = False,
+    stream: bool = False,
+    demux: bool = False,
+    streamfile: Optional[str] = None,
+    accelerators: Optional[list[int]] = None,
+    timeout: int = 365 * 24 * 60 * 60,
+    **kwargs: Any,
+) -> Union[
+    str, "Container", Generator[tuple[str, str]], Generator[str], tuple[str, str]
+]:
     """
     A toil wrapper for the python docker API.
 
@@ -109,66 +266,67 @@ def apiDockerCall(
     without output capture, the container is started and returned without
     waiting for it to finish.
 
-    :param toil.Job.job job: The Job instance for the calling function.
-    :param str image: Name of the Docker image to be used.
-                     (e.g. 'quay.io/ucsc_cgl/samtools:latest')
-    :param list[str] parameters: A list of string elements. If there are
-                                 multiple elements, these will be joined with
-                                 spaces. This handling of multiple elements
-                                 provides backwards compatibility with previous
-                                 versions which called docker using
-                                 subprocess.check_call().
-                                 If list of lists: list[list[str]], then treat
-                                 as successive commands chained with pipe.
-    :param str working_dir: The working directory.
-    :param int deferParam: Action to take on the container upon job completion.
+    :param job: The Job instance for the calling function.
+    :param image: Name of the Docker image to be used.
+                  (e.g. 'quay.io/ucsc_cgl/samtools:latest')
+    :param parameters: A list of string elements. If there are
+                       multiple elements, these will be joined with
+                       spaces. This handling of multiple elements
+                       provides backwards compatibility with previous
+                       versions which called docker using
+                       subprocess.check_call().
+                       If list of lists: list[list[str]], then treat
+                       as successive commands chained with pipe.
+    :param deferParam: Action to take on the container upon job completion.
            FORGO (0) leaves the container untouched and running.
            STOP (1) Sends SIGTERM, then SIGKILL if necessary to the container.
            RM (2) Immediately send SIGKILL to the container. This is the default
            behavior if deferParam is set to None.
-    :param str name: The name/ID of the container.
-    :param str entrypoint: Prepends commands sent to the container.  See:
-                      https://docker-py.readthedocs.io/en/stable/containers.html
-    :param bool detach: Run the container in detached mode. (equivalent to '-d')
-    :param bool stdout: Return logs from STDOUT when detach=False (default: True).
-                        Block and capture stdout to a file when detach=True
-                        (default: False). Output capture defaults to output.log,
-                        and can be specified with the "streamfile" kwarg.
-    :param bool stderr: Return logs from STDERR when detach=False (default: False).
-                        Block and capture stderr to a file when detach=True
-                        (default: False). Output capture defaults to output.log,
-                        and can be specified with the "streamfile" kwarg.
-    :param bool stream: If True and detach=False, return a log generator instead
-                        of a string. Ignored if detach=True. (default: False).
-    :param bool demux: Similar to `demux` in container.exec_run(). If True and
-                       detach=False, returns a tuple of (stdout, stderr). If
-                       stream=True, returns a log generator with tuples of
-                       (stdout, stderr). Ignored if detach=True. (default: False).
-    :param str streamfile: Collect container output to this file if detach=True and
-                        stderr and/or stdout are True. Defaults to "output.log".
-    :param dict log_config: Specify the logs to return from the container.  See:
-                      https://docker-py.readthedocs.io/en/stable/containers.html
-    :param bool remove: Remove the container on exit or not.
-    :param str user: The container will be run with the privileges of
-                     the user specified.  Can be an actual name, such
-                     as 'root' or 'lifeisaboutfishtacos', or it can be
-                     the uid or gid of the user ('0' is root; '1000' is
-                     an example of a less privileged uid or gid), or a
-                     complement of the uid:gid (RECOMMENDED), such as
-                     '0:0' (root user : root group) or '1000:1000'
-                     (some other user : some other user group).
+    :param volumes: A dictionary of volume locations to volume options.
+    :param working_dir: The working directory.
+    :param containerName: The name/ID of the container.
+    :param entrypoint: Prepends commands sent to the container.  See:
+                       https://docker-py.readthedocs.io/en/stable/containers.html
+    :param detach: Run the container in detached mode. (equivalent to '-d')
+    :param log_config: Specify the logs to return from the container.  See:
+                       https://docker-py.readthedocs.io/en/stable/containers.html
+    :param remove: Remove the container on exit or not.
+    :param user: The container will be run with the privileges of
+                 the user specified.  Can be an actual name, such
+                 as 'root' or 'lifeisaboutfishtacos', or it can be
+                 the uid or gid of the user ('0' is root; '1000' is
+                 an example of a less privileged uid or gid), or a
+                 complement of the uid:gid (RECOMMENDED), such as
+                 '0:0' (root user : root group) or '1000:1000'
+                 (some other user : some other user group).
     :param environment: Allows one to set environment variables inside of the
-                        container, such as:
-    :param int timeout: Use the given timeout in seconds for interactions with
-                        the Docker daemon. Note that the underlying docker module is
-                        not always able to abort ongoing reads and writes in order
-                        to respect the timeout. Defaults to 1 year (i.e. wait
-                        essentially indefinitely).
+                        container.
+    :param stdout: Return logs from STDOUT when detach=False (default: True).
+                   Block and capture stdout to a file when detach=True
+                   (default: False). Output capture defaults to output.log,
+                   and can be specified with the "streamfile" kwarg.
+    :param stderr: Return logs from STDERR when detach=False (default: False).
+                   Block and capture stderr to a file when detach=True
+                   (default: False). Output capture defaults to output.log,
+                   and can be specified with the "streamfile" kwarg.
+    :param stream: If True and detach=False, return a log generator instead
+                   of a string. Ignored if detach=True. (default: False).
+    :param demux: Similar to `demux` in container.exec_run(). If True and
+                  detach=False, returns a tuple of (stdout, stderr). If
+                  stream=True, returns a log generator with tuples of
+                  (stdout, stderr). Ignored if detach=True. (default: False).
+    :param streamfile: Collect container output to this file if detach=True and
+                       stderr and/or stdout are True. Defaults to "output.log".
     :param accelerators: Toil accelerator numbers (usually GPUs) to forward to
                          the container. These are interpreted in the current
                          Python process's environment. See
                          toil.lib.accelerators.get_individual_local_accelerators()
                          for the menu of available accelerators.
+    :param timeout: Use the given timeout in seconds for interactions with
+                    the Docker daemon. Note that the underlying docker module is
+                    not always able to abort ongoing reads and writes in order
+                    to respect the timeout. Defaults to 1 year (i.e. wait
+                    essentially indefinitely).
     :param kwargs: Additional keyword arguments supplied to the docker API's
                    run command.  The list is 75 keywords total, for examples
                    and full documentation see:
@@ -210,9 +368,8 @@ def apiDockerCall(
         chain_params = [
             " ".join(quote(arg) for arg in command) for command in parameters
         ]
-        command = " | ".join(chain_params)
         pipe_prefix = "set -eo pipefail && "
-        command = [pipe_prefix + command]
+        command: Union[None, str, list[str]] = [pipe_prefix + " | ".join(chain_params)]
         logger.debug("Calling docker with: " + repr(command))
 
     # If 'parameters' is a normal list, join all elements into a single string
@@ -221,8 +378,8 @@ def apiDockerCall(
     # Note that this is still a list, and the docker API prefers this as best
     # practice:
     # http://docker-py.readthedocs.io/en/stable/containers.html
-    elif len(parameters) > 0 and type(parameters) is list:
-        command = " ".join(quote(arg) for arg in parameters)
+    elif len(parameters) > 0 and isinstance(parameters, list):
+        command = " ".join(quote(arg) for arg in cast(list[str], parameters))
         logger.debug("Calling docker with: " + repr(command))
 
     # If the 'parameters' lists are empty, they are respecified as None, which
@@ -237,7 +394,7 @@ def apiDockerCall(
     if deferParam not in (None, FORGO, STOP, RM):
         raise RuntimeError("Please provide a valid value for deferParam.")
 
-    client = docker.from_env(version="auto", timeout=timeout)
+    client = docker.from_env(version="auto", timeout=timeout)  # type: ignore[attr-defined]
 
     if deferParam is None:
         deferParam = RM
@@ -259,7 +416,7 @@ def apiDockerCall(
     device_requests = []
     if accelerators:
         # Map accelerator numbers to host numbers
-        host_accelerators = []
+        host_accelerators: list[int] = []
         accelerator_mapping = get_host_accelerator_numbers()
         for our_number in accelerators:
             if our_number >= len(accelerator_mapping):
@@ -272,7 +429,8 @@ def apiDockerCall(
         # TODO: Here we assume that the host accelerators are all GPUs
         device_requests.append(
             docker.types.DeviceRequest(
-                device_ids=[",".join(host_accelerators)], capabilities=[["gpu"]]
+                device_ids=[",".join(str(x) for x in host_accelerators)],
+                capabilities=[["gpu"]],
             )
         )
 
@@ -399,7 +557,7 @@ def dockerKill(
                         to respect the timeout. Defaults to 1 year (i.e. wait
                         essentially indefinitely).
     """
-    client = docker.from_env(version="auto", timeout=timeout)
+    client = docker.from_env(version="auto", timeout=timeout)  # type: ignore[attr-defined]
     try:
         this_container = client.containers.get(container_name)
         while this_container.status == "running":
@@ -432,7 +590,9 @@ def dockerStop(container_name: str, remove: bool = False) -> None:
     dockerKill(container_name, gentleKill=True, remove=remove)
 
 
-def containerIsRunning(container_name: str, timeout: int = 365 * 24 * 60 * 60):
+def containerIsRunning(
+    container_name: str, timeout: int = 365 * 24 * 60 * 60
+) -> Optional[bool]:
     """
     Checks whether the container is running or not.
 
@@ -444,7 +604,7 @@ def containerIsRunning(container_name: str, timeout: int = 365 * 24 * 60 * 60):
     :returns: True if status is 'running', False if status is anything else,
         and None if the container does not exist.
     """
-    client = docker.from_env(version="auto", timeout=timeout)
+    client = docker.from_env(version="auto", timeout=timeout)  # type: ignore[attr-defined]
     try:
         this_container = client.containers.get(container_name)
         if this_container.status == "running":
@@ -459,7 +619,7 @@ def containerIsRunning(container_name: str, timeout: int = 365 * 24 * 60 * 60):
         raise create_api_error_from_http_exception(e)
 
 
-def getContainerName(job):
+def getContainerName(job: "Job") -> str:
     """
     Create a random string including the job name, and return it. Name will
     match ``[a-zA-Z0-9][a-zA-Z0-9_.-]``.
@@ -473,12 +633,13 @@ def getContainerName(job):
     return name
 
 
-def _multiplexed_response_stream_helper(response):
+def _multiplexed_response_stream_helper(
+    response: requests.Response,
+) -> Generator[tuple[Any, Union[bytes, Any]]]:
     """
     A generator of multiplexed data blocks coming from a response stream modified from:
     https://github.com/docker/docker-py/blob/4.3.1-release/docker/api/client.py#L370
 
-    :param response: requests.Response
     :return: a generator with tuples of (stream_type, data)
     """
     while True:

toil/lib/dockstore.py

@@ -45,8 +45,15 @@ logger = logging.getLogger(__name__)
 
 # We assume TRS_ROOT is actually a Dockstore instance.
 
+# This is a publish-able token for production Dockstore for Toil to use.
+# This is NOT a secret value.
+DEFAULT_DOCKSTORE_TOKEN = "2bff46294daddef6df185452b04db6143ea8a59f52ee3c325d3e1df418511b7d"
+
 # How should we authenticate our Dockstore requests?
-DOCKSTORE_TOKEN = os.environ.get("TOIL_DOCKSTORE_TOKEN")
+DOCKSTORE_TOKEN = os.environ.get("TOIL_DOCKSTORE_TOKEN", DEFAULT_DOCKSTORE_TOKEN)
+
+# What platform should we report metrics as?
+DOCKSTORE_PLATFORM = "TOIL"
 
 
 # This is a https://schema.org/CompletedActionStatus
@@ -346,7 +353,7 @@ def send_metrics(trs_workflow_id: str, trs_version: str, workflow_runs: list[Run
 
     # Set the submission query string metadata
     submission_params = {
-        "platform": "OTHER",
+        "platform": DOCKSTORE_PLATFORM,
         "description": "Workflow status from Toil"
     }
 
@@ -366,6 +373,7 @@ def send_metrics(trs_workflow_id: str, trs_version: str, workflow_runs: list[Run
     try:
         result = web_session.post(endpoint_url, params=submission_params, json=to_post, headers=headers)
         result.raise_for_status()
+        logger.debug("Workflow metrics were accepted by Dockstore. Dockstore response code: %s", result.status_code)
     except requests.HTTPError as e:
         logger.warning("Workflow metrics were not accepted by Dockstore. Dockstore complained: %s", e.response.text)
         raise
@@ -375,5 +383,5 @@ def get_metrics_url(trs_workflow_id: str, trs_version: str, execution_id: str) -
     Get the URL where a workflow metrics object (for a workflow, or for a set of tasks) can be fetched back from.
     """
 
-    return f"{TRS_ROOT}/api/api/ga4gh/v2/extended/{quote(trs_workflow_id, safe='')}/versions/{quote(trs_version, safe='')}/execution?platform=OTHER&executionId={quote(execution_id, safe='')}"
+    return f"{TRS_ROOT}/api/api/ga4gh/v2/extended/{quote(trs_workflow_id, safe='')}/versions/{quote(trs_version, safe='')}/execution?platform={DOCKSTORE_PLATFORM}&executionId={quote(execution_id, safe='')}"
 

toil/lib/exceptions.py

@@ -15,6 +15,8 @@
 # 5.14.2018: copied into Toil from https://github.com/BD2KGenomics/bd2k-python-lib
 
 import sys
+from typing import Optional
+import logging
 from urllib.parse import ParseResult
 
 
@@ -39,15 +41,15 @@ class panic:
     the primary exception will be reraised.
     """
 
-    def __init__(self, log=None):
+    def __init__(self, log: Optional[logging.Logger] = None) -> None:
         super().__init__()
         self.log = log
         self.exc_info = None
 
-    def __enter__(self):
+    def __enter__(self) -> None:
         self.exc_info = sys.exc_info()
 
-    def __exit__(self, *exc_info):
+    def __exit__(self, *exc_info) -> None:
         if self.log is not None and exc_info and exc_info[0]:
             self.log.warning("Exception during panic", exc_info=exc_info)
         exc_type, exc_value, traceback = self.exc_info