toil 7.0.0__py3-none-any.whl → 8.0.0__py3-none-any.whl

toil/__init__.py

@@ -11,21 +11,19 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import errno
 import logging
 import os
 import re
 import socket
 import sys
-import time
 from datetime import datetime
-from typing import TYPE_CHECKING, Optional, Tuple
+from typing import TYPE_CHECKING, Optional
 
 import requests
 
 from docker.errors import ImageNotFound
 from toil.lib.memoize import memoize
-from toil.lib.retry import retry
+from toil.lib.retry import retry as retry
 from toil.version import currentCommit
 
 if TYPE_CHECKING:
@@ -43,15 +41,15 @@ def which(cmd, mode=os.F_OK | os.X_OK, path=None) -> Optional[str]:
     `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result
     of os.environ.get("PATH"), or can be overridden with a custom search
     path.
-    
+
     :returns: The path found, or None.
     """
+
     # Check that a given file can be accessed with the correct mode.
     # Additionally check that `file` is not a directory, as on Windows
     # directories pass the os.access check.
     def _access_check(fn, mode):
-        return (os.path.exists(fn) and os.access(fn, mode)
-                and not os.path.isdir(fn))
+        return os.path.exists(fn) and os.access(fn, mode) and not os.path.isdir(fn)
 
     # If we're given a path with a directory part, look it up directly rather
     # than referring to PATH directories. This includes checking relative to the
@@ -106,17 +104,19 @@ def toilPackageDirPath() -> str:
     The return value is guaranteed to end in '/toil'.
     """
     result = os.path.dirname(os.path.realpath(__file__))
-    if not result.endswith('/toil'):
+    if not result.endswith("/toil"):
         raise RuntimeError("The top-level toil package is not named Toil.")
     return result
 
 
 def inVirtualEnv() -> bool:
     """Test if we are inside a virtualenv or Conda virtual environment."""
-    return ('VIRTUAL_ENV' in os.environ or
-            'CONDA_DEFAULT_ENV' in os.environ or
-            hasattr(sys, 'real_prefix') or
-            (hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix))
+    return (
+        "VIRTUAL_ENV" in os.environ
+        or "CONDA_DEFAULT_ENV" in os.environ
+        or hasattr(sys, "real_prefix")
+        or (hasattr(sys, "base_prefix") and sys.base_prefix != sys.prefix)
+    )
 
 
 def resolveEntryPoint(entryPoint: str) -> str:
@@ -125,7 +125,7 @@ def resolveEntryPoint(entryPoint: str) -> str:
 
     :returns: The path found, which may be an absolute or a relative path.
     """
-    if os.environ.get("TOIL_CHECK_ENV", None) == 'True' and inVirtualEnv():
+    if os.environ.get("TOIL_CHECK_ENV", None) == "True" and inVirtualEnv():
         path = os.path.join(os.path.dirname(sys.executable), entryPoint)
         # Inside a virtualenv we try to use absolute paths to the entrypoints.
         if os.path.isfile(path):
@@ -134,7 +134,9 @@ def resolveEntryPoint(entryPoint: str) -> str:
             # if Toil is installed in a virtualenv on the leader, it must be installed in
             # a virtualenv located at the same path on each worker as well.
             if not os.access(path, os.X_OK):
-                raise RuntimeError("Cannot access the Toil virtualenv. If installed in a virtualenv on a cluster, make sure that the virtualenv path is the same for the leader and workers.")
+                raise RuntimeError(
+                    "Cannot access the Toil virtualenv. If installed in a virtualenv on a cluster, make sure that the virtualenv path is the same for the leader and workers."
+                )
             return path
     # Otherwise, we aren't in a virtualenv, or we're in a virtualenv but Toil
     # came in via --system-site-packages, or we think the virtualenv might not
@@ -154,10 +156,15 @@ def physicalMemory() -> int:
     True
     """
     try:
-        return os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
+        return os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES")
     except ValueError:
         import subprocess
-        return int(subprocess.check_output(['sysctl', '-n', 'hw.memsize']).decode('utf-8').strip())
+
+        return int(
+            subprocess.check_output(["sysctl", "-n", "hw.memsize"])
+            .decode("utf-8")
+            .strip()
+        )
 
 
 def physicalDisk(directory: str) -> int:
@@ -181,15 +188,22 @@ def applianceSelf(forceDockerAppliance: bool = False) -> str:
     Setting TOIL_APPLIANCE_SELF will not be necessary in most cases.
     """
     import toil.version
-    registry = lookupEnvVar(name='docker registry',
-                            envName='TOIL_DOCKER_REGISTRY',
-                            defaultValue=toil.version.dockerRegistry)
-    name = lookupEnvVar(name='docker name',
-                        envName='TOIL_DOCKER_NAME',
-                        defaultValue=toil.version.dockerName)
-    appliance = lookupEnvVar(name='docker appliance',
-                             envName='TOIL_APPLIANCE_SELF',
-                             defaultValue=registry + '/' + name + ':' + toil.version.dockerTag)
+
+    registry = lookupEnvVar(
+        name="docker registry",
+        envName="TOIL_DOCKER_REGISTRY",
+        defaultValue=toil.version.dockerRegistry,
+    )
+    name = lookupEnvVar(
+        name="docker name",
+        envName="TOIL_DOCKER_NAME",
+        defaultValue=toil.version.dockerName,
+    )
+    appliance = lookupEnvVar(
+        name="docker appliance",
+        envName="TOIL_APPLIANCE_SELF",
+        defaultValue=registry + "/" + name + ":" + toil.version.dockerTag,
+    )
 
     checkDockerSchema(appliance)
 
@@ -211,9 +225,11 @@ def customDockerInitCmd() -> str:
 
     :returns: The custom command, or an empty string is returned if the environment variable is not set.
     """
-    command = lookupEnvVar(name='user-defined custom docker init command',
-                           envName='TOIL_CUSTOM_DOCKER_INIT_COMMAND',
-                           defaultValue='')
+    command = lookupEnvVar(
+        name="user-defined custom docker init command",
+        envName="TOIL_CUSTOM_DOCKER_INIT_COMMAND",
+        defaultValue="",
+    )
     _check_custom_bash_cmd(command)
     return command.replace("'", "'\\''")  # Ensure any single quotes are escaped.
 
@@ -224,24 +240,28 @@ def customInitCmd() -> str:
 
     The custom init command is run prior to running Toil appliance itself in workers and/or the
     primary node (i.e. this is run one stage before ``TOIL_CUSTOM_DOCKER_INIT_COMMAND``).
-    
+
     This can be useful for doing any custom initialization on instances (e.g. authenticating to
     private docker registries). Any single quotes are escaped and the command cannot contain a
     set of blacklisted chars (newline or tab).
 
     returns: the custom command or n empty string is returned if the environment variable is not set.
     """
-    command = lookupEnvVar(name='user-defined custom init command',
-                           envName='TOIL_CUSTOM_INIT_COMMAND',
-                           defaultValue='')
+    command = lookupEnvVar(
+        name="user-defined custom init command",
+        envName="TOIL_CUSTOM_INIT_COMMAND",
+        defaultValue="",
+    )
     _check_custom_bash_cmd(command)
     return command.replace("'", "'\\''")  # Ensure any single quotes are escaped.
 
 
 def _check_custom_bash_cmd(cmd_str):
     """Ensure that the Bash command doesn't contain invalid characters."""
-    if re.search(r'[\n\r\t]', cmd_str):
-        raise RuntimeError(f'"{cmd_str}" contains invalid characters (newline and/or tab).')
+    if re.search(r"[\n\r\t]", cmd_str):
+        raise RuntimeError(
+            f'"{cmd_str}" contains invalid characters (newline and/or tab).'
+        )
 
 
 def lookupEnvVar(name: str, envName: str, defaultValue: str) -> str:
@@ -256,10 +276,14 @@ def lookupEnvVar(name: str, envName: str, defaultValue: str) -> str:
     try:
         value = os.environ[envName]
     except KeyError:
-        log.info('Using default %s of %s as %s is not set.', name, defaultValue, envName)
+        log.info(
+            "Using default %s of %s as %s is not set.", name, defaultValue, envName
+        )
         return defaultValue
     else:
-        log.info('Overriding %s of %s with %s from %s.', name, defaultValue, value, envName)
+        log.info(
+            "Overriding %s of %s with %s from %s.", name, defaultValue, value, envName
+        )
         return value
 
 
@@ -278,14 +302,20 @@ def checkDockerImageExists(appliance: str) -> str:
         return appliance
     registryName, imageName, tag = parseDockerAppliance(appliance)
 
-    if registryName == 'docker.io':
-        return requestCheckDockerIo(origAppliance=appliance, imageName=imageName, tag=tag)
+    if registryName == "docker.io":
+        return requestCheckDockerIo(
+            origAppliance=appliance, imageName=imageName, tag=tag
+        )
     else:
-        return requestCheckRegularDocker(origAppliance=appliance, registryName=registryName, imageName=imageName,
-                                         tag=tag)
+        return requestCheckRegularDocker(
+            origAppliance=appliance,
+            registryName=registryName,
+            imageName=imageName,
+            tag=tag,
+        )
 
 
-def parseDockerAppliance(appliance: str) -> Tuple[str, str, str]:
+def parseDockerAppliance(appliance: str) -> tuple[str, str, str]:
     """
     Derive parsed registry, image reference, and tag from a docker image string.
 
@@ -303,21 +333,21 @@ def parseDockerAppliance(appliance: str) -> Tuple[str, str, str]:
     appliance = appliance.lower()
 
     # get the tag
-    if ':' in appliance:
-        tag = appliance.split(':')[-1]
-        appliance = appliance[:-(len(':' + tag))]  # remove only the tag
+    if ":" in appliance:
+        tag = appliance.split(":")[-1]
+        appliance = appliance[: -(len(":" + tag))]  # remove only the tag
     else:
         # default to 'latest' if no tag is specified
-        tag = 'latest'
+        tag = "latest"
 
     # get the registry and image
-    registryName = 'docker.io'  # default if not specified
+    registryName = "docker.io"  # default if not specified
     imageName = appliance  # will be true if not specified
-    if '/' in appliance and '.' in appliance.split('/')[0]:
-        registryName = appliance.split('/')[0]
-        imageName = appliance[len(registryName):]
-    registryName = registryName.strip('/')
-    imageName = imageName.strip('/')
+    if "/" in appliance and "." in appliance.split("/")[0]:
+        registryName = appliance.split("/")[0]
+        imageName = appliance[len(registryName) :]
+    registryName = registryName.strip("/")
+    imageName = imageName.strip("/")
 
     return registryName, imageName, tag
 
@@ -325,12 +355,14 @@ def parseDockerAppliance(appliance: str) -> Tuple[str, str, str]:
 def checkDockerSchema(appliance):
     if not appliance:
         raise ImageNotFound("No docker image specified.")
-    elif '://' in appliance:
-        raise ImageNotFound("Docker images cannot contain a schema (such as '://'): %s"
-                            "" % appliance)
+    elif "://" in appliance:
+        raise ImageNotFound(
+            "Docker images cannot contain a schema (such as '://'): %s" "" % appliance
+        )
     elif len(appliance) > 256:
-        raise ImageNotFound("Docker image must be less than 256 chars: %s"
-                            "" % appliance)
+        raise ImageNotFound(
+            "Docker image must be less than 256 chars: %s" "" % appliance
+        )
 
 
 class ApplianceImageNotFound(ImageNotFound):
@@ -345,22 +377,28 @@ class ApplianceImageNotFound(ImageNotFound):
     """
 
     def __init__(self, origAppliance, url, statusCode):
-        msg = ("The docker image that TOIL_APPLIANCE_SELF specifies (%s) produced "
-               "a nonfunctional manifest URL (%s). The HTTP status returned was %s. "
-               "The specifier is most likely unsupported or malformed.  "
-               "Please supply a docker image with the format: "
-               "'<websitehost>.io/<repo_path>:<tag>' or '<repo_path>:<tag>' "
-               "(for official docker.io images).  Examples: "
-               "'quay.io/ucsc_cgl/toil:latest', 'ubuntu:latest', or "
-               "'broadinstitute/genomes-in-the-cloud:2.0.0'."
-               "" % (origAppliance, url, str(statusCode)))
+        msg = (
+            "The docker image that TOIL_APPLIANCE_SELF specifies (%s) produced "
+            "a nonfunctional manifest URL (%s). The HTTP status returned was %s. "
+            "The specifier is most likely unsupported or malformed.  "
+            "Please supply a docker image with the format: "
+            "'<websitehost>.io/<repo_path>:<tag>' or '<repo_path>:<tag>' "
+            "(for official docker.io images).  Examples: "
+            "'quay.io/ucsc_cgl/toil:latest', 'ubuntu:latest', or "
+            "'broadinstitute/genomes-in-the-cloud:2.0.0'."
+            "" % (origAppliance, url, str(statusCode))
+        )
         super().__init__(msg)
 
+
 # Cache images we know exist so we don't have to ask the registry about them
 # all the time.
 KNOWN_EXTANT_IMAGES = set()
 
-def requestCheckRegularDocker(origAppliance: str, registryName: str, imageName: str, tag: str) -> bool:
+
+def requestCheckRegularDocker(
+    origAppliance: str, registryName: str, imageName: str, tag: str
+) -> bool:
     """
     Check if an image exists using the requests library.
 
@@ -384,8 +422,9 @@ def requestCheckRegularDocker(origAppliance: str, registryName: str, imageName:
         # Check the cache first
         return origAppliance
 
-    ioURL = 'https://{webhost}/v2/{pathName}/manifests/{tag}' \
-            ''.format(webhost=registryName, pathName=imageName, tag=tag)
+    ioURL = "https://{webhost}/v2/{pathName}/manifests/{tag}" "".format(
+        webhost=registryName, pathName=imageName, tag=tag
+    )
     response = requests.head(ioURL)
     if not response.ok:
         raise ApplianceImageNotFound(origAppliance, ioURL, response.status_code)
@@ -412,17 +451,20 @@ def requestCheckDockerIo(origAppliance: str, imageName: str, tag: str) -> bool:
         return origAppliance
 
     # only official images like 'busybox' or 'ubuntu'
-    if '/' not in imageName:
-        imageName = 'library/' + imageName
+    if "/" not in imageName:
+        imageName = "library/" + imageName
 
-    token_url = 'https://auth.docker.io/token?service=registry.docker.io&scope=repository:{repo}:pull'.format(
-        repo=imageName)
-    requests_url = f'https://registry-1.docker.io/v2/{imageName}/manifests/{tag}'
+    token_url = "https://auth.docker.io/token?service=registry.docker.io&scope=repository:{repo}:pull".format(
+        repo=imageName
+    )
+    requests_url = f"https://registry-1.docker.io/v2/{imageName}/manifests/{tag}"
 
     token = requests.get(token_url)
     jsonToken = token.json()
     bearer = jsonToken["token"]
-    response = requests.head(requests_url, headers={'Authorization': f'Bearer {bearer}'})
+    response = requests.head(
+        requests_url, headers={"Authorization": f"Bearer {bearer}"}
+    )
     if not response.ok:
         raise ApplianceImageNotFound(origAppliance, requests_url, response.status_code)
     else:
@@ -434,21 +476,18 @@ def logProcessContext(config: "Config") -> None:
     # toil.version.version (string) cannot be imported at top level because it conflicts with
     # toil.version (module) and Sphinx doesn't like that.
     from toil.version import version
+
     log.info("Running Toil version %s on host %s.", version, socket.gethostname())
     log.debug("Configuration: %s", config.__dict__)
 
 
 try:
-    from botocore.credentials import (JSONFileCache,
-                                      RefreshableCredentials,
-                                      create_credential_resolver)
-    from botocore.session import Session
-
-    cache_path = '~/.cache/aws/cached_temporary_credentials'
-    datetime_format = "%Y-%m-%dT%H:%M:%SZ"  # incidentally the same as the format used by AWS
+    cache_path = "~/.cache/aws/cached_temporary_credentials"
+    datetime_format = (
+        "%Y-%m-%dT%H:%M:%SZ"  # incidentally the same as the format used by AWS
+    )
     log = logging.getLogger(__name__)
 
-
     # But in addition to our manual cache, we also are going to turn on boto3's
     # new built-in caching layer.
 
@@ -461,7 +500,6 @@ try:
         """
         return dt.strftime(datetime_format)
 
-
     def str_to_datetime(s):
         """
         Convert a string, explicitly UTC into a naive (implicitly UTC) datetime object.

toil/batchSystems/__init__.py

@@ -18,6 +18,7 @@ class DeadlockException(Exception):
     Exception thrown by the Leader or BatchSystem when a deadlock is encountered due to insufficient
     resources to run the workflow
     """
+
     def __init__(self, msg):
         self.msg = f"Deadlock encountered: {msg}"
         super().__init__()