toil 5.12.0__py3-none-any.whl → 6.1.0a1__py3-none-any.whl

toil/lib/accelerators.py

@@ -16,7 +16,7 @@
 
 import os
 import subprocess
-from typing import Dict, List, Optional, Set, Union
+from typing import Dict, List, Set, Union, cast
 from xml.dom import minidom
 
 from toil.job import AcceleratorRequirement
@@ -92,10 +92,15 @@ def count_nvidia_gpus() -> int:
     # <https://github.com/common-workflow-language/cwltool/blob/6f29c59fb1b5426ef6f2891605e8fa2d08f1a8da/cwltool/cuda.py>
     # Some example output is here: <https://gist.github.com/loretoparisi/2620b777562c2dfd50d6b618b5f20867>
     try:
-        return int(minidom.parseString(
-            subprocess.check_output(["nvidia-smi", "-q", "-x"])
-        ).getElementsByTagName("attached_gpus")[0].firstChild.data)
-    except (FileNotFoundError, subprocess.CalledProcessError, IndexError, ValueError, PermissionError):
+        return int(
+            cast(
+                minidom.Text,
+                minidom.parseString(subprocess.check_output(["nvidia-smi", "-q", "-x"]))
+                .getElementsByTagName("attached_gpus")[0]
+                .firstChild,
+            ).data
+        )
+    except: 
         return 0
 
     # TODO: Parse each gpu > product_name > text content and convert to some

toil/lib/aws/__init__.py

@@ -11,27 +11,15 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import collections
-import inspect
 import json
 import logging
 import os
 import re
 import socket
-import threading
-from functools import lru_cache
-from typing import (Any,
-                    Callable,
-                    Dict,
-                    Iterable,
-                    List,
-                    MutableMapping,
-                    Optional,
-                    TypeVar,
-                    Union)
+from http.client import HTTPException
+from typing import Dict, MutableMapping, Optional
 from urllib.error import URLError
 from urllib.request import urlopen
-from http.client import HTTPException
 
 logger = logging.getLogger(__name__)
 
@@ -80,6 +68,7 @@ def get_aws_zone_from_metadata() -> Optional[str]:
         try:
             # Use the EC2 metadata service
             import boto
+            str(boto)  # to prevent removal of the import
             from boto.utils import get_instance_metadata
             logger.debug("Fetch AZ from EC2 metadata")
             return get_instance_metadata()['placement']['availability-zone']

toil/lib/aws/ami.py

@@ -1,12 +1,12 @@
 import json
 import logging
 import os
-import time
 import urllib.request
-from urllib.error import HTTPError
-from typing import Dict, Optional, Iterator, cast
+from typing import Dict, Iterator, Optional, cast
+from urllib.error import HTTPError, URLError
 
 from botocore.client import BaseClient
+from botocore.exceptions import ClientError
 
 from toil.lib.retry import retry
 
@@ -110,6 +110,12 @@ def flatcar_release_feed_amis(region: str, architecture: str = 'amd64', source:
             # Try again
             try_number += 1
             continue
+        except URLError:
+            # Could be a connection timeout
+            logger.exception(f'Failed to retrieve {source} Flatcar release feed JSON')
+            # Try again
+            try_number += 1
+            continue
     if try_number == MAX_TRIES:
         # We could not get the JSON
         logger.error(f'Could not get a readable {source} Flatcar release feed JSON')
@@ -150,11 +156,18 @@ def feed_flatcar_ami_release(ec2_client: BaseClient, architecture: str = 'amd64'
                 
     for ami in flatcar_release_feed_amis(region, architecture, source):
         # verify it exists on AWS
-        response = ec2_client.describe_images(Filters=[{'Name': 'image-id', 'Values': [ami]}])  # type: ignore
-        if len(response['Images']) == 1 and response['Images'][0]['State'] == 'available':
-            return ami 
-        else:
-            logger.warning(f'Flatcar release feed suggests image {ami} which does not exist on AWS in {region}')
+        try:
+            response = ec2_client.describe_images(Filters=[{'Name': 'image-id', 'Values': [ami]}])  # type: ignore
+            if len(response['Images']) == 1 and response['Images'][0]['State'] == 'available':
+                return ami
+            else:
+                logger.warning(f'Flatcar release feed suggests image {ami} which does not exist on AWS in {region}')
+        except ClientError:
+            # Sometimes we get back nonsense like:
+            # botocore.exceptions.ClientError: An error occurred (AuthFailure) when calling the DescribeImages operation: AWS was not able to validate the provided access credentials
+            # Don't hold that against the AMI.
+            logger.exception(f'Unable to check if AMI {ami} exists on AWS in {region}; assuming it does')
+            return ami
     # We didn't find it
     logger.warning(f'Flatcar release feed does not have an image for region {region} that exists on AWS')
     return None
@@ -162,7 +175,7 @@ def feed_flatcar_ami_release(ec2_client: BaseClient, architecture: str = 'amd64'
 
 @retry()  # TODO: What errors do we get for timeout, JSON parse failure, etc?
 def aws_marketplace_flatcar_ami_search(ec2_client: BaseClient, architecture: str = 'amd64') -> Optional[str]:
-    """Query AWS for all AMI names matching 'Flatcar-stable-*' and return the most recent one."""
+    """Query AWS for all AMI names matching ``Flatcar-stable-*`` and return the most recent one."""
 
     # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ec2.html#EC2.Client.describe_images
     # Possible arch choices on AWS: 'i386'|'x86_64'|'arm64'|'x86_64_mac'

toil/lib/aws/iam.py

@@ -3,16 +3,15 @@ import json
 import logging
 from collections import defaultdict
 from functools import lru_cache
-from typing import Any, Dict, List, Optional, Set, cast, Union, Sequence
+from typing import Dict, List, Optional, Union, cast
 
 import boto3
 from mypy_boto3_iam import IAMClient
-from mypy_boto3_iam.type_defs import AttachedPolicyTypeDef
+from mypy_boto3_iam.type_defs import (AttachedPolicyTypeDef,
+                                      PolicyDocumentDictTypeDef)
 from mypy_boto3_sts import STSClient
 
-from toil.lib.aws import zone_to_region
 from toil.lib.aws.session import client as get_client
-from toil.provisioners.aws import get_best_aws_zone
 
 logger = logging.getLogger(__name__)
 
@@ -121,7 +120,7 @@ def permission_matches_any(perm: str, list_perms: List[str]) -> bool:
             return True
     return False
 
-def get_actions_from_policy_document(policy_doc: Dict[str, Any]) -> AllowedActionCollection:
+def get_actions_from_policy_document(policy_doc: PolicyDocumentDictTypeDef) -> AllowedActionCollection:
     '''
     Given a policy document, go through each statement and create an AllowedActionCollection representing the
     permissions granted in the policy document.
@@ -138,11 +137,16 @@ def get_actions_from_policy_document(policy_doc: Dict[str, Any]) -> AllowedActio
             for resource in statement["Resource"]:
                 for key in ["Action", "NotAction"]:
                     if key in statement.keys():
-                        if isinstance(statement[key], list):
-                            allowed_actions[resource][key] += statement[key]
+                        # mypy_boto3_iam declares policy document as a TypedDict
+                        # This type expects 4 string keys, of which NotAction is not an option
+                        # Thus mypy complains. NotAction seems to be valid according to Amazon:
+                        # https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_notaction.html
+                        # so type: ignore for now
+                        if isinstance(statement[key], list):  # type: ignore[literal-required]
+                            allowed_actions[resource][key] += statement[key]  # type: ignore[literal-required]
                         else:
                             #Assumes that if it isn't a list it's probably a string
-                            allowed_actions[resource][key].append(statement[key])
+                            allowed_actions[resource][key].append(statement[key])  # type: ignore[literal-required]
 
     return allowed_actions
 def allowed_actions_attached(iam: IAMClient, attached_policies: List[AttachedPolicyTypeDef]) -> AllowedActionCollection:
@@ -181,24 +185,28 @@ def allowed_actions_roles(iam: IAMClient, policy_names: List[str], role_name: st
             PolicyName=policy_name
         )
         logger.debug("Checking role policy")
-        policy_document = json.loads(role_policy["PolicyDocument"])
+        # PolicyDocument is now a TypedDict, but an instance of TypedDict is not an instance of dict?
+        if isinstance(role_policy["PolicyDocument"], str):
+            policy_document = json.loads(role_policy["PolicyDocument"])
+        else:
+            policy_document = role_policy["PolicyDocument"]
 
         allowed_actions = add_to_action_collection(allowed_actions, get_actions_from_policy_document(policy_document))
 
     return allowed_actions
 
 
-def collect_policy_actions(policy_documents: Sequence[Union[str, Dict[str, Any]]]) -> AllowedActionCollection:
+def collect_policy_actions(policy_documents: List[Union[str, PolicyDocumentDictTypeDef]]) -> AllowedActionCollection:
     """
     Collect all of the actions allowed by the given policy documents into one AllowedActionCollection.
     """
     allowed_actions: AllowedActionCollection = init_action_collection()
     for policy_str in policy_documents:
         # sometimes a string is returned from the api, so convert to a dictionary
-        if isinstance(policy_str, dict):
-            policy_dict = policy_str
-        else:
+        if isinstance(policy_str, str):
             policy_dict = json.loads(policy_str)
+        else:
+            policy_dict = policy_str
         allowed_actions = add_to_action_collection(allowed_actions, get_actions_from_policy_document(policy_dict))
     return allowed_actions
 

toil/lib/aws/session.py

@@ -12,24 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import collections
-import inspect
 import logging
 import os
-import re
-import socket
 import threading
-from typing import (Any,
-                    Callable,
-                    Dict,
-                    Iterable,
-                    List,
-                    Optional,
-                    Tuple,
-                    TypeVar,
-                    Union,
-                    cast)
-from urllib.error import URLError
-from urllib.request import urlopen
+from typing import Dict, Optional, Tuple, cast
 
 import boto3
 import boto3.resources.base
@@ -37,8 +23,8 @@ import boto.connection
 import botocore
 from boto3 import Session
 from botocore.client import Config
-from botocore.utils import JSONFileCache
 from botocore.session import get_session
+from botocore.utils import JSONFileCache
 
 logger = logging.getLogger(__name__)
 

toil/lib/aws/utils.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import errno
-import json
 import logging
 import os
 import socket
@@ -21,15 +20,13 @@ from typing import (Any,
                     Callable,
                     ContextManager,
                     Dict,
-                    Hashable,
                     Iterable,
                     Iterator,
                     List,
                     Optional,
                     Set,
                     Union,
-                    cast,
-                    MutableMapping)
+                    cast)
 from urllib.parse import ParseResult
 
 from toil.lib.aws import session
@@ -345,6 +342,8 @@ def get_object_for_url(url: ParseResult, existing: Optional[bool] = None) -> "Ob
         """
         Extracts a key (object) from a given parsed s3:// URL.
 
+        If existing is true and the object does not exist, raises FileNotFoundError.
+
         :param bool existing: If True, key is expected to exist. If False, key is expected not to
                 exists and it will be created. If None, the key will be created if it doesn't exist.
         """
@@ -386,7 +385,7 @@ def get_object_for_url(url: ParseResult, existing: Optional[bool] = None) -> "Ob
             else:
                 raise
         if existing is True and not objExists:
-            raise RuntimeError(f"Key '{key_name}' does not exist in bucket '{bucket_name}'.")
+            raise FileNotFoundError(f"Key '{key_name}' does not exist in bucket '{bucket_name}'.")
         elif existing is False and objExists:
             raise RuntimeError(f"Key '{key_name}' exists in bucket '{bucket_name}'.")
 

toil/lib/compatibility.py

@@ -1,6 +1,6 @@
 import functools
 import warnings
-from typing import Any, Dict, Callable, Union, TypeVar, overload
+from typing import Any, Callable, Union
 
 
 def deprecated(new_function_name: str) -> Callable[..., Any]:

toil/lib/conversions.py

@@ -46,8 +46,10 @@ def convert_units(num: float,
                   src_unit: str,
                   dst_unit: str = 'B') -> float:
     """Returns a float representing the converted input in dst_units."""
-    assert src_unit.lower() in VALID_PREFIXES, f"{src_unit} not a valid unit, valid units are {VALID_PREFIXES}."
-    assert dst_unit.lower() in VALID_PREFIXES, f"{dst_unit} not a valid unit, valid units are {VALID_PREFIXES}."
+    if not src_unit.lower() in VALID_PREFIXES:
+        raise RuntimeError(f"{src_unit} not a valid unit, valid units are {VALID_PREFIXES}.")
+    if not dst_unit.lower() in VALID_PREFIXES:
+        raise RuntimeError(f"{dst_unit} not a valid unit, valid units are {VALID_PREFIXES}.")
     return (num * bytes_in_unit(src_unit)) / bytes_in_unit(dst_unit)
 
 
@@ -60,7 +62,8 @@ def parse_memory_string(string: str) -> Tuple[float, str]:
         # find the first character of the unit
         if character not in '0123456789.-_ ':
             units = string[i:].strip()
-            assert units.lower() in VALID_PREFIXES, f"{units} not a valid unit, valid units are {VALID_PREFIXES}."
+            if not units.lower() in VALID_PREFIXES:
+                raise RuntimeError(f"{units} not a valid unit, valid units are {VALID_PREFIXES}.")
             return float(string[:i]), units
     return float(string), 'b'
 
@@ -71,6 +74,7 @@ def human2bytes(string: str) -> int:
     integer number of bytes.
     """
     value, unit = parse_memory_string(string)
+
     return int(convert_units(value, src_unit=unit, dst_unit='b'))
 
 

toil/lib/docker.py

@@ -17,7 +17,7 @@ import os
 import re
 import struct
 from shlex import quote
-from typing import Optional, List
+from typing import List, Optional
 
 import requests
 
@@ -27,7 +27,6 @@ from docker.errors import (ContainerError,
                            NotFound,
                            create_api_error_from_http_exception)
 from docker.utils.socket import consume_socket_output, demux_adaptor
-
 from toil.lib.accelerators import get_host_accelerator_numbers
 
 logger = logging.getLogger(__name__)
@@ -84,16 +83,17 @@ def apiDockerCall(job,
     jobs, with the intention that failed/orphaned docker jobs be handled
     appropriately.
 
-    Example of using dockerCall in toil to index a FASTA file with SAMtools:
-    def toil_job(job):
-        working_dir = job.fileStore.getLocalTempDir()
-        path = job.fileStore.readGlobalFile(ref_id,
-                                          os.path.join(working_dir, 'ref.fasta')
-        parameters = ['faidx', path]
-        apiDockerCall(job,
-                      image='quay.io/ucgc_cgl/samtools:latest',
-                      working_dir=working_dir,
-                      parameters=parameters)
+    Example of using dockerCall in toil to index a FASTA file with SAMtools::
+
+        def toil_job(job):
+            working_dir = job.fileStore.getLocalTempDir()
+            path = job.fileStore.readGlobalFile(ref_id,
+                                              os.path.join(working_dir, 'ref.fasta')
+            parameters = ['faidx', path]
+            apiDockerCall(job,
+                          image='quay.io/ucgc_cgl/samtools:latest',
+                          working_dir=working_dir,
+                          parameters=parameters)
 
     Note that when run with detach=False, or with detach=True and stdout=True
     or stderr=True, this is a blocking call. When run with detach=True and
@@ -103,13 +103,13 @@ def apiDockerCall(job,
     :param toil.Job.job job: The Job instance for the calling function.
     :param str image: Name of the Docker image to be used.
                      (e.g. 'quay.io/ucsc_cgl/samtools:latest')
-    :param list[str] parameters: A list of string elements.  If there are
+    :param list[str] parameters: A list of string elements. If there are
                                  multiple elements, these will be joined with
-                                 spaces.  This handling of multiple elements
+                                 spaces. This handling of multiple elements
                                  provides backwards compatibility with previous
                                  versions which called docker using
                                  subprocess.check_call().
-                                 **If list of lists: list[list[str]], then treat
+                                 If list of lists: list[list[str]], then treat
                                  as successive commands chained with pipe.
     :param str working_dir: The working directory.
     :param int deferParam: Action to take on the container upon job completion.
@@ -225,8 +225,8 @@ def apiDockerCall(job,
     working_dir = os.path.abspath(working_dir)
 
     # Ensure the user has passed a valid value for deferParam
-    assert deferParam in (None, FORGO, STOP, RM), \
-        'Please provide a valid value for deferParam.'
+    if deferParam not in (None, FORGO, STOP, RM):
+        raise RuntimeError('Please provide a valid value for deferParam.')
 
     client = docker.from_env(version='auto', timeout=timeout)
 
@@ -413,12 +413,11 @@ def containerIsRunning(container_name: str, timeout: int = 365 * 24 * 60 * 60):
 
     :param container_name: Name of the container being checked.
     :param int timeout: Use the given timeout in seconds for interactions with
-                        the Docker daemon. Note that the underlying docker module is
-                        not always able to abort ongoing reads and writes in order
-                        to respect the timeout. Defaults to 1 year (i.e. wait
-                        essentially indefinitely).
+        the Docker daemon. Note that the underlying docker module is not always
+        able to abort ongoing reads and writes in order to respect the timeout.
+        Defaults to 1 year (i.e. wait essentially indefinitely).
     :returns: True if status is 'running', False if status is anything else,
-    and None if the container does not exist.
+        and None if the container does not exist.
     """
     client = docker.from_env(version='auto', timeout=timeout)
     try:
@@ -439,7 +438,7 @@ def containerIsRunning(container_name: str, timeout: int = 365 * 24 * 60 * 60):
 def getContainerName(job):
     """
     Create a random string including the job name, and return it. Name will
-    match [a-zA-Z0-9][a-zA-Z0-9_.-]
+    match ``[a-zA-Z0-9][a-zA-Z0-9_.-]``.
     """
     parts = ['toil', str(job.description), base64.b64encode(os.urandom(9), b'-_').decode('utf-8')]
     name = re.sub('[^a-zA-Z0-9_.-]', '', '--'.join(parts))

toil/lib/ec2.py

@@ -103,11 +103,13 @@ def wait_instances_running(ec2, instances: Iterable[Boto2Instance]) -> Iterable[
             if i.state == 'pending':
                 pending_ids.add(i.id)
             elif i.state == 'running':
-                assert i.id not in running_ids
+                if i.id in running_ids:
+                    raise RuntimeError("An instance was already added to the list of running instance IDs. Maybe there is a duplicate.")
                 running_ids.add(i.id)
                 yield i
             else:
-                assert i.id not in other_ids
+                if i.id in other_ids:
+                    raise RuntimeError("An instance was already added to the list of other instances. Maybe there is a duplicate.")
                 other_ids.add(i.id)
                 yield i
         logger.info('%i instance(s) pending, %i running, %i other.',
@@ -130,10 +132,10 @@ def wait_spot_requests_active(ec2, requests: Iterable[SpotInstanceRequest], time
     :param requests: The requests to wait on.
 
     :param timeout: Maximum time in seconds to spend waiting or None to wait forever. If a
-    timeout occurs, the remaining open requests will be cancelled.
+        timeout occurs, the remaining open requests will be cancelled.
 
     :param tentative: if True, give up on a spot request at the earliest indication of it
-    not being fulfilled immediately
+        not being fulfilled immediately
 
     """
 
@@ -166,11 +168,13 @@ def wait_spot_requests_active(ec2, requests: Iterable[SpotInstanceRequest], time
                             'Request %s entered status %s indicating that it will not be '
                             'fulfilled anytime soon.', r.id, r.status.code)
                 elif r.state == 'active':
-                    assert r.id not in active_ids
+                    if r.id in active_ids:
+                        raise RuntimeError("A request was already added to the list of active requests. Maybe there are duplicate requests.")
                     active_ids.add(r.id)
                     batch.append(r)
                 else:
-                    assert r.id not in other_ids
+                    if r.id in other_ids:
+                        raise RuntimeError("A request was already added to the list of other IDs. Maybe there are duplicate requests.")
                     other_ids.add(r.id)
                     batch.append(r)
             if batch: