toil 9.1.1__py3-none-any.whl → 9.2.0__py3-none-any.whl

toil/lib/dockstore.py

@@ -18,23 +18,19 @@ Contains functions for integrating Toil with UCSC Dockstore, for reporting metri
 For basic TRS functionality for fetching workflows, see trs.py.
 """
 
-import datetime
 import logging
 import math
 import os
 import re
 import sys
-import uuid
-from typing import Any, Literal, Optional, Union, TypedDict, cast
+from typing import Any, Literal, TypedDict, Union
+from urllib.parse import quote
 
-from urllib.parse import urlparse, unquote, quote
 import requests
 
-from toil.lib.misc import unix_seconds_to_timestamp, seconds_to_duration
+from toil.lib.misc import seconds_to_duration, unix_seconds_to_timestamp
 from toil.lib.trs import TRS_ROOT
-from toil.lib.retry import retry
 from toil.lib.web import web_session
-from toil.version import baseVersion
 
 if sys.version_info < (3, 11):
     from typing_extensions import NotRequired
@@ -47,7 +43,9 @@ logger = logging.getLogger(__name__)
 
 # This is a publish-able token for production Dockstore for Toil to use.
 # This is NOT a secret value.
-DEFAULT_DOCKSTORE_TOKEN = "2bff46294daddef6df185452b04db6143ea8a59f52ee3c325d3e1df418511b7d"
+DEFAULT_DOCKSTORE_TOKEN = (
+    "2bff46294daddef6df185452b04db6143ea8a59f52ee3c325d3e1df418511b7d"
+)
 
 # How should we authenticate our Dockstore requests?
 DOCKSTORE_TOKEN = os.environ.get("TOIL_DOCKSTORE_TOKEN", DEFAULT_DOCKSTORE_TOKEN)
@@ -59,7 +57,15 @@ DOCKSTORE_PLATFORM = "TOIL"
 # This is a https://schema.org/CompletedActionStatus
 # The values here are from expanding the type info in the Docksotre docs at
 # <https://dockstore.org/api/static/swagger-ui/index.html#/extendedGA4GH/executionMetricsPost>
-ExecutionStatus = Union[Literal["ALL"], Literal["SUCCESSFUL"], Literal["FAILED"], Literal["FAILED_SEMANTIC_INVALID"], Literal["FAILED_RUNTIME_INVALID"], Literal["ABORTED"]]
+ExecutionStatus = Union[
+    Literal["ALL"],
+    Literal["SUCCESSFUL"],
+    Literal["FAILED"],
+    Literal["FAILED_SEMANTIC_INVALID"],
+    Literal["FAILED_RUNTIME_INVALID"],
+    Literal["ABORTED"],
+]
+
 
 class Cost(TypedDict):
     """
@@ -71,6 +77,7 @@ class Cost(TypedDict):
     Cost in US Dollars.
     """
 
+
 class RunExecution(TypedDict):
     """
     Dockstore metrics data for a workflow or task run.
@@ -83,7 +90,7 @@ class RunExecution(TypedDict):
 
     dateExecuted: str
     """
-    ISO 8601 UTC timestamp when the execution happend.
+    ISO 8601 UTC timestamp when the execution happened.
     """
 
     executionStatus: ExecutionStatus
@@ -125,6 +132,7 @@ class RunExecution(TypedDict):
     Dockstore can take any JSON-able structured data.
     """
 
+
 class TaskExecutions(TypedDict):
     """
     Dockstore metrics data for all the tasks in a workflow.
@@ -140,7 +148,7 @@ class TaskExecutions(TypedDict):
 
     dateExecuted: str
     """
-    ISO 8601 UTC timestamp when the execution happend.
+    ISO 8601 UTC timestamp when the execution happened.
     """
 
     taskExecutions: list[RunExecution]
@@ -155,6 +163,7 @@ class TaskExecutions(TypedDict):
     Dockstore can take any JSON-able structured data.
     """
 
+
 def ensure_valid_id(execution_id: str) -> None:
     """
     Make sure the given execution ID is in Dockstore format and will be accepted by Dockstore.
@@ -173,29 +182,30 @@ def ensure_valid_id(execution_id: str) -> None:
     if not re.fullmatch("[a-zA-Z0-9_]+", execution_id):
         raise ValueError("Execution ID must be alphanumeric with internal underscores")
 
+
 def pack_workflow_metrics(
-        execution_id: str,
-        start_time: float,
-        runtime: float,
-        succeeded: bool,
-        job_store_type: Optional[str] = None,
-        batch_system: Optional[str] = None,
-        caching: Optional[bool] = None,
-        toil_version: Optional[str] = None,
-        python_version: Optional[str] = None,
-        platform_system: Optional[str] = None,
-        platform_machine: Optional[str] = None
-    ) -> RunExecution:
+    execution_id: str,
+    start_time: float,
+    runtime: float,
+    succeeded: bool,
+    job_store_type: str | None = None,
+    batch_system: str | None = None,
+    caching: bool | None = None,
+    toil_version: str | None = None,
+    python_version: str | None = None,
+    platform_system: str | None = None,
+    platform_machine: str | None = None,
+) -> RunExecution:
     """
     Pack up per-workflow metrics into a format that can be submitted to Dockstore.
 
     :param execution_id: Unique ID for the workflow execution. Must be in
-        Dockstore format. 
+        Dockstore format.
     :param start_time: Execution start time in seconds since the Unix epoch.
     :param rutime: Execution duration in seconds.
     :param jobstore_type: Kind of job store used, like "file" or "aws".
     :param batch_system: Python class name implementing the batch system used.
-    :param caching: Whether Toil filestore-level cahcing was used.
+    :param caching: Whether Toil filestore-level caching was used.
     :param toil_version: Version of Toil used (without any Git hash).
     :param python_version: Version of Python used.
     :param platform_system: Operating system type (like "Darwin" or "Linux").
@@ -210,7 +220,7 @@ def pack_workflow_metrics(
         executionId=execution_id,
         dateExecuted=unix_seconds_to_timestamp(start_time),
         executionTime=seconds_to_duration(runtime),
-        executionStatus="SUCCESSFUL" if succeeded else "FAILED"
+        executionStatus="SUCCESSFUL" if succeeded else "FAILED",
     )
 
     # TODO: Just use kwargs here?
@@ -242,22 +252,23 @@ def pack_workflow_metrics(
 
     return result
 
+
 def pack_single_task_metrics(
-        execution_id: str,
-        start_time: float,
-        runtime: float,
-        succeeded: bool,
-        job_name: Optional[str] = None,
-        cores: Optional[float] = None,
-        cpu_seconds: Optional[float] = None,
-        memory_bytes: Optional[int] = None,
-        disk_bytes: Optional[int] = None,
-    ) -> RunExecution:
+    execution_id: str,
+    start_time: float,
+    runtime: float,
+    succeeded: bool,
+    job_name: str | None = None,
+    cores: float | None = None,
+    cpu_seconds: float | None = None,
+    memory_bytes: int | None = None,
+    disk_bytes: int | None = None,
+) -> RunExecution:
     """
     Pack up metrics for a single task execution in a format that can be used in a Dockstore submission.
 
     :param execution_id: Unique ID for the workflow execution. Must be in
-        Dockstore format. 
+        Dockstore format.
     :param start_time: Execution start time in seconds since the Unix epoch.
     :param rutime: Execution duration in seconds.
     :param succeeded: Whether the execution succeeded.
@@ -278,7 +289,7 @@ def pack_single_task_metrics(
         executionId=execution_id,
         dateExecuted=unix_seconds_to_timestamp(start_time),
         executionTime=seconds_to_duration(runtime),
-        executionStatus="SUCCESSFUL" if succeeded else "FAILED"
+        executionStatus="SUCCESSFUL" if succeeded else "FAILED",
     )
 
     if memory_bytes is not None:
@@ -310,12 +321,14 @@ def pack_single_task_metrics(
     return result
 
 
-def pack_workflow_task_set_metrics(execution_id: str, start_time: float, tasks: list[RunExecution]) -> TaskExecutions:
+def pack_workflow_task_set_metrics(
+    execution_id: str, start_time: float, tasks: list[RunExecution]
+) -> TaskExecutions:
     """
     Pack up metrics for all the tasks in a workflow execution into a format that can be submitted to Dockstore.
-    
+
     :param execution_id: Unique ID for the workflow execution. Must be in
-        Dockstore format. 
+        Dockstore format.
     :param start_time: Execution start time for the overall workflow execution
         in seconds since the Unix epoch.
     :param tasks: Packed tasks from pack_single_task_metrics()
@@ -327,10 +340,16 @@ def pack_workflow_task_set_metrics(execution_id: str, start_time: float, tasks:
     return TaskExecutions(
         executionId=execution_id,
         dateExecuted=unix_seconds_to_timestamp(start_time),
-        taskExecutions=tasks
+        taskExecutions=tasks,
     )
 
-def send_metrics(trs_workflow_id: str, trs_version: str, workflow_runs: list[RunExecution], workflow_task_sets: list[TaskExecutions]) -> None:
+
+def send_metrics(
+    trs_workflow_id: str,
+    trs_version: str,
+    workflow_runs: list[RunExecution],
+    workflow_task_sets: list[TaskExecutions],
+) -> None:
     """
     Send packed workflow and/or task metrics to Dockstore.
 
@@ -340,7 +359,7 @@ def send_metrics(trs_workflow_id: str, trs_version: str, workflow_runs: list[Run
         each workflow. Each workflow should have one entry containing all its
         tasks. Does not have to be the same order/set of workflows as
         workflow_runs.
-    
+
     :raises requests.HTTPError: if Dockstore does not accept the metrics.
     """
 
@@ -348,13 +367,13 @@ def send_metrics(trs_workflow_id: str, trs_version: str, workflow_runs: list[Run
     to_post = {
         "runExecutions": workflow_runs,
         "taskExecutions": workflow_task_sets,
-        "validationExecutions": []
+        "validationExecutions": [],
     }
 
     # Set the submission query string metadata
     submission_params = {
         "platform": DOCKSTORE_PLATFORM,
-        "description": "Workflow status from Toil"
+        "description": "Workflow status from Toil",
     }
 
     # Set the headers. Even though user agent isn't in here, it still gets
@@ -371,17 +390,25 @@ def send_metrics(trs_workflow_id: str, trs_version: str, workflow_runs: list[Run
     logger.debug("With headers: %s", headers)
 
     try:
-        result = web_session.post(endpoint_url, params=submission_params, json=to_post, headers=headers)
+        result = web_session.post(
+            endpoint_url, params=submission_params, json=to_post, headers=headers
+        )
         result.raise_for_status()
-        logger.debug("Workflow metrics were accepted by Dockstore. Dockstore response code: %s", result.status_code)
+        logger.debug(
+            "Workflow metrics were accepted by Dockstore. Dockstore response code: %s",
+            result.status_code,
+        )
     except requests.HTTPError as e:
-        logger.warning("Workflow metrics were not accepted by Dockstore. Dockstore complained: %s", e.response.text)
+        logger.warning(
+            "Workflow metrics were not accepted by Dockstore. Dockstore complained: %s",
+            e.response.text,
+        )
         raise
 
+
 def get_metrics_url(trs_workflow_id: str, trs_version: str, execution_id: str) -> str:
     """
     Get the URL where a workflow metrics object (for a workflow, or for a set of tasks) can be fetched back from.
     """
 
     return f"{TRS_ROOT}/api/api/ga4gh/v2/extended/{quote(trs_workflow_id, safe='')}/versions/{quote(trs_version, safe='')}/execution?platform={DOCKSTORE_PLATFORM}&executionId={quote(execution_id, safe='')}"
-

toil/lib/ec2.py

@@ -1,19 +1,12 @@
+import binascii
 import logging
 import time
-from base64 import b64encode, b64decode
-import binascii
-from collections.abc import Generator, Iterable, Mapping
-from typing import (
-    TYPE_CHECKING,
-    Any,
-    Callable,
-    Literal,
-    Optional,
-    Union,
-)
+from base64 import b64decode, b64encode
+from collections.abc import Callable, Generator, Iterable, Mapping
+from typing import TYPE_CHECKING, Any, Literal
 
 from toil.lib.aws.session import establish_boto3_session
-from toil.lib.aws.utils import flatten_tags, boto3_pager
+from toil.lib.aws.utils import boto3_pager, flatten_tags
 from toil.lib.exceptions import panic
 from toil.lib.retry import (
     ErrorCondition,
@@ -37,15 +30,13 @@ a_short_time = 5
 a_long_time = 60 * 60
 logger = logging.getLogger(__name__)
 
+
 def is_base64(value: str) -> bool:
     """
     Return True if value is base64-decodeable, and False otherwise.
     """
     try:
-        b64decode(
-            value.encode("utf-8"),
-            validate=True
-        )
+        b64decode(value.encode("utf-8"), validate=True)
         return True
     except binascii.Error:
         return False
@@ -91,7 +82,9 @@ def retry_ec2(t=a_short_time, retry_for=10 * a_short_time, retry_while=not_found
 class UnexpectedResourceState(Exception):
     def __init__(self, resource, to_state, state):
         super().__init__(
-            "Expected state of %s to be '%s' but got '%s'" % (resource, to_state, state)
+            "Expected state of {} to be '{}' but got '{}'".format(
+                resource, to_state, state
+            )
         )
 
 
@@ -176,7 +169,7 @@ def wait_instances_running(
                 reservations = boto3_pager(
                     boto3_ec2.describe_instances,
                     "Reservations",
-                    InstanceIds=list(pending_ids)
+                    InstanceIds=list(pending_ids),
                 )
                 instances = [
                     instance
@@ -228,7 +221,9 @@ def wait_spot_requests_active(
             batch = []
             for r in requests:
                 r: "SpotInstanceRequestTypeDef"  # pycharm thinks it is a string
-                assert isinstance(r, dict), f"Found garbage posing as a spot request: {r}"
+                assert isinstance(
+                    r, dict
+                ), f"Found garbage posing as a spot request: {r}"
                 if r["State"] == "open":
                     open_ids.add(r["SpotInstanceRequestId"])
                     if r["Status"]["Code"] == "pending-evaluation":
@@ -320,7 +315,9 @@ def create_spot_instances(
     )  # boto3 image id is in the launch specification
 
     user_data = spec["LaunchSpecification"].get("UserData", "")
-    assert is_base64(user_data), f"Spot user data needs to be base64-encoded: {user_data}"
+    assert is_base64(
+        user_data
+    ), f"Spot user data needs to be base64-encoded: {user_data}"
 
     for attempt in retry_ec2(
         retry_for=a_long_time, retry_while=inconsistencies_detected
@@ -375,7 +372,9 @@ def create_spot_instances(
             page = boto3_ec2.describe_instances(InstanceIds=instance_ids)
             while page.get("NextToken") is not None:
                 yield page
-                page = boto3_ec2.describe_instances(InstanceIds=instance_ids, NextToken=page["NextToken"])
+                page = boto3_ec2.describe_instances(
+                    InstanceIds=instance_ids, NextToken=page["NextToken"]
+                )
             yield page
     if not num_active:
         message = "None of the spot requests entered the active state"
@@ -410,7 +409,9 @@ def create_ondemand_instances(
     user_data: str = spec.get("UserData", "")
     if user_data:
         # Hope any real user data contains some characters not allowed in base64
-        assert not is_base64(user_data), f"On-demand user data needs to not be base64-encoded: {user_data}"
+        assert not is_base64(
+            user_data
+        ), f"On-demand user data needs to not be base64-encoded: {user_data}"
 
     instance_type = spec["InstanceType"]
     logger.info("Creating %s instance(s) ... ", instance_type)
@@ -485,13 +486,13 @@ def create_instances(
     key_name: str,
     instance_type: str,
     num_instances: int = 1,
-    security_group_ids: Optional[list] = None,
-    user_data: Optional[Union[str, bytes]] = None,
-    block_device_map: Optional[list[dict]] = None,
-    instance_profile_arn: Optional[str] = None,
-    placement_az: Optional[str] = None,
+    security_group_ids: list | None = None,
+    user_data: str | bytes | None = None,
+    block_device_map: list[dict] | None = None,
+    instance_profile_arn: str | None = None,
+    placement_az: str | None = None,
     subnet_id: str = None,
-    tags: Optional[dict[str, str]] = None,
+    tags: dict[str, str] | None = None,
 ) -> list["Instance"]:
     """
     Replaces create_ondemand_instances.  Uses boto3 and returns a list of Boto3 instance dicts.
@@ -553,13 +554,13 @@ def create_launch_template(
     image_id: str,
     key_name: str,
     instance_type: str,
-    security_group_ids: Optional[list] = None,
-    user_data: Optional[Union[str, bytes]] = None,
-    block_device_map: Optional[list[dict]] = None,
-    instance_profile_arn: Optional[str] = None,
-    placement_az: Optional[str] = None,
-    subnet_id: Optional[str] = None,
-    tags: Optional[dict[str, str]] = None,
+    security_group_ids: list | None = None,
+    user_data: str | bytes | None = None,
+    block_device_map: list[dict] | None = None,
+    instance_profile_arn: str | None = None,
+    placement_az: str | None = None,
+    subnet_id: str | None = None,
+    tags: dict[str, str] | None = None,
 ) -> str:
     """
     Creates a launch template with the given name for launching instances with the given parameters.
@@ -640,10 +641,10 @@ def create_auto_scaling_group(
     vpc_subnets: list[str],
     min_size: int,
     max_size: int,
-    instance_types: Optional[Iterable[str]] = None,
-    spot_bid: Optional[float] = None,
+    instance_types: Iterable[str] | None = None,
+    spot_bid: float | None = None,
     spot_cheapest: bool = False,
-    tags: Optional[dict[str, str]] = None,
+    tags: dict[str, str] | None = None,
 ) -> None:
     """
     Create a new Auto Scaling Group with the given name (which is also its

toil/lib/ec2nodes.py

@@ -18,7 +18,7 @@ import os
 import re
 import shutil
 import textwrap
-from typing import Any, Union
+from typing import Any
 
 import enlighten  # type: ignore
 
@@ -130,7 +130,7 @@ def is_number(s: str) -> bool:
 
 def parse_storage(
     storage_info: str,
-) -> Union[list[int], tuple[Union[int, float], float]]:
+) -> list[int] | tuple[int | float, float]:
     """
     Parses EC2 JSON storage param string into a number.
 
@@ -158,8 +158,15 @@ def parse_storage(
             and specs[3] == "SSD"
         ):
             return 1, float(specs[0].replace(",", ""))
-        elif is_number(specs[0]) and specs[1].lower() == "x" and is_number(specs[2][:-2]) and specs[2][-2:] == "GB":
-            return float(specs[0].replace(",", "")), float(specs[2][:-2].replace(",", ""))
+        elif (
+            is_number(specs[0])
+            and specs[1].lower() == "x"
+            and is_number(specs[2][:-2])
+            and specs[2][-2:] == "GB"
+        ):
+            return float(specs[0].replace(",", "")), float(
+                specs[2][:-2].replace(",", "")
+            )
         else:
             raise RuntimeError(
                 f"EC2 JSON format has likely changed.  Error parsing disk specs : {storage_info.strip()}"

toil/lib/exceptions.py

@@ -14,11 +14,10 @@
 
 # 5.14.2018: copied into Toil from https://github.com/BD2KGenomics/bd2k-python-lib
 
-import sys
-from typing import Optional
 import logging
-from urllib.parse import ParseResult
+import sys
 from types import TracebackType
+from urllib.parse import ParseResult
 
 
 # TODO: isn't this built in to Python 3 now?
@@ -42,7 +41,7 @@ class panic:
     the primary exception will be reraised.
     """
 
-    def __init__(self, log: Optional[logging.Logger] = None) -> None:
+    def __init__(self, log: logging.Logger | None = None) -> None:
         super().__init__()
         self.log = log
         self.exc_info = None
@@ -57,7 +56,11 @@ class panic:
         raise_(exc_type, exc_value, traceback)
 
 
-def raise_(exc_type: Optional[type[BaseException]], exc_value: Optional[BaseException], traceback: Optional[TracebackType]) -> None:
+def raise_(
+    exc_type: type[BaseException] | None,
+    exc_value: BaseException | None,
+    traceback: TracebackType | None,
+) -> None:
     if exc_value is not None:
         exc = exc_value
     else:

toil/lib/ftp_utils.py

@@ -20,7 +20,7 @@ import logging
 import netrc
 import os
 from contextlib import closing
-from typing import Optional, Any, cast, IO
+from typing import IO, Any, cast
 from urllib.parse import urlparse
 from urllib.request import urlopen
 
@@ -33,11 +33,10 @@ class FtpFsAccess:
 
     Taken and modified from https://github.com/ohsu-comp-bio/cwl-tes/blob/03f0096f9fae8acd527687d3460a726e09190c3a/cwl_tes/ftp.py#L37-L251
     """
+
     # TODO: Properly support FTP over SSL
 
-    def __init__(
-        self, cache: Optional[dict[Any, ftplib.FTP]] = None
-    ):
+    def __init__(self, cache: dict[Any, ftplib.FTP] | None = None):
         """
         FTP object to handle FTP connections. By default, connect over FTP with TLS.
 
@@ -107,14 +106,12 @@ class FtpFsAccess:
         """
         if "r" in mode:
             host, port, user, passwd, path = self._parse_url(fn)
-            handle = urlopen("ftp://{}:{}@{}:{}/{}".format(user, passwd, host, port, path))
+            handle = urlopen(f"ftp://{user}:{passwd}@{host}:{port}/{path}")
             return cast(IO[bytes], closing(handle))
         # TODO: support write mode
         raise Exception("Write mode FTP not implemented")
 
-    def _parse_url(
-        self, url: str
-    ) -> tuple[str, int, Optional[str], Optional[str], str]:
+    def _parse_url(self, url: str) -> tuple[str, int, str | None, str | None, str]:
         """
         Parse an FTP url into hostname, username, password, and path
         :param url:
@@ -147,7 +144,7 @@ class FtpFsAccess:
                         user = "anonymous"
         return host, port, user, passwd, path
 
-    def _connect(self, url: str) -> Optional[ftplib.FTP]:
+    def _connect(self, url: str) -> ftplib.FTP | None:
         """
         Connect to an FTP server. Handles authentication.
         :param url: FTP url
@@ -177,9 +174,7 @@ class FtpFsAccess:
             return ftp
         return None
 
-    def _recall_credentials(
-        self, desired_host: str
-    ) -> tuple[Optional[str], Optional[str]]:
+    def _recall_credentials(self, desired_host: str) -> tuple[str | None, str | None]:
         """
         Grab the cached credentials
         :param desired_host: FTP hostname
@@ -190,7 +185,7 @@ class FtpFsAccess:
                 return user, passwd
         return None, None
 
-    def size(self, fn: str) -> Optional[int]:
+    def size(self, fn: str) -> int | None:
         """
         Get the size of an FTP object
         :param fn: FTP url
@@ -207,7 +202,7 @@ class FtpFsAccess:
                     # https://stackoverflow.com/questions/22090001/get-folder-size-using-ftplib/22093848#22093848
                     ftp.voidcmd("TYPE I")
                     return ftp.size(path)
-                handle = urlopen("ftp://{}:{}@{}:{}/{}".format(user, passwd, host, port, path))
+                handle = urlopen(f"ftp://{user}:{passwd}@{host}:{port}/{path}")
                 info = handle.info()
                 handle.close()
                 if "Content-length" in info: