toil 7.0.0__py3-none-any.whl → 8.1.0b1__py3-none-any.whl

toil/server/utils.py

@@ -16,24 +16,25 @@ import logging
 import os
 from abc import abstractmethod
 from datetime import datetime
-from typing import Dict, Optional, Tuple
+from typing import Optional
 from urllib.parse import urlparse
 
-import requests
-
 from toil.lib.io import AtomicFileCreate
 from toil.lib.retry import retry
+from toil.lib.web import web_session
 
 try:
     from toil.lib.aws import get_current_aws_region
     from toil.lib.aws.session import client
     from toil.lib.aws.utils import retry_s3
+
     HAVE_S3 = True
 except ImportError:
     HAVE_S3 = False
 
 logger = logging.getLogger(__name__)
 
+
 def get_iso_time() -> str:
     """
     Return the current time in ISO 8601 format.
@@ -55,23 +56,30 @@ def link_file(src: str, dest: str) -> None:
         os.symlink(src, dest)
 
 
-def download_file_from_internet(src: str, dest: str, content_type: Optional[str] = None) -> None:
+def download_file_from_internet(
+    src: str, dest: str, content_type: Optional[str] = None
+) -> None:
     """
     Download a file from the Internet and write it to dest.
     """
-    response = requests.get(src)
+    response = web_session.get(src)
 
     if not response.ok:
         raise RuntimeError("Request failed with a client error or a server error.")
 
-    if content_type and not response.headers.get("Content-Type", "").startswith(content_type):
+    if content_type and not response.headers.get("Content-Type", "").startswith(
+        content_type
+    ):
         val = response.headers.get("Content-Type")
         raise RuntimeError(f"Expected content type to be '{content_type}'.  Not {val}.")
 
     with open(dest, "wb") as f:
         f.write(response.content)
 
-def download_file_from_s3(src: str, dest: str, content_type: Optional[str] = None) -> None:
+
+def download_file_from_s3(
+    src: str, dest: str, content_type: Optional[str] = None
+) -> None:
     """
     Download a file from Amazon S3 and write it to dest.
     """
@@ -81,10 +89,11 @@ def download_file_from_s3(src: str, dest: str, content_type: Optional[str] = Non
     except ImportError:
         raise RuntimeError("Cannot access S3 as AWS modules are not available")
 
-    with open(dest, 'wb') as out_stream:
+    with open(dest, "wb") as out_stream:
         obj = get_object_for_url(urlparse(src), existing=True)
         obj.download_fileobj(out_stream)
 
+
 def get_file_class(path: str) -> str:
     """
     Return the type of the file as a human readable string.
@@ -97,6 +106,7 @@ def get_file_class(path: str) -> str:
         return "Directory"
     return "Unknown"
 
+
 @retry(errors=[OSError, BlockingIOError])
 def safe_read_file(file: str) -> Optional[str]:
     """
@@ -153,6 +163,7 @@ def safe_write_file(file: str, s: str) -> None:
             with open(temp_name, "w") as file_obj:
                 file_obj.write(s)
 
+
 class MemoryStateCache:
     """
     An in-memory place to store workflow state.
@@ -164,7 +175,7 @@ class MemoryStateCache:
         """
 
         super().__init__()
-        self._data: Dict[Tuple[str, str], Optional[str]] = {}
+        self._data: dict[tuple[str, str], Optional[str]] = {}
 
     def get(self, workflow_id: str, key: str) -> Optional[str]:
         """
@@ -185,6 +196,7 @@ class MemoryStateCache:
         else:
             self._data[(workflow_id, key)] = value
 
+
 class AbstractStateStore:
     """
     A place for the WES server to keep its state: the set of workflows that
@@ -250,6 +262,7 @@ class AbstractStateStore:
         """
         self._cache.set(workflow_id, key, value)
 
+
 class MemoryStateStore(MemoryStateCache, AbstractStateStore):
     """
     An in-memory place to store workflow state, for testing.
@@ -261,6 +274,7 @@ class MemoryStateStore(MemoryStateCache, AbstractStateStore):
     def __init__(self):
         super().__init__()
 
+
 class FileStateStore(AbstractStateStore):
     """
     A place to store workflow state that uses a POSIX-compatible file system.
@@ -275,7 +289,7 @@ class FileStateStore(AbstractStateStore):
         """
         super().__init__()
         parse = urlparse(url)
-        if parse.scheme.lower() not in ['file', '']:
+        if parse.scheme.lower() not in ["file", ""]:
             # We want to catch if we get the wrong argument.
             raise RuntimeError(f"{url} doesn't look like a local path")
         if not os.path.exists(parse.path):
@@ -309,7 +323,9 @@ class FileStateStore(AbstractStateStore):
             # Set the value in the file
             safe_write_file(file_path, value)
 
+
 if HAVE_S3:
+
     class S3StateStore(AbstractStateStore):
         """
         A place to store workflow state that uses an S3-compatible object store.
@@ -327,7 +343,7 @@ if HAVE_S3:
 
             parse = urlparse(url)
 
-            if parse.scheme.lower() != 's3':
+            if parse.scheme.lower() != "s3":
                 # We want to catch if we get the wrong argument.
                 raise RuntimeError(f"{url} doesn't look like an S3 URL")
 
@@ -335,12 +351,14 @@ if HAVE_S3:
             # urlparse keeps the leading '/', but here we want a path in the
             # bucket without a leading '/'. We also need to support an empty
             # path.
-            self._base_path = parse.path[1:] if parse.path.startswith('/') else parse.path
-            self._client = client('s3', region_name=get_current_aws_region())
+            self._base_path = (
+                parse.path[1:] if parse.path.startswith("/") else parse.path
+            )
+            self._client = client("s3", region_name=get_current_aws_region())
 
             logger.debug("Connected to S3StateStore at %s", url)
 
-        def _get_bucket_and_path(self, workflow_id: str, key: str) -> Tuple[str, str]:
+        def _get_bucket_and_path(self, workflow_id: str, key: str) -> tuple[str, str]:
             """
             Get the bucket and path in the bucket at which a key value belongs.
             """
@@ -354,13 +372,12 @@ if HAVE_S3:
             bucket, path = self._get_bucket_and_path(workflow_id, key)
             for attempt in retry_s3():
                 try:
-                    logger.debug('Fetch %s path %s', bucket, path)
+                    logger.debug("Fetch %s path %s", bucket, path)
                     response = self._client.get_object(Bucket=bucket, Key=path)
-                    return response['Body'].read().decode('utf-8')
+                    return response["Body"].read().decode("utf-8")
                 except self._client.exceptions.NoSuchKey:
                     return None
 
-
         def set(self, workflow_id: str, key: str, value: Optional[str]) -> None:
             """
             Set or clear a key value on S3.
@@ -369,18 +386,21 @@ if HAVE_S3:
             for attempt in retry_s3():
                 if value is None:
                     # Get rid of it.
-                    logger.debug('Clear %s path %s', bucket, path)
+                    logger.debug("Clear %s path %s", bucket, path)
                     self._client.delete_object(Bucket=bucket, Key=path)
                     return
                 else:
                     # Store it, clobbering anything there already.
-                    logger.debug('Set %s path %s', bucket, path)
-                    self._client.put_object(Bucket=bucket, Key=path,
-                                            Body=value.encode('utf-8'))
+                    logger.debug("Set %s path %s", bucket, path)
+                    self._client.put_object(
+                        Bucket=bucket, Key=path, Body=value.encode("utf-8")
+                    )
                     return
 
+
 # We want to memoize state stores so we can cache on them.
-state_store_cache: Dict[str, AbstractStateStore] = {}
+state_store_cache: dict[str, AbstractStateStore] = {}
+
 
 def connect_to_state_store(url: str) -> AbstractStateStore:
     """
@@ -392,25 +412,30 @@ def connect_to_state_store(url: str) -> AbstractStateStore:
     if url not in state_store_cache:
         # We need to actually make the state store
         parse = urlparse(url)
-        if parse.scheme.lower() == 's3':
+        if parse.scheme.lower() == "s3":
             # It's an S3 URL
             if HAVE_S3:
                 # And we can use S3, so make the right implementation for S3.
                 state_store_cache[url] = S3StateStore(url)
             else:
                 # We can't actually use S3, so complain.
-                raise RuntimeError(f'Cannot connect to {url} because Toil AWS '
-                                   f'dependencies are not available. Did you '
-                                   f'install Toil with the [aws] extra?')
-        elif parse.scheme.lower() in ['file', '']:
+                raise RuntimeError(
+                    f"Cannot connect to {url} because Toil AWS "
+                    f"dependencies are not available. Did you "
+                    f"install Toil with the [aws] extra?"
+                )
+        elif parse.scheme.lower() in ["file", ""]:
             # It's a file URL or path
             state_store_cache[url] = FileStateStore(url)
         else:
-            raise RuntimeError(f'Cannot connect to {url} because we do not '
-                               f'implement its URL scheme')
+            raise RuntimeError(
+                f"Cannot connect to {url} because we do not "
+                f"implement its URL scheme"
+            )
 
     return state_store_cache[url]
 
+
 class WorkflowStateStore:
     """
     Slice of a state store for the state of a particular workflow.
@@ -463,6 +488,7 @@ def connect_to_workflow_state_store(url: str, workflow_id: str) -> WorkflowState
 
     return WorkflowStateStore(connect_to_state_store(url), workflow_id)
 
+
 # When we see one of these terminal states, we stay there forever.
 TERMINAL_STATES = {"COMPLETE", "EXECUTOR_ERROR", "SYSTEM_ERROR", "CANCELED"}
 
@@ -470,6 +496,7 @@ TERMINAL_STATES = {"COMPLETE", "EXECUTOR_ERROR", "SYSTEM_ERROR", "CANCELED"}
 # workflow running task is gone and move it to CANCELED?
 MAX_CANCELING_SECONDS = 30
 
+
 class WorkflowStateMachine:
     """
     Class for managing the WES workflow state machine.
@@ -628,5 +655,3 @@ class WorkflowStateMachine:
             state = "UNKNOWN"
 
         return state
-
-

toil/server/wes/abstract_backend.py

@@ -4,7 +4,7 @@ import json
 import logging
 import os
 from abc import abstractmethod
-from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+from typing import Any, Callable, Optional, Union
 from urllib.parse import urldefrag
 
 import connexion  # type: ignore
@@ -16,18 +16,25 @@ logger = logging.getLogger(__name__)
 
 # Define a type for WES task log entries in responses
 # TODO: make this a typed dict with all the WES task log field names and their types.
-TaskLog = Dict[str, Union[str, int, None]]
+TaskLog = dict[str, Union[str, int, None]]
 
 
 class VersionNotImplementedException(Exception):
     """
     Raised when the requested workflow version is not implemented.
     """
-    def __init__(self,
-                 wf_type: str, version: Optional[str] = None, supported_versions: Optional[List[str]] = None) -> None:
+
+    def __init__(
+        self,
+        wf_type: str,
+        version: Optional[str] = None,
+        supported_versions: Optional[list[str]] = None,
+    ) -> None:
         if version:
-            message = ("workflow_type '{}' requires 'workflow_type_version' to be one of '{}'.  "
-                       "Got '{}' instead.".format(wf_type, str(supported_versions), version))
+            message = (
+                "workflow_type '{}' requires 'workflow_type_version' to be one of '{}'.  "
+                "Got '{}' instead.".format(wf_type, str(supported_versions), version)
+            )
         else:
             message = f"workflow_type '{wf_type}' is not supported."
 
@@ -38,6 +45,7 @@ class MalformedRequestException(Exception):
     """
     Raised when the request is malformed.
     """
+
     def __init__(self, message: str) -> None:
         super().__init__(message)
 
@@ -46,6 +54,7 @@ class WorkflowNotFoundException(Exception):
     """
     Raised when the requested run ID is not found.
     """
+
     def __init__(self) -> None:
         super().__init__("The requested workflow run wasn't found.")
 
@@ -54,6 +63,7 @@ class WorkflowConflictException(Exception):
     """
     Raised when the requested workflow is not in the expected state.
     """
+
     def __init__(self, run_id: str):
         super().__init__(f"Workflow {run_id} exists when it shouldn't.")
 
@@ -62,6 +72,7 @@ class OperationForbidden(Exception):
     """
     Raised when the request is forbidden.
     """
+
     def __init__(self, message: str) -> None:
         super().__init__(message)
 
@@ -70,6 +81,7 @@ class WorkflowExecutionException(Exception):
     """
     Raised when an internal error occurred during the execution of the workflow.
     """
+
     def __init__(self, message: str) -> None:
         super().__init__(message)
 
@@ -81,8 +93,10 @@ def handle_errors(func: Callable[..., Any]) -> Callable[..., Any]:
     GA4GH WES spec.
     """
 
-    def error(msg: Any, code: int = 500) -> Tuple[Dict[str, Any], int]:
-        logger.warning(f"Exception raised when calling '{func.__name__}()':", exc_info=True)
+    def error(msg: Any, code: int = 500) -> tuple[dict[str, Any], int]:
+        logger.warning(
+            f"Exception raised when calling '{func.__name__}()':", exc_info=True
+        )
         return {"msg": str(msg), "status_code": code}, code
 
     @functools.wraps(func)
@@ -114,7 +128,7 @@ class WESBackend:
     to handle user requests when they hit different endpoints.
     """
 
-    def __init__(self, options: List[str]):
+    def __init__(self, options: list[str]):
         """
         :param options: A list of default engine options to use when executing
                         a workflow.  Example options:
@@ -135,7 +149,7 @@ class WESBackend:
         return getattr(self, operation_id.split(".")[-1])
 
     @abstractmethod
-    def get_service_info(self) -> Dict[str, Any]:
+    def get_service_info(self) -> dict[str, Any]:
         """
         Get information about the Workflow Execution Service.
 
@@ -144,7 +158,9 @@ class WESBackend:
         raise NotImplementedError
 
     @abstractmethod
-    def list_runs(self, page_size: Optional[int] = None, page_token: Optional[str] = None) -> Dict[str, Any]:
+    def list_runs(
+        self, page_size: Optional[int] = None, page_token: Optional[str] = None
+    ) -> dict[str, Any]:
         """
         List the workflow runs.
 
@@ -153,7 +169,7 @@ class WESBackend:
         raise NotImplementedError
 
     @abstractmethod
-    def run_workflow(self) -> Dict[str, str]:
+    def run_workflow(self) -> dict[str, str]:
         """
         Run a workflow. This endpoint creates a new workflow run and returns
         a `RunId` to monitor its progress.
@@ -163,7 +179,7 @@ class WESBackend:
         raise NotImplementedError
 
     @abstractmethod
-    def get_run_log(self, run_id: str) -> Dict[str, Any]:
+    def get_run_log(self, run_id: str) -> dict[str, Any]:
         """
         Get detailed info about a workflow run.
 
@@ -172,7 +188,7 @@ class WESBackend:
         raise NotImplementedError
 
     @abstractmethod
-    def cancel_run(self, run_id: str) -> Dict[str, str]:
+    def cancel_run(self, run_id: str) -> dict[str, str]:
         """
         Cancel a running workflow.
 
@@ -181,7 +197,7 @@ class WESBackend:
         raise NotImplementedError
 
     @abstractmethod
-    def get_run_status(self, run_id: str) -> Dict[str, str]:
+    def get_run_status(self, run_id: str) -> dict[str, str]:
         """
         Get quick status info about a workflow run, returning a simple result
         with the overall state of the workflow run.
@@ -199,9 +215,17 @@ class WESBackend:
 
     @staticmethod
     def secure_path(path: str) -> str:
-        return os.path.join(*[str(secure_filename(p)) for p in path.split("/") if p not in ("", ".", "..")])
-
-    def collect_attachments(self, run_id: Optional[str], temp_dir: Optional[str]) -> Tuple[str, Dict[str, Any]]:
+        return os.path.join(
+            *[
+                str(secure_filename(p))
+                for p in path.split("/")
+                if p not in ("", ".", "..")
+            ]
+        )
+
+    def collect_attachments(
+        self, run_id: Optional[str], temp_dir: Optional[str]
+    ) -> tuple[str, dict[str, Any]]:
         """
         Collect attachments from the current request by staging uploaded files
         to temp_dir, and return the temp_dir and parsed body of the request.
@@ -212,7 +236,7 @@ class WESBackend:
         """
         if not temp_dir:
             temp_dir = mkdtemp()
-        body: Dict[str, Any] = {}
+        body: dict[str, Any] = {}
         has_attachments = False
         for key, ls in connexion.request.files.lists():
             try:
@@ -223,12 +247,20 @@ class WESBackend:
                         dest = os.path.join(temp_dir, self.secure_path(value.filename))
                         if not os.path.isdir(os.path.dirname(dest)):
                             os.makedirs(os.path.dirname(dest))
-                        self.log_for_run(run_id, f"Staging attachment '{value.filename}' to '{dest}'")
+                        self.log_for_run(
+                            run_id, f"Staging attachment '{value.filename}' to '{dest}'"
+                        )
                         value.save(dest)
                         has_attachments = True
-                        body[key] = f"file://{temp_dir}"  # Reference to temp working dir.
-
-                    elif key in ("workflow_params", "tags", "workflow_engine_parameters"):
+                        body[key] = (
+                            f"file://{temp_dir}"  # Reference to temp working dir.
+                        )
+
+                    elif key in (
+                        "workflow_params",
+                        "tags",
+                        "workflow_engine_parameters",
+                    ):
                         content = value.read()
                         body[key] = json.loads(content.decode("utf-8"))
                     else:
@@ -252,17 +284,23 @@ class WESBackend:
             url, ref = urldefrag(body["workflow_url"])
             if ":" not in url:
                 if not has_attachments:
-                    raise MalformedRequestException("Relative 'workflow_url' but missing 'workflow_attachment'")
+                    raise MalformedRequestException(
+                        "Relative 'workflow_url' but missing 'workflow_attachment'"
+                    )
                 body["workflow_url"] = self.secure_path(url)  # keep this relative
                 if ref:
                     # append "#ref" after the url
                     body["workflow_url"] += "#" + self.secure_path(ref)
-            self.log_for_run(run_id, "Using workflow_url '%s'" % body.get("workflow_url"))
+            self.log_for_run(
+                run_id, "Using workflow_url '%s'" % body.get("workflow_url")
+            )
         else:
             raise MalformedRequestException("Missing 'workflow_url' in submission")
 
         if "workflow_params" in body and not isinstance(body["workflow_params"], dict):
             # They sent us something silly like "workflow_params": "5"
-            raise MalformedRequestException("Got a 'workflow_params' which does not decode to a JSON object")
+            raise MalformedRequestException(
+                "Got a 'workflow_params' which does not decode to a JSON object"
+            )
 
         return temp_dir, body

toil/server/wes/amazon_wes_utils.py

@@ -20,21 +20,15 @@
 
 import json
 import logging
-import sys
 import zipfile
 from os import path
-from typing import IO, List, Optional, cast
-
-if sys.version_info >= (3, 8):
-    from typing import TypedDict
-else:
-    from typing_extensions import TypedDict
-
+from typing import IO, Optional, TypedDict, cast
 from urllib.parse import ParseResult, urlparse
 
 from toil.bus import JobStatus
-from toil.server.wes.abstract_backend import \
-    MalformedRequestException as InvalidRequestError
+from toil.server.wes.abstract_backend import (
+    MalformedRequestException as InvalidRequestError,
+)
 from toil.server.wes.abstract_backend import TaskLog
 
 logger = logging.getLogger(__name__)
@@ -53,20 +47,25 @@ Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 
 # The official spec we are working with here is: https://aws.github.io/amazon-genomics-cli/docs/concepts/workflows/#multi-file-workflows
 
+
 class WorkflowPlan(TypedDict):
     """
     These functions pass around dicts of a certain type, with `data` and `files` keys.
     """
+
     data: "DataDict"
     files: "FilesDict"
 
+
 class DataDict(TypedDict, total=False):
     """
     Under `data`, there can be:
     * `workflowUrl` (required if no `workflowSource`): URL to main workflow code.
     """
+
     workflowUrl: str
 
+
 class FilesDict(TypedDict, total=False):
     """
     Under `files`, there can be:
@@ -75,11 +74,13 @@ class FilesDict(TypedDict, total=False):
     * `workflowOptions`: Open binary-mode file for a JSON of options sent along with the workflow.
     * `workflowDependencies`: Open binary-mode file for the zip the workflow came in, if any.
     """
+
     workflowSource: IO[bytes]
-    workflowInputFiles: List[IO[bytes]]
+    workflowInputFiles: list[IO[bytes]]
     workflowOptions: IO[bytes]
     workflowDependencies: IO[bytes]
 
+
 def parse_workflow_zip_file(file: str, workflow_type: str) -> WorkflowPlan:
     r"""
     Processes a workflow zip bundle
@@ -163,9 +164,9 @@ def parse_workflow_manifest_file(manifest_file: str) -> WorkflowPlan:
     :rtype: dict of `data` and `files`
 
     MANIFEST.json is expected to be formatted like:
-    
+
     .. code-block:: json
-        
+
        {
            "mainWorkflowURL": "relpath/to/workflow",
            "inputFileURLs": [
@@ -235,7 +236,9 @@ def parse_workflow_manifest_file(manifest_file: str) -> WorkflowPlan:
     return {"data": data, "files": files}
 
 
-def workflow_manifest_url_to_path(url: ParseResult, parent_dir: Optional[str] = None) -> str:
+def workflow_manifest_url_to_path(
+    url: ParseResult, parent_dir: Optional[str] = None
+) -> str:
     """
     Interpret a possibly-relative parsed URL, relative to the given parent directory.
     """
@@ -244,6 +247,7 @@ def workflow_manifest_url_to_path(url: ParseResult, parent_dir: Optional[str] =
         return path.join(parent_dir, relpath)
     return relpath
 
+
 # This one is all UCSC code
 def task_filter(task: TaskLog, job_status: JobStatus) -> Optional[TaskLog]:
     """
@@ -264,6 +268,8 @@ def task_filter(task: TaskLog, job_status: JobStatus) -> Optional[TaskLog]:
 
     modified_task = dict(task)
     # Tack the batch ID onto the end of the name with the required separator
-    modified_task["name"] = "|".join([cast(str, modified_task.get("name", "")), batch_id])
+    modified_task["name"] = "|".join(
+        [cast(str, modified_task.get("name", "")), batch_id]
+    )
     logger.info("Transformed task %s to %s", task, modified_task)
     return modified_task