toil 8.0.0__py3-none-any.whl → 8.1.0b1__py3-none-any.whl

toil/lib/io.py

@@ -11,8 +11,29 @@ from contextlib import contextmanager
 from io import BytesIO
 from typing import IO, Any, Callable, Optional, Protocol, Union
 
+from toil.lib.memoize import memoize
+
 logger = logging.getLogger(__name__)
 
+@memoize
+def get_toil_home() -> str:
+    """
+    Get the Toil home directory for storing configuration and global state.
+
+    Raises an error if it does not exist and cannot be created. Safe to run
+    simultaneously in multiple processes.
+    """
+   
+    # TODO: should this use an XDG config directory or ~/.config to not clutter the
+    # base home directory?
+    toil_home_dir = os.path.join(os.path.expanduser("~"), ".toil")
+
+    dir_path = try_path(toil_home_dir)
+    if dir_path is None:
+        raise RuntimeError(
+            f"Cannot create or access Toil configuration directory {toil_home_dir}"
+        )
+    return dir_path
 
 TOIL_URI_SCHEME = "toilfile:"
 
@@ -177,7 +198,7 @@ def atomic_install(tmp_path, final_path) -> None:
 def AtomicFileCreate(final_path: str, keep: bool = False) -> Iterator[str]:
     """Context manager to create a temporary file.  Entering returns path to
     the temporary file in the same directory as finalPath.  If the code in
-    context succeeds, the file renamed to its actually name.  If an error
+    context succeeds, the file renamed to its actual name.  If an error
     occurs, the file is not installed and is removed unless keep is specified.
     """
     tmp_path = atomic_tmp_file(final_path)

toil/lib/misc.py

@@ -63,6 +63,24 @@ def unix_now_ms() -> float:
     """Return the current time in milliseconds since the Unix epoch."""
     return time.time() * 1000
 
+def unix_seconds_to_timestamp(timestamp: float) -> str:
+    """
+    Convert a time in seconds since the Unix epoch to an ISO 8601 string.
+    """
+    return datetime.datetime.fromtimestamp(timestamp, tz=datetime.timezone.utc).isoformat()
+
+def unix_seconds_to_local_time(timestamp: float) -> datetime.datetime:
+    """
+    Returns a local time corresponding to the given number of seconds since the Unix epoch.
+    """
+    return datetime.datetime.fromtimestamp(timestamp, tz=datetime.timezone.utc).astimezone()
+
+def seconds_to_duration(time_difference: float) -> str:
+    """
+    Convert a time difference in seconds to an ISO 8601 duration string.
+    """
+    return f"PT{time_difference:.3f}S"
+
 
 def slow_down(seconds: float) -> float:
     """

toil/lib/retry.py

@@ -24,21 +24,21 @@ objects wrapping Exceptions to include additional conditions.
 
 For example, retrying on a one Exception (HTTPError)::
 
-    from requests import get
+    from toil.lib.web import web_session
     from requests.exceptions import HTTPError
 
     @retry(errors=[HTTPError])
     def update_my_wallpaper():
-        return get('https://www.deviantart.com/')
+        return web_session.get('https://www.deviantart.com/')
 
 Or::
 
-    from requests import get
+    from toil.lib.web import web_session
     from requests.exceptions import HTTPError
 
     @retry(errors=[HTTPError, ValueError])
     def update_my_wallpaper():
-        return get('https://www.deviantart.com/')
+        return web_session.get('https://www.deviantart.com/')
 
 The examples above will retry for the default interval on any errors specified
 the "errors=" arg list.
@@ -46,7 +46,7 @@ the "errors=" arg list.
 To retry on specifically 500/502/503/504 errors, you could specify an ErrorCondition
 object instead, for example::
 
-    from requests import get
+    from toil.lib.web import web_session
     from requests.exceptions import HTTPError
 
     @retry(errors=[
@@ -55,11 +55,11 @@ object instead, for example::
                    error_codes=[500, 502, 503, 504]
                )])
     def update_my_wallpaper():
-        return requests.get('https://www.deviantart.com/')
+        return web_session.get('https://www.deviantart.com/')
 
 To retry on specifically errors containing the phrase "NotFound"::
 
-    from requests import get
+    from toil.lib.web import web_session
     from requests.exceptions import HTTPError
 
     @retry(errors=[
@@ -68,11 +68,11 @@ To retry on specifically errors containing the phrase "NotFound"::
             error_message_must_include="NotFound"
         )])
     def update_my_wallpaper():
-        return requests.get('https://www.deviantart.com/')
+        return web_session.get('https://www.deviantart.com/')
 
 To retry on all HTTPError errors EXCEPT an HTTPError containing the phrase "NotFound"::
 
-    from requests import get
+    from toil.lib.web import web_session
     from requests.exceptions import HTTPError
 
     @retry(errors=[
@@ -83,7 +83,7 @@ To retry on all HTTPError errors EXCEPT an HTTPError containing the phrase "NotF
                    retry_on_this_condition=False
                )])
     def update_my_wallpaper():
-        return requests.get('https://www.deviantart.com/')
+        return web_session.get('https://www.deviantart.com/')
 
 To retry on boto3's specific status errors, an example of the implementation is::
 

toil/lib/{integration.py → trs.py}

@@ -13,8 +13,8 @@
 # limitations under the License.
 
 """
-Contains functions for integrating Toil with external services such as
-Dockstore.
+Contains functions for integrating Toil with GA4GH Tool Registry Service
+servers, for fetching workflows.
 """
 
 import hashlib
@@ -24,35 +24,29 @@ import shutil
 import sys
 import tempfile
 import zipfile
-from typing import Any, Dict, List, Optional, Set, Tuple, cast
+from typing import Any, Literal, Optional, Union, TypedDict, cast
 
 from urllib.parse import urlparse, unquote, quote
 import requests
 
 from toil.lib.retry import retry
 from toil.lib.io import file_digest, robust_rmtree
-from toil.version import baseVersion
+from toil.lib.web import web_session
 
 logger = logging.getLogger(__name__)
 
-# We manage a Requests session at the module level in case we're supposed to be
-# doing cookies, and to send a sensible user agent.
-# We expect the Toil and Python version to not be personally identifiable even
-# in theory (someone might make a new Toil version first, buit there's no way
-# to know for sure that nobody else did the same thing).
-session = requests.Session()
-session.headers.update({"User-Agent": f"Toil {baseVersion} on Python {'.'.join([str(v) for v in sys.version_info])}"})
+TRS_ROOT = "https://dockstore.org" if "TOIL_TRS_ROOT" not in os.environ else os.environ["TOIL_TRS_ROOT"]
 
-def is_dockstore_workflow(workflow: str) -> bool:
+def is_trs_workflow(workflow: str) -> bool:
     """
-    Returns True if a workflow string smells Dockstore-y.
+    Returns True if a workflow string smells like TRS.
 
     Detects Dockstore page URLs and strings that could be Dockstore TRS IDs.
     """
 
-    return workflow.startswith("https://dockstore.org/workflows/") or workflow.startswith("#workflow/")
+    return workflow.startswith(f"{TRS_ROOT}/workflows/") or workflow.startswith(f"{TRS_ROOT}/my-workflows/") or workflow.startswith("#workflow/")
 
-def find_trs_spec(workflow: str) -> str:
+def extract_trs_spec(workflow: str) -> str:
     """
     Parse a Dockstore workflow URL or TSR ID to a string that is definitely a TRS ID.
     """
@@ -63,13 +57,16 @@ def find_trs_spec(workflow: str) -> str:
         logger.debug("Workflow %s is a TRS specifier already", workflow)
         trs_spec = workflow
     else:
-        # We need to get the right TRS ID from the Docstore URL
+        # We need to get the right TRS ID from the Dockstore URL
         parsed = urlparse(workflow)
-        # TODO: We assume the Docksotre page URL structure and the TRS IDs are basically the same.
+        # TODO: We assume the Dockstore page URL structure and the TRS IDs are basically the same.
         page_path = unquote(parsed.path)
-        if not page_path.startswith("/workflows/"):
+        if page_path.startswith("/workflows/"):
+            trs_spec = "#workflow/" + page_path[len("/workflows/"):]
+        elif page_path.startswith("/my-workflows/"):
+            trs_spec = "#workflow/" + page_path[len("/my-workflows/"):]
+        else:
             raise RuntimeError("Cannot parse Dockstore URL " + workflow)
-        trs_spec = "#workflow/" + page_path[len("/workflows/"):]
         logger.debug("Translated %s to TRS: %s", workflow, trs_spec)
 
     return trs_spec
@@ -88,27 +85,36 @@ def parse_trs_spec(trs_spec: str) -> tuple[str, Optional[str]]:
         trs_version = None
     return trs_workflow_id, trs_version
 
+def compose_trs_spec(trs_workflow_id: str, trs_version: str) -> str:
+    """
+    Compose a TRS ID from a workflow ID and version ID.
+    """
+    return f"{trs_workflow_id}:{trs_version}"
+
 @retry(errors=[requests.exceptions.ConnectionError])
-def get_workflow_root_from_dockstore(workflow: str, supported_languages: Optional[set[str]] = None) -> str:
+def find_workflow(workflow: str, supported_languages: Optional[set[str]] = None) -> tuple[str, str, str]:
     """
-    Given a Dockstore URL or TRS identifier, get the root WDL or CWL URL for the workflow.
+    Given a Dockstore URL or TRS identifier, get the root WDL or CWL URL for the workflow, along with the TRS workflow ID and version.
 
     Accepts inputs like:
 
         - https://dockstore.org/workflows/github.com/dockstore-testing/md5sum-checker:master?tab=info
         - #workflow/github.com/dockstore-testing/md5sum-checker
 
-    Assumes the input is actually one of the supported formats. See is_dockstore_workflow().
+    Assumes the input is actually one of the supported formats. See is_trs_workflow().
 
     TODO: Needs to handle multi-workflow files if Dockstore can.
 
+    :raises FileNotFoundError: if the workflow or version doesn't exist.
+    :raises ValueError: if the version is not specified but cannot be
+        automatically determined. 
     """
 
     if supported_languages is not None and len(supported_languages) == 0:
         raise ValueError("Set of supported languages must be nonempty if provided.")
 
     # Get the TRS id[:version] string from what might be a Dockstore URL
-    trs_spec = find_trs_spec(workflow)
+    trs_spec = extract_trs_spec(workflow)
     # Parse out workflow and possible version
     trs_workflow_id, trs_version = parse_trs_spec(trs_spec)
 
@@ -116,8 +122,14 @@ def get_workflow_root_from_dockstore(workflow: str, supported_languages: Optiona
 
     # Fetch the main TRS document.
     # See e.g. https://dockstore.org/api/ga4gh/trs/v2/tools/%23workflow%2Fgithub.com%2Fdockstore-testing%2Fmd5sum-checker
-    trs_workflow_url = f"https://dockstore.org/api/ga4gh/trs/v2/tools/{quote(trs_workflow_id, safe='')}"
-    trs_workflow_document = session.get(trs_workflow_url).json()
+    trs_workflow_url = f"{TRS_ROOT}/api/ga4gh/trs/v2/tools/{quote(trs_workflow_id, safe='')}"
+    logger.debug("Get versions: %s", trs_workflow_url)
+    trs_workflow_response = web_session.get(trs_workflow_url)
+    if trs_workflow_response.status_code in (400, 404):
+        # If the workflow ID isn't in Dockstore's accepted format (and also thus doesn't exist), we can get a 400
+        raise FileNotFoundError(f"Workflow {trs_workflow_id} does not exist.")
+    trs_workflow_response.raise_for_status()
+    trs_workflow_document = trs_workflow_response.json()
 
     # Make a map from version to version info. We will need the
     # "descriptor_type" array to find eligible languages, and the "url" field
@@ -146,12 +158,10 @@ def get_workflow_root_from_dockstore(workflow: str, supported_languages: Optiona
                 continue
         eligible_workflow_versions.add(version_name)
 
-    for default_version in ['main', 'master']:
-        if trs_version is None and default_version in eligible_workflow_versions:
-            # Fill in a version if the user didn't provide one.
-            trs_version = default_version
-            logger.debug("Defaulting to workflow version %s", default_version)
-            break
+    # TODO: Dockstore has a concept of a "default version", but doesn't expose
+    # it over TRS. To avoid defaulting to something that *isn't* the Dockstore
+    # default version, we refuse to choose a version when there are multiple
+    # possibilities.
 
     if trs_version is None and len(eligible_workflow_versions) == 1:
         # If there's just one version use that.
@@ -161,26 +171,31 @@ def get_workflow_root_from_dockstore(workflow: str, supported_languages: Optiona
 
     # If we don't like what we found we compose a useful error message.
     problems: list[str] = []
+    problem_type: type[Exception] = RuntimeError
     if trs_version is None:
         problems.append(f"Workflow {workflow} does not specify a version")
+        problem_type = ValueError
     elif trs_version not in workflow_versions:
         problems.append(f"Workflow version {trs_version} from {workflow} does not exist")
+        problem_type = FileNotFoundError
     elif trs_version not in eligible_workflow_versions:
         message = f"Workflow version {trs_version} from {workflow} is not available"
         if supported_languages is not None:
             message += f" in any of: {', '.join(supported_languages)}"
         problems.append(message)
+        problem_type = FileNotFoundError
     if len(problems) > 0:
         if len(eligible_workflow_versions) == 0:
             message = "No versions of the workflow are available"
             if supported_languages is not None:
                 message += f" in any of: {', '.join(supported_languages)}"
             problems.append(message)
+            problem_type = FileNotFoundError
         elif trs_version is None:
             problems.append(f"Add ':' and the name of a workflow version ({', '.join(eligible_workflow_versions)}) after '{trs_workflow_id}'")
         else:
             problems.append(f"Replace '{trs_version}' with one of ({', '.join(eligible_workflow_versions)})")
-        raise RuntimeError("; ".join(problems))
+        raise problem_type("; ".join(problems))
 
     # Tell MyPy we now have a version, or we would have raised
     assert trs_version is not None
@@ -192,12 +207,35 @@ def get_workflow_root_from_dockstore(workflow: str, supported_languages: Optiona
             language = candidate_language
 
     logger.debug("Going to use %s version %s in %s", trs_workflow_id, trs_version, language)
-    trs_version_url = workflow_versions[trs_version]["url"]
+
+    return trs_workflow_id, trs_version, language
+    
+@retry(errors=[requests.exceptions.ConnectionError])
+def fetch_workflow(trs_workflow_id: str, trs_version: str, language: str) -> str:
+    """
+    Returns a URL or local path to a workflow's primary descriptor file.
+
+    The file will be in context with its required files so it can actually run.
+
+    :raises FileNotFoundError: if the workflow or version doesn't exist.
+    """
+
+    # TODO: We should probably use HATEOAS and pull this from the worflow
+    # document we probably already fetched but aren't passing.
+    trs_version_url = f"{TRS_ROOT}/api/ga4gh/trs/v2/tools/{quote(trs_workflow_id, safe='')}/versions/{quote(trs_version, safe='')}"
 
     # Fetch the list of all the files
     trs_files_url = f"{trs_version_url}/{language}/files"
     logger.debug("Workflow files URL: %s", trs_files_url)
-    trs_files_document = session.get(trs_files_url).json()
+    trs_files_response = web_session.get(trs_files_url)
+    if trs_files_response.status_code in (204, 400, 404):
+        # We can get a 204 No Content response if the version doesn't exist.
+        # That's successful, so we need to handle it specifically. See
+        # <https://github.com/dockstore/dockstore/issues/6048>
+        # We can also get a 400 if the workflow ID is not in Dockstore's expected format (3 slash-separated segments).
+        raise FileNotFoundError(f"Workflow {trs_workflow_id} version {trs_version} in language {language} does not exist.")
+    trs_files_response.raise_for_status()
+    trs_files_document = trs_files_response.json()
 
     # Find the information we need to ID the primary descriptor file
     primary_descriptor_path: Optional[str] = None
@@ -210,7 +248,7 @@ def get_workflow_root_from_dockstore(workflow: str, supported_languages: Optiona
             primary_descriptor_hash = file_info["checksum"]["checksum"]
             break
     if primary_descriptor_path is None or primary_descriptor_hash is None or primary_descriptor_hash_algorithm is None:
-        raise RuntimeError("Could not find a primary descriptor file for the workflow")
+        raise RuntimeError(f"Could not find a primary descriptor file for workflow {trs_workflow_id} version {trs_version} in language {language}")
     primary_descriptor_basename = os.path.basename(primary_descriptor_path)
 
     # Work out how to compute the hash we are looking for. See
@@ -263,7 +301,7 @@ def get_workflow_root_from_dockstore(workflow: str, supported_languages: Optiona
             }
             # If we don't set stream=True, we can't actually read anything from the
             # raw stream, since Requests will have done it already.
-            with session.get(trs_zip_file_url, headers=headers, stream=True) as response:
+            with web_session.get(trs_zip_file_url, headers=headers, stream=True) as response:
                 response_content_length = response.headers.get("Content-Length")
                 logger.debug("Server reports content length: %s", response_content_length)
                 shutil.copyfileobj(response.raw, zip_file)
@@ -308,27 +346,38 @@ def get_workflow_root_from_dockstore(workflow: str, supported_languages: Optiona
                     logger.debug("Rejected %s because its %s hash %s is not %s", file_path, python_hash_name, file_hash, primary_descriptor_hash)
     if found_path is None:
         # We couldn't find the promised primary descriptor
-        raise RuntimeError(f"Could not find a {primary_descriptor_basename} with {primary_descriptor_hash_algorithm} hash {primary_descriptor_hash}")
+        raise RuntimeError(f"Could not find a {primary_descriptor_basename} with {primary_descriptor_hash_algorithm} hash {primary_descriptor_hash} for workflow {trs_workflow_id} version {trs_version} in language {language}")
 
     return found_path
 
-def resolve_workflow(workflow: str, supported_languages: Optional[set[str]] = None) -> str:
+def resolve_workflow(workflow: str, supported_languages: Optional[set[str]] = None) -> tuple[str, Optional[str]]:
     """
     Find the real workflow URL or filename from a command line argument.
 
     Transform a workflow URL or path that might actually be a Dockstore page
-    URL or TRS specifier to an actual URL or path to a workflow document.
+    URL or TRS specifier to an actual URL or path to a workflow document, and
+    optional TRS specifier.
+
+    Accepts inputs like
+
+        - https://dockstore.org/workflows/github.com/dockstore-testing/md5sum-checker:master?tab=info
+        - #workflow/github.com/dockstore-testing/md5sum-checker
+        - ./local.cwl
+        - https://example.com/~myuser/workflow/main.cwl
+
+    :raises FileNotFoundError: if the workflow or version should be in Dockstore but doesn't seem to exist.
     """
 
-    if is_dockstore_workflow(workflow):
-        # Ask Dockstore where to find Dockstore-y things
-        resolved = get_workflow_root_from_dockstore(workflow, supported_languages=supported_languages)
-        logger.info("Resolved Dockstore workflow %s to %s", workflow, resolved)
-        return resolved
+    if is_trs_workflow(workflow):
+        # Ask TRS host where to find TRS-looking things
+        trs_workflow_id, trs_version, language = find_workflow(workflow, supported_languages)
+        resolved = fetch_workflow(trs_workflow_id, trs_version, language)
+        logger.info("Resolved TRS workflow %s to %s", workflow, resolved)
+        return resolved, compose_trs_spec(trs_workflow_id, trs_version)
     else:
         # Pass other things through.
-        return workflow
-
+        # TODO: Find out if they have TRS names.
+        return workflow, None
 
 
 

toil/lib/web.py

@@ -0,0 +1,38 @@
+# Copyright (C) 2024 Regents of the University of California
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Contains functions for making web requests with Toil.
+
+All web requests should go through this module, to make sure they use the right
+user agent.
+
+>>> from toil.lib.web import web_session
+>>> web_session.get("https://example.com")
+
+"""
+
+import logging
+import requests
+import sys
+
+from toil.version import baseVersion
+
+# We manage a Requests session at the module level in case we're supposed to be
+# doing cookies, and to send a sensible user agent.
+# We expect the Toil and Python version to not be personally identifiable even
+# in theory (someone might make a new Toil version first, but there's no way
+# to know for sure that nobody else did the same thing).
+web_session = requests.Session()
+web_session.headers.update({"User-Agent": f"Toil {baseVersion} on Python {'.'.join([str(v) for v in sys.version_info])}"})

toil/options/common.py

@@ -362,12 +362,13 @@ def add_base_toil_options(
         action="store_true",
         help="Do not capture standard output and error from batch system jobs.",
     )
+    # TODO: Should this be deprecated since we always save stats now for history tracking?
     core_options.add_argument(
         "--stats",
         dest="stats",
         default=False,
         action="store_true",
-        help="Records statistics about the toil workflow to be used by 'toil stats'.",
+        help="Keep statistics about the toil workflow to be used by 'toil stats'.",
     )
     clean_choices = ["always", "onError", "never", "onSuccess"]
     core_options.add_argument(
@@ -466,7 +467,8 @@ def add_base_toil_options(
     )
 
     caching = file_store_options.add_mutually_exclusive_group()
-    caching_help = "Enable or disable caching for your workflow, specifying this overrides default from job store"
+    caching_help = ("Enable or disable worker level file caching for your workflow, specifying this overrides default from batch system. "
+                    "Does not affect CWL or WDL task caching.")
     caching.add_argument(
         "--caching",
         dest="caching",
@@ -1102,6 +1104,19 @@ def add_base_toil_options(
         default=False,
         help="Disables the progress bar shown when standard error is a terminal.",
     )
+    misc_options.add_argument(
+        "--publishWorkflowMetrics",
+        dest="publish_workflow_metrics",
+        choices=["all", "current", "no"],
+        default=None,
+        help="Whether to publish workflow metrics reports (including unique workflow "
+        "and task run IDs, job names, and version and Toil feature use information) to "
+        "Dockstore when a workflow completes. Selecting \"current\" will publish metrics "
+        "for the current workflow. Selecting \"all\" will also publish prior workflow "
+        "runs from the Toil history database, even if they themselves were run with \"no\". "
+        "Note that once published, workflow metrics CANNOT be deleted or un-published; they "
+        "will stay published forever!"
+    )
 
     # Debug options
     debug_options = parser.add_argument_group(

toil/options/cwl.py

@@ -419,3 +419,13 @@ def add_cwl_options(parser: ArgumentParser, suppress: bool = True) -> None:
         type=str,
         help=suppress_help or "Specify a cloud bucket endpoint for output files.",
     )
+    parser.add_argument(
+        "--cachedir",
+        type=str,
+        help=suppress_help
+        or "Directory to cache intermediate workflow outputs to avoid "
+        "recomputing steps. Can be very helpful in the development and "
+        "troubleshooting of CWL documents. This automatically bypasses the file store."
+        " Not to be confused with --caching.",
+        dest="cachedir"
+    )

toil/provisioners/gceProvisioner.py

@@ -19,7 +19,6 @@ import time
 import uuid
 from typing import Optional
 
-import requests
 from libcloud.compute.drivers.gce import GCEFailedNode
 from libcloud.compute.providers import get_driver
 from libcloud.compute.types import Provider
@@ -27,6 +26,7 @@ from libcloud.compute.types import Provider
 from toil.jobStores.googleJobStore import GoogleJobStore
 from toil.lib.compatibility import compat_bytes_recursive
 from toil.lib.conversions import human2bytes
+from toil.lib.web import web_session
 from toil.provisioners import NoSuchClusterException
 from toil.provisioners.abstractProvisioner import AbstractProvisioner, Shape
 from toil.provisioners.node import Node
@@ -83,11 +83,11 @@ class GCEProvisioner(AbstractProvisioner):
         """
         metadata_server = "http://metadata/computeMetadata/v1/instance/"
         metadata_flavor = {"Metadata-Flavor": "Google"}
-        zone = requests.get(metadata_server + "zone", headers=metadata_flavor).text
+        zone = web_session.get(metadata_server + "zone", headers=metadata_flavor).text
         self._zone = zone.split("/")[-1]
 
         project_metadata_server = "http://metadata/computeMetadata/v1/project/"
-        self._projectId = requests.get(
+        self._projectId = web_session.get(
             project_metadata_server + "project-id", headers=metadata_flavor
         ).text
 
@@ -95,7 +95,7 @@ class GCEProvisioner(AbstractProvisioner):
         self._googleJson = ""
         self._clientEmail = ""
 
-        self._tags = requests.get(
+        self._tags = web_session.get(
             metadata_server + "description", headers=metadata_flavor
         ).text
         tags = json.loads(self._tags)

toil/server/cli/wes_cwl_runner.py

@@ -10,13 +10,13 @@ from io import BytesIO
 from typing import Any, Optional, cast
 from urllib.parse import urldefrag, urljoin, urlparse
 
-import requests
 import ruamel.yaml
 import schema_salad
 from configargparse import ArgumentParser
 from wes_client.util import WESClient  # type: ignore
 from wes_client.util import wes_reponse as wes_response
 
+from toil.lib.web import web_session
 from toil.wdl.utils import get_version as get_wdl_version
 
 """
@@ -117,7 +117,7 @@ class WESClientWithWorkflowEngineParameters(WESClient):  # type: ignore
         proto, host = endpoint.split("://")
         super().__init__(
             {
-                # TODO: use the auth argument in requests.post so we don't need to encode it ourselves
+                # TODO: use the auth argument in requests' post so we don't need to encode it ourselves
                 "auth": (
                     {
                         "Authorization": "Basic "
@@ -314,7 +314,7 @@ class WESClientWithWorkflowEngineParameters(WESClient):  # type: ignore
         data, files = self.build_wes_request(
             workflow_file, workflow_params_file, attachments, workflow_engine_parameters
         )
-        post_result = requests.post(
+        post_result = web_session.post(
             urljoin(f"{self.proto}://{self.host}", "/ga4gh/wes/v1/runs"),
             data=data,
             files=files,

toil/server/utils.py

@@ -19,10 +19,9 @@ from datetime import datetime
 from typing import Optional
 from urllib.parse import urlparse
 
-import requests
-
 from toil.lib.io import AtomicFileCreate
 from toil.lib.retry import retry
+from toil.lib.web import web_session
 
 try:
     from toil.lib.aws import get_current_aws_region
@@ -63,7 +62,7 @@ def download_file_from_internet(
     """
     Download a file from the Internet and write it to dest.
     """
-    response = requests.get(src)
+    response = web_session.get(src)
 
     if not response.ok:
         raise RuntimeError("Request failed with a client error or a server error.")