toil 6.1.0a1__py3-none-any.whl → 8.0.0__py3-none-any.whl

toil/lib/humanize.py

@@ -25,7 +25,9 @@ def bytes2human(n: SupportsInt) -> str:
     """
     Convert n bytes into a human readable string.
     """
-    logger.warning('Deprecated toil method.  Please use "toil.lib.conversions.bytes2human()" instead."')
+    logger.warning(
+        'Deprecated toil method.  Please use "toil.lib.conversions.bytes2human()" instead."'
+    )
     return b2h(n)
 
 
@@ -36,5 +38,7 @@ def human2bytes(s: str) -> int:
 
     When unable to recognize the format ValueError is raised.
     """
-    logger.warning('Deprecated toil method.  Please use "toil.lib.conversions.human2bytes()" instead."')
+    logger.warning(
+        'Deprecated toil method.  Please use "toil.lib.conversions.human2bytes()" instead."'
+    )
     return h2b(s)

toil/lib/integration.py

@@ -0,0 +1,341 @@
+# Copyright (C) 2024 Regents of the University of California
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Contains functions for integrating Toil with external services such as
+Dockstore.
+"""
+
+import hashlib
+import logging
+import os
+import shutil
+import sys
+import tempfile
+import zipfile
+from typing import Any, Dict, List, Optional, Set, Tuple, cast
+
+from urllib.parse import urlparse, unquote, quote
+import requests
+
+from toil.lib.retry import retry
+from toil.lib.io import file_digest, robust_rmtree
+from toil.version import baseVersion
+
+logger = logging.getLogger(__name__)
+
+# We manage a Requests session at the module level in case we're supposed to be
+# doing cookies, and to send a sensible user agent.
+# We expect the Toil and Python version to not be personally identifiable even
+# in theory (someone might make a new Toil version first, buit there's no way
+# to know for sure that nobody else did the same thing).
+session = requests.Session()
+session.headers.update({"User-Agent": f"Toil {baseVersion} on Python {'.'.join([str(v) for v in sys.version_info])}"})
+
+def is_dockstore_workflow(workflow: str) -> bool:
+    """
+    Returns True if a workflow string smells Dockstore-y.
+
+    Detects Dockstore page URLs and strings that could be Dockstore TRS IDs.
+    """
+
+    return workflow.startswith("https://dockstore.org/workflows/") or workflow.startswith("#workflow/")
+
+def find_trs_spec(workflow: str) -> str:
+    """
+    Parse a Dockstore workflow URL or TSR ID to a string that is definitely a TRS ID.
+    """
+
+    if workflow.startswith("#workflow/"):
+        # Looks like a Dockstore TRS ID already.
+        # TODO: Does Dockstore guartantee we can recognize its TRS IDs like this?
+        logger.debug("Workflow %s is a TRS specifier already", workflow)
+        trs_spec = workflow
+    else:
+        # We need to get the right TRS ID from the Docstore URL
+        parsed = urlparse(workflow)
+        # TODO: We assume the Docksotre page URL structure and the TRS IDs are basically the same.
+        page_path = unquote(parsed.path)
+        if not page_path.startswith("/workflows/"):
+            raise RuntimeError("Cannot parse Dockstore URL " + workflow)
+        trs_spec = "#workflow/" + page_path[len("/workflows/"):]
+        logger.debug("Translated %s to TRS: %s", workflow, trs_spec)
+
+    return trs_spec
+
+def parse_trs_spec(trs_spec: str) -> tuple[str, Optional[str]]:
+    """
+    Parse a TRS ID to workflow and optional version.
+    """
+    parts = trs_spec.split(':', 1)
+    trs_workflow_id = parts[0]
+    if len(parts) > 1:
+        # The ID has the version we want after a colon
+        trs_version = parts[1]
+    else:
+        # We don't know the version we want, we will have to pick one somehow.
+        trs_version = None
+    return trs_workflow_id, trs_version
+
+@retry(errors=[requests.exceptions.ConnectionError])
+def get_workflow_root_from_dockstore(workflow: str, supported_languages: Optional[set[str]] = None) -> str:
+    """
+    Given a Dockstore URL or TRS identifier, get the root WDL or CWL URL for the workflow.
+
+    Accepts inputs like:
+
+        - https://dockstore.org/workflows/github.com/dockstore-testing/md5sum-checker:master?tab=info
+        - #workflow/github.com/dockstore-testing/md5sum-checker
+
+    Assumes the input is actually one of the supported formats. See is_dockstore_workflow().
+
+    TODO: Needs to handle multi-workflow files if Dockstore can.
+
+    """
+
+    if supported_languages is not None and len(supported_languages) == 0:
+        raise ValueError("Set of supported languages must be nonempty if provided.")
+
+    # Get the TRS id[:version] string from what might be a Dockstore URL
+    trs_spec = find_trs_spec(workflow)
+    # Parse out workflow and possible version
+    trs_workflow_id, trs_version = parse_trs_spec(trs_spec)
+
+    logger.debug("TRS %s parses to workflow %s and version %s", trs_spec, trs_workflow_id, trs_version)
+
+    # Fetch the main TRS document.
+    # See e.g. https://dockstore.org/api/ga4gh/trs/v2/tools/%23workflow%2Fgithub.com%2Fdockstore-testing%2Fmd5sum-checker
+    trs_workflow_url = f"https://dockstore.org/api/ga4gh/trs/v2/tools/{quote(trs_workflow_id, safe='')}"
+    trs_workflow_document = session.get(trs_workflow_url).json()
+
+    # Make a map from version to version info. We will need the
+    # "descriptor_type" array to find eligible languages, and the "url" field
+    # to get the version's base URL.
+    workflow_versions: dict[str, dict[str, Any]] = {}
+
+    # We also check which we actually know how to run
+    eligible_workflow_versions: set[str] = set()
+
+    for version_info in trs_workflow_document.get("versions", []):
+        version_name: str = version_info["name"]
+        workflow_versions[version_name] = version_info
+        version_languages: list[str] = version_info["descriptor_type"]
+        if supported_languages is not None:
+            # Filter to versions that have a language we know
+            has_supported_language = False
+            for language in version_languages:
+                if language in supported_languages:
+                    # TODO: Also use "descriptor_type_version" dict to make
+                    # sure we support all needed language versions to actually
+                    # use this workflow version.
+                    has_supported_language = True
+                    continue
+            if not has_supported_language:
+                # Can't actually run this one.
+                continue
+        eligible_workflow_versions.add(version_name)
+
+    for default_version in ['main', 'master']:
+        if trs_version is None and default_version in eligible_workflow_versions:
+            # Fill in a version if the user didn't provide one.
+            trs_version = default_version
+            logger.debug("Defaulting to workflow version %s", default_version)
+            break
+
+    if trs_version is None and len(eligible_workflow_versions) == 1:
+        # If there's just one version use that.
+        trs_version = next(iter(eligible_workflow_versions))
+        logger.debug("Defaulting to only eligible workflow version %s", trs_version)
+
+
+    # If we don't like what we found we compose a useful error message.
+    problems: list[str] = []
+    if trs_version is None:
+        problems.append(f"Workflow {workflow} does not specify a version")
+    elif trs_version not in workflow_versions:
+        problems.append(f"Workflow version {trs_version} from {workflow} does not exist")
+    elif trs_version not in eligible_workflow_versions:
+        message = f"Workflow version {trs_version} from {workflow} is not available"
+        if supported_languages is not None:
+            message += f" in any of: {', '.join(supported_languages)}"
+        problems.append(message)
+    if len(problems) > 0:
+        if len(eligible_workflow_versions) == 0:
+            message = "No versions of the workflow are available"
+            if supported_languages is not None:
+                message += f" in any of: {', '.join(supported_languages)}"
+            problems.append(message)
+        elif trs_version is None:
+            problems.append(f"Add ':' and the name of a workflow version ({', '.join(eligible_workflow_versions)}) after '{trs_workflow_id}'")
+        else:
+            problems.append(f"Replace '{trs_version}' with one of ({', '.join(eligible_workflow_versions)})")
+        raise RuntimeError("; ".join(problems))
+
+    # Tell MyPy we now have a version, or we would have raised
+    assert trs_version is not None
+
+    # Select the language we will actually run
+    chosen_version_languages: list[str] = workflow_versions[trs_version]["descriptor_type"]
+    for candidate_language in chosen_version_languages:
+        if supported_languages is None or candidate_language in supported_languages:
+            language = candidate_language
+
+    logger.debug("Going to use %s version %s in %s", trs_workflow_id, trs_version, language)
+    trs_version_url = workflow_versions[trs_version]["url"]
+
+    # Fetch the list of all the files
+    trs_files_url = f"{trs_version_url}/{language}/files"
+    logger.debug("Workflow files URL: %s", trs_files_url)
+    trs_files_document = session.get(trs_files_url).json()
+
+    # Find the information we need to ID the primary descriptor file
+    primary_descriptor_path: Optional[str] = None
+    primary_descriptor_hash_algorithm: Optional[str] = None
+    primary_descriptor_hash: Optional[str] = None
+    for file_info in trs_files_document:
+        if file_info["file_type"] == "PRIMARY_DESCRIPTOR":
+            primary_descriptor_path = file_info["path"]
+            primary_descriptor_hash_algorithm = file_info["checksum"]["type"]
+            primary_descriptor_hash = file_info["checksum"]["checksum"]
+            break
+    if primary_descriptor_path is None or primary_descriptor_hash is None or primary_descriptor_hash_algorithm is None:
+        raise RuntimeError("Could not find a primary descriptor file for the workflow")
+    primary_descriptor_basename = os.path.basename(primary_descriptor_path)
+
+    # Work out how to compute the hash we are looking for. See
+    # <https://github.com/ga4gh-discovery/ga4gh-checksum/blob/master/hash-alg.csv>
+    # for the GA4GH names and <https://docs.python.org/3/library/hashlib.html>
+    # for the Python names.
+    #
+    # TODO: We don't support the various truncated hash flavors or the other checksums not in hashlib.
+    python_hash_name = primary_descriptor_hash_algorithm.replace("sha-", "sha").replace("blake2b-512", "blake2b").replace("-", "_")
+    if python_hash_name not in hashlib.algorithms_available:
+        raise RuntimeError(f"Primary descriptor is identified by a {primary_descriptor_hash_algorithm} hash but {python_hash_name} is not available in hashlib")
+
+    # Figure out where to store the workflow. We don't want to deal with temp
+    # dir cleanup since we don't want to run the whole workflow setup and
+    # execution in a context manager. So we declare a cache.
+    # Note that it's still not safe to symlink out of this cache since XDG
+    # cache directories aren't guaranteed to be on shared storage.
+    cache_base_dir = os.path.join(os.environ.get("XDG_CACHE_HOME", os.path.expanduser("~/.cache")), "toil/workflows")
+
+    # Hash the workflow file list.
+    hasher = hashlib.sha256()
+    for file_info in sorted(trs_files_document, key=lambda rec: rec["path"]):
+        hasher.update(file_info["path"].encode("utf-8"))
+        hasher.update(b"\0")
+        hasher.update(file_info["checksum"]["type"].encode("utf-8"))
+        hasher.update(b"\0")
+        hasher.update(file_info["checksum"]["checksum"].encode("utf-8"))
+        hasher.update(b"\0")
+    cache_workflow_dir = os.path.join(cache_base_dir, hasher.hexdigest())
+
+    if os.path.exists(cache_workflow_dir):
+        logger.debug("Workflow already cached at %s", cache_workflow_dir)
+    else:
+        # Need to download the workflow
+
+        # Download the ZIP to a temporary file
+        trs_zip_file_url = f"{trs_files_url}?format=zip"
+        logger.debug("Workflow ZIP URL: %s", trs_zip_file_url)
+        with tempfile.NamedTemporaryFile(suffix=".zip") as zip_file:
+            # We want to stream the zip to a file, but when we do it with the Requests
+            # file object like <https://stackoverflow.com/a/39217788> we don't get
+            # Requests' decoding of gzip or deflate response encodings. Since this file
+            # is already compressed the response compression can't help a lot anyway,
+            # so we tell the server that we can't understand it.
+            headers = {
+                "Accept-Encoding": "identity",
+                # Help Dockstore avoid serving ZIP with a JSON content type. See
+                # <https://github.com/dockstore/dockstore/issues/6010>.
+                "Accept": "application/zip"
+            }
+            # If we don't set stream=True, we can't actually read anything from the
+            # raw stream, since Requests will have done it already.
+            with session.get(trs_zip_file_url, headers=headers, stream=True) as response:
+                response_content_length = response.headers.get("Content-Length")
+                logger.debug("Server reports content length: %s", response_content_length)
+                shutil.copyfileobj(response.raw, zip_file)
+            zip_file.flush()
+
+            logger.debug("Downloaded ZIP to %s", zip_file.name)
+
+            # Unzip it to a directory next to where it will live
+            os.makedirs(cache_base_dir, exist_ok=True)
+            workflow_temp_dir = tempfile.mkdtemp(dir=cache_base_dir)
+            with zipfile.ZipFile(zip_file.name, "r") as zip_ref:
+                zip_ref.extractall(workflow_temp_dir)
+            logger.debug("Extracted workflow ZIP to %s", workflow_temp_dir)
+
+            # Try to atomically install into the cache
+            try:
+                os.rename(workflow_temp_dir, cache_workflow_dir)
+                logger.debug("Moved workflow to %s", cache_workflow_dir)
+            except OSError:
+                # Collision. Someone else installed the workflow before we could.
+                robust_rmtree(workflow_temp_dir)
+                logger.debug("Workflow cached at %s by someone else while we were donwloading it", cache_workflow_dir)
+
+    # Hunt throught he directory for a file with the right basename and hash
+    found_path: Optional[str] = None
+    for containing_dir, subdirectories, files in os.walk(cache_workflow_dir):
+        for filename in files:
+            if filename == primary_descriptor_basename:
+                # This could be it. Open the file off disk and hash it with the right algorithm.
+                file_path = os.path.join(containing_dir, filename)
+                file_hash = file_digest(open(file_path, "rb"), python_hash_name).hexdigest()
+                if file_hash == primary_descriptor_hash:
+                    # This looks like the right file
+                    logger.debug("Found candidate primary descriptor %s", file_path)
+                    if found_path is not None:
+                        # But there are multiple instances of it so we can't know which to run.
+                        # TODO: Find out the right path from Dockstore somehow!
+                        raise RuntimeError(f"Workflow contains multiple files named {primary_descriptor_basename} with {python_hash_name} hash {file_hash}: {found_path} and {file_path}")
+                    # This is the first file with the right name and hash
+                    found_path = file_path
+                else:
+                    logger.debug("Rejected %s because its %s hash %s is not %s", file_path, python_hash_name, file_hash, primary_descriptor_hash)
+    if found_path is None:
+        # We couldn't find the promised primary descriptor
+        raise RuntimeError(f"Could not find a {primary_descriptor_basename} with {primary_descriptor_hash_algorithm} hash {primary_descriptor_hash}")
+
+    return found_path
+
+def resolve_workflow(workflow: str, supported_languages: Optional[set[str]] = None) -> str:
+    """
+    Find the real workflow URL or filename from a command line argument.
+
+    Transform a workflow URL or path that might actually be a Dockstore page
+    URL or TRS specifier to an actual URL or path to a workflow document.
+    """
+
+    if is_dockstore_workflow(workflow):
+        # Ask Dockstore where to find Dockstore-y things
+        resolved = get_workflow_root_from_dockstore(workflow, supported_languages=supported_languages)
+        logger.info("Resolved Dockstore workflow %s to %s", workflow, resolved)
+        return resolved
+    else:
+        # Pass other things through.
+        return workflow
+
+
+
+
+
+
+
+
+
+
+

toil/lib/io.py

@@ -1,22 +1,71 @@
+import hashlib
 import logging
 import os
 import shutil
 import stat
+import sys
 import tempfile
 import uuid
+from collections.abc import Iterator
 from contextlib import contextmanager
 from io import BytesIO
-from typing import IO, Any, Callable, Iterator, Optional, Union
+from typing import IO, Any, Callable, Optional, Protocol, Union
 
 logger = logging.getLogger(__name__)
 
-def mkdtemp(suffix: Optional[str] = None, prefix: Optional[str] = None, dir: Optional[str] = None) -> str:
+
+TOIL_URI_SCHEME = "toilfile:"
+
+
+STANDARD_SCHEMES = ["http:", "https:", "s3:", "gs:", "ftp:"]
+REMOTE_SCHEMES = STANDARD_SCHEMES + [TOIL_URI_SCHEME]
+ALL_SCHEMES = REMOTE_SCHEMES + ["file:"]
+
+def is_standard_url(filename: str) -> bool:
+    return is_url_with_scheme(filename, STANDARD_SCHEMES)
+
+def is_remote_url(filename: str) -> bool:
+    """
+    Decide if a filename is a known, non-file kind of URL
+    """
+    return is_url_with_scheme(filename, REMOTE_SCHEMES)
+
+def is_any_url(filename: str) -> bool:
+    """
+    Decide if a string is a URI like http:// or file://.
+
+    Otherwise it might be a bare path.
+    """
+    return is_url_with_scheme(filename, ALL_SCHEMES)
+
+def is_url_with_scheme(filename: str, schemes: list[str]) -> bool:
+    """
+    Return True if filename is a URL with any of the given schemes and False otherwise.
+    """
+    # TODO: "http:myfile.dat" is a valid filename and *not* a valid URL
+    for scheme in schemes:
+        if filename.startswith(scheme):
+            return True
+    return False
+
+def is_toil_url(filename: str) -> bool:
+    return is_url_with_scheme(filename, [TOIL_URI_SCHEME])
+
+def is_file_url(filename: str) -> bool:
+    return is_url_with_scheme(filename, ["file:"])
+
+
+def mkdtemp(
+    suffix: Optional[str] = None,
+    prefix: Optional[str] = None,
+    dir: Optional[str] = None,
+) -> str:
     """
     Make a temporary directory like tempfile.mkdtemp, but with relaxed permissions.
 
     The permissions on the directory will be 711 instead of 700, allowing the
     group and all other users to traverse the directory. This is necessary if
-    the direcotry is on NFS and the Docker daemon would like to mount it or a
+    the directory is on NFS and the Docker daemon would like to mount it or a
     file inside it into a container, because on NFS even the Docker daemon
     appears bound by the file permissions.
 
@@ -27,10 +76,13 @@ def mkdtemp(suffix: Optional[str] = None, prefix: Optional[str] = None, dir: Opt
     # Make the directory
     result = tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=dir)
     # Grant all the permissions: full control for user, and execute for group and other
-    os.chmod(result, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+    os.chmod(
+        result, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
+    )
     # Return the path created
     return result
 
+
 def robust_rmtree(path: Union[str, bytes]) -> None:
     """
     Robustly tries to delete paths.
@@ -45,7 +97,7 @@ def robust_rmtree(path: Union[str, bytes]) -> None:
     if not isinstance(path, bytes):
         # Internally we must work in bytes, in case we find an undecodeable
         # filename.
-        path = path.encode('utf-8')
+        path = path.encode("utf-8")
 
     if not os.path.exists(path):
         # Nothing to do!
@@ -107,7 +159,7 @@ def atomic_tmp_file(final_path: str) -> str:
     as finalPath.  It the final path is in /dev (/dev/null, /dev/stdout), it is
     returned unchanged and atomic_tmp_install will do nothing."""
     final_dir = os.path.dirname(os.path.normpath(final_path))  # can be empty
-    if final_dir == '/dev':
+    if final_dir == "/dev":
         return final_path
     final_basename = os.path.basename(final_path)
     final_ext = os.path.splitext(final_path)[1]
@@ -117,9 +169,10 @@ def atomic_tmp_file(final_path: str) -> str:
 
 def atomic_install(tmp_path, final_path) -> None:
     """atomic install of tmp_path as final_path"""
-    if os.path.dirname(os.path.normpath(final_path)) != '/dev':
+    if os.path.dirname(os.path.normpath(final_path)) != "/dev":
         os.rename(tmp_path, final_path)
 
+
 @contextmanager
 def AtomicFileCreate(final_path: str, keep: bool = False) -> Iterator[str]:
     """Context manager to create a temporary file.  Entering returns path to
@@ -140,7 +193,9 @@ def AtomicFileCreate(final_path: str, keep: bool = False) -> Iterator[str]:
         raise
 
 
-def atomic_copy(src_path: str, dest_path: str, executable: Optional[bool] = None) -> None:
+def atomic_copy(
+    src_path: str, dest_path: str, executable: Optional[bool] = None
+) -> None:
     """Copy a file using posix atomic creations semantics."""
     if executable is None:
         executable = os.stat(src_path).st_mode & stat.S_IXUSR != 0
@@ -150,26 +205,42 @@ def atomic_copy(src_path: str, dest_path: str, executable: Optional[bool] = None
             os.chmod(dest_path_tmp, os.stat(dest_path_tmp).st_mode | stat.S_IXUSR)
 
 
-def atomic_copyobj(src_fh: BytesIO, dest_path: str, length: int = 16384, executable: bool = False) -> None:
+def atomic_copyobj(
+    src_fh: BytesIO, dest_path: str, length: int = 16384, executable: bool = False
+) -> None:
     """Copy an open file using posix atomic creations semantics."""
     with AtomicFileCreate(dest_path) as dest_path_tmp:
-        with open(dest_path_tmp, 'wb') as dest_path_fh:
+        with open(dest_path_tmp, "wb") as dest_path_fh:
             shutil.copyfileobj(src_fh, dest_path_fh, length=length)
         if executable:
             os.chmod(dest_path_tmp, os.stat(dest_path_tmp).st_mode | stat.S_IXUSR)
 
 
-def make_public_dir(in_directory: Optional[str] = None) -> str:
+def make_public_dir(in_directory: str, suggested_name: Optional[str] = None) -> str:
     """
+    Make a publicly-accessible directory in the given directory.
+
+    :param suggested_name: Use this directory name first if possible.
+
     Try to make a random directory name with length 4 that doesn't exist, with the given prefix.
     Otherwise, try length 5, length 6, etc, up to a max of 32 (len of uuid4 with dashes replaced).
     This function's purpose is mostly to avoid having long file names when generating directories.
     If somehow this fails, which should be incredibly unlikely, default to a normal uuid4, which was
     our old default.
     """
-    for i in range(4, 32 + 1):  # make random uuids and truncate to lengths starting at 4 and working up to max 32
+    if suggested_name is not None:
+        generated_dir_path: str = os.path.join(in_directory, suggested_name)
+        try:
+            os.mkdir(generated_dir_path)
+            os.chmod(generated_dir_path, 0o777)
+            return generated_dir_path
+        except FileExistsError:
+            pass
+    for i in range(
+        4, 32 + 1
+    ):  # make random uuids and truncate to lengths starting at 4 and working up to max 32
         for _ in range(10):  # make 10 attempts for each length
-            truncated_uuid: str = str(uuid.uuid4()).replace('-', '')[:i]
+            truncated_uuid: str = str(uuid.uuid4()).replace("-", "")[:i]
             generated_dir_path: str = os.path.join(in_directory, truncated_uuid)
             try:
                 os.mkdir(generated_dir_path)
@@ -182,17 +253,44 @@ def make_public_dir(in_directory: Optional[str] = None) -> str:
     os.chmod(this_should_never_happen, 0o777)
     return this_should_never_happen
 
-def try_path(path: str) -> Optional[str]:
+
+def try_path(path: str, min_size: int = 100 * 1024 * 1024) -> Optional[str]:
     """
     Try to use the given path. Return it if it exists or can be made,
     and we can make things within it, or None otherwise.
+
+    :param min_size: Reject paths on filesystems smaller than this many bytes.
     """
+
     try:
         os.makedirs(path, exist_ok=True)
     except OSError:
         # Maybe we lack permissions
         return None
-    return path if os.path.exists(path) and os.access(path, os.W_OK) else None
+
+    if not os.path.exists(path):
+        # We didn't manage to make it
+        return None
+
+    if not os.access(path, os.W_OK):
+        # It doesn't look writable
+        return None
+
+    try:
+        stats = os.statvfs(path)
+    except OSError:
+        # Maybe we lack permissions
+        return None
+
+    # Is the filesystem big enough?
+    # We need to look at the FS size and not the free space so we don't change
+    # over to a different filesystem when this one fills up.
+    fs_size = stats.f_frsize * stats.f_blocks
+    if fs_size < min_size:
+        # Too small
+        return None
+
+    return path
 
 
 class WriteWatchingStream:
@@ -253,3 +351,31 @@ class WriteWatchingStream:
         """
 
         self.backingStream.close()
+
+class ReadableFileObj(Protocol):
+    """
+    Protocol that is more specific than what file_digest takes as an argument.
+    Also guarantees a read() method.
+    Would extend the protocol from Typeshed for hashlib but those are only
+    declared for 3.11+.
+    """
+    def readinto(self, buf: bytearray, /) -> int: ...
+    def readable(self) -> bool: ...
+    def read(self, number: int) -> bytes: ...
+
+# hashlib._Hash seems to not appear at runtime
+def file_digest(f: ReadableFileObj, alg_name: str) -> "hashlib._Hash":
+    """
+    Polyfilled hashlib.file_digest that works on Python <3.11.
+    """
+    if sys.version_info >= (3, 11):
+        return hashlib.file_digest(f, alg_name)
+    BUFFER_SIZE = 1024 * 1024
+    hasher = hashlib.new(alg_name)
+    buffer = f.read(BUFFER_SIZE)
+    while buffer:
+        hasher.update(buffer)
+        buffer = f.read(BUFFER_SIZE)
+    return hasher
+
+

toil/lib/iterables.py

@@ -12,8 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from collections.abc import Iterable, Iterator
+
 # 5.14.2018: copied into Toil from https://github.com/BD2KGenomics/bd2k-python-lib
-from typing import Any, Iterable, Iterator, TypeVar
+from typing import Any, TypeVar
 
 IT = TypeVar("IT")
 
@@ -102,7 +104,7 @@ class concat:
                 try:
                     i = x.__iter__()
                 except AttributeError:
-                    i = x,
+                    i = (x,)
             else:
                 i = x
             return i