toil 8.1.0b1__py3-none-any.whl → 8.2.0__py3-none-any.whl

toil/lib/history.py

@@ -27,6 +27,7 @@ import uuid
 from dataclasses import dataclass
 from typing import Any, Iterable, Iterator, Optional, TypeVar, Callable
 
+from toil.lib.conversions import strtobool
 from toil.lib.io import get_toil_home
 from toil.lib.retry import ErrorCondition, retry
 
@@ -126,15 +127,26 @@ class HistoryManager:
     Class responsible for managing the history of Toil runs.
     """
 
-    # Should workflow run history be recorded?
-    WORKFLOW_HISTORY_ENABLED = True
-    # Should job history be recorded? Can only be true if
-    # WORKFLOW_HISTORY_ENABLED is also true.
-    #
-    # TODO: When Dockstore can take job metrics alongside whole-workflow
-    # metrics, and we've tested to make sure history recording doesn't slow
-    # down our leader job processing rate, turn on actual job history logging.
-    JOB_HISTORY_ENABLED = False
+    @classmethod
+    def enabled(cls) -> bool:
+        """
+        Return True if history should be read from and written to the database.
+
+        If False, no access at all shoulf be made to the database.
+        """
+        return strtobool(os.environ.get("TOIL_HISTORY", 'True')) 
+
+    @classmethod
+    def enabled_job(cls) -> bool:
+        """
+        Return True if job history should be read from and written to the database.
+
+        Always returns False if enabled() returns False.
+        """
+        # TODO: When Dockstore can take job metrics alongside whole-workflow
+        # metrics, and we've tested to make sure history recording doesn't slow
+        # down our leader job processing rate, turn on actual job history logging.
+        return cls.enabled() and strtobool(os.environ.get("TOIL_JOB_HISTORY", 'False')) 
 
     # For testing, we can move the database path for the class.
     database_path_override: Optional[str] = None
@@ -164,6 +176,12 @@ class HistoryManager:
         on Python versions that support it. In order to run any commands
         outside of a transaction use the no_transaction context manager.
         """
+
+        if not cls.enabled():
+            # Make sure we're not missing an enabled check along any codepath
+            # that wants to access the database.
+            raise RuntimeError("Attempting to connect to database when HistoryManager is disabled!")
+
         if not os.path.exists(cls.database_path()):
             # Make the database and protect it from snoopers and busybodies
             con = sqlite3.connect(cls.database_path())
@@ -345,7 +363,7 @@ class HistoryManager:
             updated.
         """
 
-        if not cls.WORKFLOW_HISTORY_ENABLED:
+        if not cls.enabled():
             return
 
         logger.info("Recording workflow creation of %s in %s", workflow_id, job_store_spec)
@@ -375,7 +393,7 @@ class HistoryManager:
 
         # TODO: Make name of this function less general?
 
-        if not cls.WORKFLOW_HISTORY_ENABLED:
+        if not cls.enabled():
             return
 
         logger.info("Workflow %s is a run of %s", workflow_id, workflow_name)
@@ -431,7 +449,7 @@ class HistoryManager:
         :param disk_bytes: Observed job disk usage.
         """
 
-        if not cls.WORKFLOW_HISTORY_ENABLED or not cls.JOB_HISTORY_ENABLED:
+        if not cls.enabled_job():
             return
 
         logger.debug("Workflow %s ran job %s", workflow_id, job_name)
@@ -506,7 +524,7 @@ class HistoryManager:
         :param platform_machine: CPU type ("AMD64", etc.) used to run the workflow leader.
         """
 
-        if not cls.WORKFLOW_HISTORY_ENABLED:
+        if not cls.enabled():
             return
 
         logger.info("Workflow %s stopped. Success: %s", workflow_id, succeeded)
@@ -577,6 +595,9 @@ class HistoryManager:
         List all known workflows and their summary statistics.
         """
 
+        if not cls.enabled():
+            return []
+
         workflows = []
 
         con = cls.connection()
@@ -632,6 +653,9 @@ class HistoryManager:
         :param limit: Get no more than this many.
         """
 
+        if not cls.enabled():
+            return []
+
         attempts = []
 
         con = cls.connection()
@@ -706,6 +730,9 @@ class HistoryManager:
         :param limit: Get no more than this many.
         """
 
+        if not cls.enabled_job():
+            return []
+
         attempts = []
 
         con = cls.connection()
@@ -783,6 +810,9 @@ class HistoryManager:
 
         # TODO: Consolidate with the other 2 ways to query workflow attempts!
 
+        if not cls.enabled():
+            return None
+
         attempts = []
 
         con = cls.connection()
@@ -857,6 +887,9 @@ class HistoryManager:
         Doesn't check to make sure the workflow has a TRS ID.
         """
 
+        if not cls.enabled_job():
+            return []
+
         attempts = []
 
         con = cls.connection()
@@ -922,6 +955,9 @@ class HistoryManager:
         Does not mark the workflow attempt's job attempts as submitted.
         """
 
+        if not cls.enabled():
+            return
+
         con = cls.connection()
         cur = con.cursor()
         try:
@@ -945,6 +981,9 @@ class HistoryManager:
         Mark a collection of job attempts as submitted to Dockstore in a single transaction.
         """
 
+        if not cls.enabled_job():
+            return
+
         con = cls.connection()
         cur = con.cursor()
         try:
@@ -969,6 +1008,10 @@ class HistoryManager:
         """
         Count workflows in the database.
         """
+        
+        if not cls.enabled():
+            return 0
+
         con = cls.connection()
         cur = con.cursor()
         try:
@@ -994,6 +1037,10 @@ class HistoryManager:
         """
         Count workflow attempts in the database.
         """
+
+        if not cls.enabled():
+            return 0
+
         con = cls.connection()
         cur = con.cursor()
         try:
@@ -1019,6 +1066,10 @@ class HistoryManager:
         """
         Count job attempts in the database.
         """
+
+        if not cls.enabled_job():
+            return 0
+
         con = cls.connection()
         cur = con.cursor()
         try:
@@ -1044,6 +1095,10 @@ class HistoryManager:
         """
         Get workflows that have a successful attempt and no unsubmitted attempts or job attempts.
         """
+
+        if not cls.enabled():
+            return []
+
         ids = []
 
         con = cls.connection()
@@ -1105,6 +1160,9 @@ class HistoryManager:
         Get workflows that are old.
         """
 
+        if not cls.enabled():
+            return []
+
         ids = []
 
         con = cls.connection()
@@ -1150,6 +1208,9 @@ class HistoryManager:
         Succeeds if the workflow does not exist.
         """
 
+        if not cls.enabled():
+            return
+
         con = cls.connection()
         cur = con.cursor()
         try:
@@ -1173,6 +1234,9 @@ class HistoryManager:
         Get the total number of bytes used by the database.
         """
 
+        if not cls.enabled():
+            return 0
+
         con = cls.connection()
         cur = con.cursor()
         try:
@@ -1202,6 +1266,9 @@ class HistoryManager:
         """
         Shrink the database to remove unused space.
         """
+
+        if not cls.enabled():
+            return
         
         con = cls.connection()
         cur = con.cursor()
@@ -1226,6 +1293,9 @@ class HistoryManager:
         important.
         """
 
+        if not cls.enabled():
+            return
+
         db_size = cls.get_database_byte_size()
 
         if db_size < limit:
@@ -1264,6 +1334,10 @@ class HistoryManager:
 
         For debugging tests.
         """
+
+        if not cls.enabled():
+            return []
+
         return cls.connection().iterdump()
 
 

toil/lib/history_submission.py

@@ -82,7 +82,7 @@ def job_execution_id(job_attempt: JobAttemptSummary) -> str:
 def get_parsed_trs_spec(workflow_attempt: WorkflowAttemptSummary) -> tuple[str, str]:
     """
     Get the TRS ID and version of the workflow, or raise an error.
-    
+
     :returns: The TRS ID and the TRS version of the wrokflow run.
     :raises: ValueError if the workflow does not have a TRS spec or if the spec
         does not contain a version.
@@ -259,7 +259,7 @@ class Submission:
         return all_submitted_and_marked
 
 
-def create_history_submission(batch_size: int = 10, desired_tasks: int = 0) -> Submission:
+def create_history_submission(batch_size: Optional[int] = None, desired_tasks: Optional[int] = None) -> Submission:
     """
     Make a package of data about recent workflow runs to send in.
 
@@ -271,6 +271,12 @@ def create_history_submission(batch_size: int = 10, desired_tasks: int = 0) -> S
         batch. Use 0 to not submit any task information.
     """
 
+    # By default, include the things we are set to track history for.
+    if batch_size is None:
+        batch_size = 10 if HistoryManager.enabled() else 0
+    if desired_tasks is None:
+        desired_tasks = 50 if HistoryManager.enabled_job() else 0
+
     # Collect together some workflows and some lists of tasks into a submission.
     submission = Submission()
 
@@ -325,15 +331,16 @@ def create_current_submission(workflow_id: str, attempt_number: int) -> Submissi
     submission = Submission()
     try:
         workflow_attempt = HistoryManager.get_workflow_attempt(workflow_id, attempt_number)
-        if workflow_attempt is not None:
+        if workflow_attempt is not None and HistoryManager.enabled():
             if not workflow_attempt.submitted_to_dockstore:
                 submission.add_workflow_attempt(workflow_attempt)
-            try:
-                job_attempts = HistoryManager.get_unsubmitted_job_attempts(workflow_attempt.workflow_id, workflow_attempt.attempt_number)
-                submission.add_job_attempts(workflow_attempt, job_attempts)
-            except:
-                logger.exception("Could not compose metrics report for workflow task set")
-                # Keep going with just the workflow.
+            if HistoryManager.enabled_job():
+                try:
+                    job_attempts = HistoryManager.get_unsubmitted_job_attempts(workflow_attempt.workflow_id, workflow_attempt.attempt_number)
+                    submission.add_job_attempts(workflow_attempt, job_attempts)
+                except:
+                    logger.exception("Could not compose metrics report for workflow task set")
+                    # Keep going with just the workflow.
     except:
         logger.exception("Could not compose metrics report for workflow execution")
         # Keep going with an empty submission.
@@ -493,6 +500,13 @@ def display_dialog_tkinter(title: str, text: str, options: dict[KeyType, str], t
             # If we run out of time, hide the window and move on without a choice.
             root.after(int(timeout * 1000), close_root)
 
+        # To make the dialog pop up over the terminal instead of behind it, we
+        # lift it and temporarily make it topmost. We don't keep it topmost
+        # because we want to let the user switch away from it.
+        root.attributes('-topmost', True)
+        root.after(10, lambda: root.attributes('-topmost', False))
+        root.lift()
+
         # Run the window's main loop
         root.mainloop()
 

toil/lib/io.py

@@ -6,15 +6,17 @@ import stat
 import sys
 import tempfile
 import uuid
-from collections.abc import Iterator
+from collections.abc import Iterator, Iterable
 from contextlib import contextmanager
 from io import BytesIO
 from typing import IO, Any, Callable, Optional, Protocol, Union
 
 from toil.lib.memoize import memoize
+from toil.lib.misc import StrPath
 
 logger = logging.getLogger(__name__)
 
+
 @memoize
 def get_toil_home() -> str:
     """
@@ -43,6 +45,9 @@ REMOTE_SCHEMES = STANDARD_SCHEMES + [TOIL_URI_SCHEME]
 ALL_SCHEMES = REMOTE_SCHEMES + ["file:"]
 
 def is_standard_url(filename: str) -> bool:
+    """
+    Return True if the given URL is a non-Toil, non-file: URL.
+    """
     return is_url_with_scheme(filename, STANDARD_SCHEMES)
 
 def is_remote_url(filename: str) -> bool:
@@ -70,16 +75,23 @@ def is_url_with_scheme(filename: str, schemes: list[str]) -> bool:
     return False
 
 def is_toil_url(filename: str) -> bool:
+    """
+    Return True if a URL is a toilfile: URL.
+    """
     return is_url_with_scheme(filename, [TOIL_URI_SCHEME])
 
 def is_file_url(filename: str) -> bool:
-    return is_url_with_scheme(filename, ["file:"])
+    """
+    Return True if a URL is a file: URL.
 
+    Will return False for bare paths.
+    """
+    return is_url_with_scheme(filename, ["file:"])
 
 def mkdtemp(
     suffix: Optional[str] = None,
     prefix: Optional[str] = None,
-    dir: Optional[str] = None,
+    dir: Optional[StrPath] = None,
 ) -> str:
     """
     Make a temporary directory like tempfile.mkdtemp, but with relaxed permissions.
@@ -174,28 +186,28 @@ def robust_rmtree(path: Union[str, bytes]) -> None:
             raise
 
 
-def atomic_tmp_file(final_path: str) -> str:
+def atomic_tmp_file(final_path: StrPath) -> str:
     """Return a tmp file name to use with atomic_install.  This will be in the
     same directory as final_path. The temporary file will have the same extension
     as finalPath.  It the final path is in /dev (/dev/null, /dev/stdout), it is
     returned unchanged and atomic_tmp_install will do nothing."""
     final_dir = os.path.dirname(os.path.normpath(final_path))  # can be empty
     if final_dir == "/dev":
-        return final_path
+        return str(final_path)
     final_basename = os.path.basename(final_path)
     final_ext = os.path.splitext(final_path)[1]
-    base_name = f"{final_basename}.{uuid.uuid4()}.tmp{final_ext}"
+    base_name = f"{final_basename}.{str(uuid.uuid4())}.tmp{final_ext}"
     return os.path.join(final_dir, base_name)
 
 
-def atomic_install(tmp_path, final_path) -> None:
+def atomic_install(tmp_path: StrPath, final_path: StrPath) -> None:
     """atomic install of tmp_path as final_path"""
     if os.path.dirname(os.path.normpath(final_path)) != "/dev":
         os.rename(tmp_path, final_path)
 
 
 @contextmanager
-def AtomicFileCreate(final_path: str, keep: bool = False) -> Iterator[str]:
+def AtomicFileCreate(final_path: StrPath, keep: bool = False) -> Iterator[str]:
     """Context manager to create a temporary file.  Entering returns path to
     the temporary file in the same directory as finalPath.  If the code in
     context succeeds, the file renamed to its actual name.  If an error
@@ -250,23 +262,23 @@ def make_public_dir(in_directory: str, suggested_name: Optional[str] = None) ->
     our old default.
     """
     if suggested_name is not None:
-        generated_dir_path: str = os.path.join(in_directory, suggested_name)
+        generated_dir_path1 = os.path.join(in_directory, suggested_name)
         try:
-            os.mkdir(generated_dir_path)
-            os.chmod(generated_dir_path, 0o777)
-            return generated_dir_path
+            os.mkdir(generated_dir_path1)
+            os.chmod(generated_dir_path1, 0o777)
+            return generated_dir_path1
         except FileExistsError:
             pass
     for i in range(
         4, 32 + 1
     ):  # make random uuids and truncate to lengths starting at 4 and working up to max 32
         for _ in range(10):  # make 10 attempts for each length
-            truncated_uuid: str = str(uuid.uuid4()).replace("-", "")[:i]
-            generated_dir_path: str = os.path.join(in_directory, truncated_uuid)
+            truncated_uuid = str(uuid.uuid4()).replace("-", "")[:i]
+            generated_dir_path2 = os.path.join(in_directory, truncated_uuid)
             try:
-                os.mkdir(generated_dir_path)
-                os.chmod(generated_dir_path, 0o777)
-                return generated_dir_path
+                os.mkdir(generated_dir_path2)
+                os.chmod(generated_dir_path2, 0o777)
+                return generated_dir_path2
             except FileExistsError:
                 pass
     this_should_never_happen: str = os.path.join(in_directory, str(uuid.uuid4()))
@@ -328,7 +340,7 @@ class WriteWatchingStream:
 
         self.backingStream = backingStream
         # We have no write listeners yet
-        self.writeListeners = []
+        self.writeListeners: list[Callable[[int], None]] = []
 
     def onWrite(self, listener: Callable[[int], None]) -> None:
         """
@@ -339,7 +351,7 @@ class WriteWatchingStream:
 
     # Implement the file API from https://docs.python.org/2.4/lib/bltin-file-objects.html
 
-    def write(self, data):
+    def write(self, data: bytes) -> None:
         """
         Write the given data to the file.
         """
@@ -351,7 +363,7 @@ class WriteWatchingStream:
             # Send out notifications
             listener(len(data))
 
-    def writelines(self, datas):
+    def writelines(self, datas: Iterable[bytes]) -> None:
         """
         Write each string from the given iterable, without newlines.
         """
@@ -359,14 +371,14 @@ class WriteWatchingStream:
         for data in datas:
             self.write(data)
 
-    def flush(self):
+    def flush(self) -> None:
         """
         Flush the backing stream.
         """
 
         self.backingStream.flush()
 
-    def close(self):
+    def close(self) -> None:
         """
         Close the backing stream.
         """

toil/lib/misc.py

@@ -9,10 +9,16 @@ import sys
 import time
 from collections.abc import Iterator
 from contextlib import closing
-from typing import Optional
+from typing import Optional, Union
+if sys.version_info >= (3, 10):
+    from typing import TypeAlias
+else:
+    from typing_extensions import TypeAlias
 
 logger = logging.getLogger(__name__)
 
+StrPath: TypeAlias = Union[str, os.PathLike[str]]
+FileDescriptorOrPath: TypeAlias = Union[int, bytes, os.PathLike[bytes], StrPath]
 
 def get_public_ip() -> str:
     """Get the IP that this machine uses to contact the internet.

toil/lib/resources.py

@@ -17,6 +17,7 @@ import os
 import resource
 import sys
 
+from toil.lib.misc import StrPath
 
 class ResourceMonitor:
     """
@@ -89,7 +90,7 @@ class ResourceMonitor:
         )
 
 
-def glob(glob_pattern: str, directoryname: str) -> list[str]:
+def glob(glob_pattern: str, directoryname: StrPath) -> list[str]:
     """
     Walks through a directory and its subdirectories looking for files matching
     the glob_pattern and returns a list=[].

toil/lib/threading.py

@@ -36,12 +36,13 @@ import psutil
 
 from toil.lib.exceptions import raise_
 from toil.lib.io import robust_rmtree
+from toil.lib.misc import StrPath
 
 logger = logging.getLogger(__name__)
 
 
 def ensure_filesystem_lockable(
-    path: str, timeout: float = 30, hint: Optional[str] = None
+    path: StrPath, timeout: float = 30, hint: Optional[str] = None
 ) -> None:
     """
     Make sure that the filesystem used at the given path is one where locks are safe to use.
@@ -71,7 +72,7 @@ def ensure_filesystem_lockable(
             # Start a child process to stat the path. See <https://unix.stackexchange.com/a/402236>.
             # We really should call statfs but no bindings for it are in PyPI.
             completed = subprocess.run(
-                ["stat", "-f", "-c", "%T", path],
+                ["stat", "-f", "-c", "%T", str(path)],
                 check=True,
                 capture_output=True,
                 timeout=timeout,
@@ -85,7 +86,7 @@ def ensure_filesystem_lockable(
             # Stat didn't work. Maybe we don't have the right version of stat installed?
             logger.warning(
                 "Could not determine filesystem type at %s because of: %s",
-                path,
+                str(path),
                 e.stderr.decode("utf-8", errors="replace").strip(),
             )
             # If we don't know the filesystem type, keep going anyway.
@@ -107,7 +108,7 @@ def ensure_filesystem_lockable(
             # flaky with regard to locks actually locking anything).
             logger.debug(
                 "Detected that %s has lockable filesystem type: %s",
-                path,
+                str(path),
                 filesystem_type,
             )
 
@@ -518,7 +519,7 @@ def process_name_exists(base_dir: str, name: str) -> bool:
 # Similar to the process naming system above, we define a global mutex system
 # for critical sections, based just around file locks.
 @contextmanager
-def global_mutex(base_dir: str, mutex: str) -> Iterator[None]:
+def global_mutex(base_dir: StrPath, mutex: str) -> Iterator[None]:
     """
     Context manager that locks a mutex. The mutex is identified by the given
     name, and scoped to the given directory. Works across all containers that
@@ -527,7 +528,7 @@ def global_mutex(base_dir: str, mutex: str) -> Iterator[None]:
 
     Only works between processes, NOT between threads.
 
-    :param str base_dir: Base directory to work in. Defines the shared namespace.
+    :param base_dir: Base directory to work in. Defines the shared namespace.
     :param str mutex: Mutex to lock. Must be a permissible path component.
     """
 
@@ -674,7 +675,7 @@ class LastProcessStandingArena:
     Consider using a try/finally; this class is not a context manager.
     """
 
-    def __init__(self, base_dir: str, name: str) -> None:
+    def __init__(self, base_dir: StrPath, name: str) -> None:
         """
         Connect to the arena specified by the given base_dir and name.
 
@@ -683,12 +684,12 @@ class LastProcessStandingArena:
 
         Doesn't enter or leave the arena.
 
-        :param str base_dir: Base directory to work in. Defines the shared namespace.
-        :param str name: Name of the arena. Must be a permissible path component.
+        :param base_dir: Base directory to work in. Defines the shared namespace.
+        :param name: Name of the arena. Must be a permissible path component.
         """
 
         # Save the base_dir which namespaces everything
-        self.base_dir = base_dir
+        self.base_dir = str(base_dir)
 
         # We need a mutex name to allow only one process to be entering or
         # leaving at a time.

toil/options/common.py

@@ -860,6 +860,14 @@ def add_base_toil_options(
         help=f"Number of times to retry a failing job before giving up and "
         f"labeling job failed. default={1}",
     )
+    job_options.add_argument(
+        "--stopOnFirstFailure",
+        dest="stop_on_first_failure",
+        type=strtobool,
+        default=False,
+        metavar="BOOL",
+        help="Stop the workflow at the first complete job failure.",
+    )
     job_options.add_argument(
         "--enableUnlimitedPreemptibleRetries",
         "--enableUnlimitedPreemptableRetries",

toil/options/wdl.py

@@ -86,3 +86,14 @@ def add_wdl_options(parser: ArgumentParser, suppress: bool = True) -> None:
         default=None,
         help=suppress_help or "Keep and return all call outputs as workflow outputs"
     )
+
+    strict_arguments = ["--wdlStrict"] + (
+        ["--strict"] if not suppress else []
+    )
+    parser.add_argument(
+        *strict_arguments,
+        dest="strict",
+        type=strtobool,
+        default=False,
+        help=suppress_help or "Exit runner if workflow has any lint warnings"
+    )