toil 8.1.0b1__py3-none-any.whl → 9.0.0__py3-none-any.whl

toil/lib/history.py

@@ -27,6 +27,7 @@ import uuid
 from dataclasses import dataclass
 from typing import Any, Iterable, Iterator, Optional, TypeVar, Callable
 
+from toil.lib.conversions import strtobool
 from toil.lib.io import get_toil_home
 from toil.lib.retry import ErrorCondition, retry
 
@@ -126,15 +127,26 @@ class HistoryManager:
     Class responsible for managing the history of Toil runs.
     """
 
-    # Should workflow run history be recorded?
-    WORKFLOW_HISTORY_ENABLED = True
-    # Should job history be recorded? Can only be true if
-    # WORKFLOW_HISTORY_ENABLED is also true.
-    #
-    # TODO: When Dockstore can take job metrics alongside whole-workflow
-    # metrics, and we've tested to make sure history recording doesn't slow
-    # down our leader job processing rate, turn on actual job history logging.
-    JOB_HISTORY_ENABLED = False
+    @classmethod
+    def enabled(cls) -> bool:
+        """
+        Return True if history should be read from and written to the database.
+
+        If False, no access at all shoulf be made to the database.
+        """
+        return strtobool(os.environ.get("TOIL_HISTORY", 'True')) 
+
+    @classmethod
+    def enabled_job(cls) -> bool:
+        """
+        Return True if job history should be read from and written to the database.
+
+        Always returns False if enabled() returns False.
+        """
+        # TODO: When Dockstore can take job metrics alongside whole-workflow
+        # metrics, and we've tested to make sure history recording doesn't slow
+        # down our leader job processing rate, turn on actual job history logging.
+        return cls.enabled() and strtobool(os.environ.get("TOIL_JOB_HISTORY", 'False')) 
 
     # For testing, we can move the database path for the class.
     database_path_override: Optional[str] = None
@@ -164,6 +176,12 @@ class HistoryManager:
         on Python versions that support it. In order to run any commands
         outside of a transaction use the no_transaction context manager.
         """
+
+        if not cls.enabled():
+            # Make sure we're not missing an enabled check along any codepath
+            # that wants to access the database.
+            raise RuntimeError("Attempting to connect to database when HistoryManager is disabled!")
+
         if not os.path.exists(cls.database_path()):
             # Make the database and protect it from snoopers and busybodies
             con = sqlite3.connect(cls.database_path())
@@ -345,7 +363,7 @@ class HistoryManager:
             updated.
         """
 
-        if not cls.WORKFLOW_HISTORY_ENABLED:
+        if not cls.enabled():
             return
 
         logger.info("Recording workflow creation of %s in %s", workflow_id, job_store_spec)
@@ -375,7 +393,7 @@ class HistoryManager:
 
         # TODO: Make name of this function less general?
 
-        if not cls.WORKFLOW_HISTORY_ENABLED:
+        if not cls.enabled():
             return
 
         logger.info("Workflow %s is a run of %s", workflow_id, workflow_name)
@@ -431,7 +449,7 @@ class HistoryManager:
         :param disk_bytes: Observed job disk usage.
         """
 
-        if not cls.WORKFLOW_HISTORY_ENABLED or not cls.JOB_HISTORY_ENABLED:
+        if not cls.enabled_job():
             return
 
         logger.debug("Workflow %s ran job %s", workflow_id, job_name)
@@ -506,7 +524,7 @@ class HistoryManager:
         :param platform_machine: CPU type ("AMD64", etc.) used to run the workflow leader.
         """
 
-        if not cls.WORKFLOW_HISTORY_ENABLED:
+        if not cls.enabled():
             return
 
         logger.info("Workflow %s stopped. Success: %s", workflow_id, succeeded)
@@ -577,6 +595,9 @@ class HistoryManager:
         List all known workflows and their summary statistics.
         """
 
+        if not cls.enabled():
+            return []
+
         workflows = []
 
         con = cls.connection()
@@ -632,6 +653,9 @@ class HistoryManager:
         :param limit: Get no more than this many.
         """
 
+        if not cls.enabled():
+            return []
+
         attempts = []
 
         con = cls.connection()
@@ -706,6 +730,9 @@ class HistoryManager:
         :param limit: Get no more than this many.
         """
 
+        if not cls.enabled_job():
+            return []
+
         attempts = []
 
         con = cls.connection()
@@ -783,6 +810,9 @@ class HistoryManager:
 
         # TODO: Consolidate with the other 2 ways to query workflow attempts!
 
+        if not cls.enabled():
+            return None
+
         attempts = []
 
         con = cls.connection()
@@ -857,6 +887,9 @@ class HistoryManager:
         Doesn't check to make sure the workflow has a TRS ID.
         """
 
+        if not cls.enabled_job():
+            return []
+
         attempts = []
 
         con = cls.connection()
@@ -922,6 +955,9 @@ class HistoryManager:
         Does not mark the workflow attempt's job attempts as submitted.
         """
 
+        if not cls.enabled():
+            return
+
         con = cls.connection()
         cur = con.cursor()
         try:
@@ -945,6 +981,9 @@ class HistoryManager:
         Mark a collection of job attempts as submitted to Dockstore in a single transaction.
         """
 
+        if not cls.enabled_job():
+            return
+
         con = cls.connection()
         cur = con.cursor()
         try:
@@ -969,6 +1008,10 @@ class HistoryManager:
         """
         Count workflows in the database.
         """
+        
+        if not cls.enabled():
+            return 0
+
         con = cls.connection()
         cur = con.cursor()
         try:
@@ -994,6 +1037,10 @@ class HistoryManager:
         """
         Count workflow attempts in the database.
         """
+
+        if not cls.enabled():
+            return 0
+
         con = cls.connection()
         cur = con.cursor()
         try:
@@ -1019,6 +1066,10 @@ class HistoryManager:
         """
         Count job attempts in the database.
         """
+
+        if not cls.enabled_job():
+            return 0
+
         con = cls.connection()
         cur = con.cursor()
         try:
@@ -1044,6 +1095,10 @@ class HistoryManager:
         """
         Get workflows that have a successful attempt and no unsubmitted attempts or job attempts.
         """
+
+        if not cls.enabled():
+            return []
+
         ids = []
 
         con = cls.connection()
@@ -1105,6 +1160,9 @@ class HistoryManager:
         Get workflows that are old.
         """
 
+        if not cls.enabled():
+            return []
+
         ids = []
 
         con = cls.connection()
@@ -1150,6 +1208,9 @@ class HistoryManager:
         Succeeds if the workflow does not exist.
         """
 
+        if not cls.enabled():
+            return
+
         con = cls.connection()
         cur = con.cursor()
         try:
@@ -1173,6 +1234,9 @@ class HistoryManager:
         Get the total number of bytes used by the database.
         """
 
+        if not cls.enabled():
+            return 0
+
         con = cls.connection()
         cur = con.cursor()
         try:
@@ -1202,6 +1266,9 @@ class HistoryManager:
         """
         Shrink the database to remove unused space.
         """
+
+        if not cls.enabled():
+            return
         
         con = cls.connection()
         cur = con.cursor()
@@ -1226,6 +1293,9 @@ class HistoryManager:
         important.
         """
 
+        if not cls.enabled():
+            return
+
         db_size = cls.get_database_byte_size()
 
         if db_size < limit:
@@ -1264,6 +1334,10 @@ class HistoryManager:
 
         For debugging tests.
         """
+
+        if not cls.enabled():
+            return []
+
         return cls.connection().iterdump()
 
 

toil/lib/history_submission.py

@@ -82,7 +82,7 @@ def job_execution_id(job_attempt: JobAttemptSummary) -> str:
 def get_parsed_trs_spec(workflow_attempt: WorkflowAttemptSummary) -> tuple[str, str]:
     """
     Get the TRS ID and version of the workflow, or raise an error.
-    
+
     :returns: The TRS ID and the TRS version of the wrokflow run.
     :raises: ValueError if the workflow does not have a TRS spec or if the spec
         does not contain a version.
@@ -259,7 +259,7 @@ class Submission:
         return all_submitted_and_marked
 
 
-def create_history_submission(batch_size: int = 10, desired_tasks: int = 0) -> Submission:
+def create_history_submission(batch_size: Optional[int] = None, desired_tasks: Optional[int] = None) -> Submission:
     """
     Make a package of data about recent workflow runs to send in.
 
@@ -271,6 +271,12 @@ def create_history_submission(batch_size: int = 10, desired_tasks: int = 0) -> S
         batch. Use 0 to not submit any task information.
     """
 
+    # By default, include the things we are set to track history for.
+    if batch_size is None:
+        batch_size = 10 if HistoryManager.enabled() else 0
+    if desired_tasks is None:
+        desired_tasks = 50 if HistoryManager.enabled_job() else 0
+
     # Collect together some workflows and some lists of tasks into a submission.
     submission = Submission()
 
@@ -325,15 +331,16 @@ def create_current_submission(workflow_id: str, attempt_number: int) -> Submissi
     submission = Submission()
     try:
         workflow_attempt = HistoryManager.get_workflow_attempt(workflow_id, attempt_number)
-        if workflow_attempt is not None:
+        if workflow_attempt is not None and HistoryManager.enabled():
             if not workflow_attempt.submitted_to_dockstore:
                 submission.add_workflow_attempt(workflow_attempt)
-            try:
-                job_attempts = HistoryManager.get_unsubmitted_job_attempts(workflow_attempt.workflow_id, workflow_attempt.attempt_number)
-                submission.add_job_attempts(workflow_attempt, job_attempts)
-            except:
-                logger.exception("Could not compose metrics report for workflow task set")
-                # Keep going with just the workflow.
+            if HistoryManager.enabled_job():
+                try:
+                    job_attempts = HistoryManager.get_unsubmitted_job_attempts(workflow_attempt.workflow_id, workflow_attempt.attempt_number)
+                    submission.add_job_attempts(workflow_attempt, job_attempts)
+                except:
+                    logger.exception("Could not compose metrics report for workflow task set")
+                    # Keep going with just the workflow.
     except:
         logger.exception("Could not compose metrics report for workflow execution")
         # Keep going with an empty submission.
@@ -493,6 +500,13 @@ def display_dialog_tkinter(title: str, text: str, options: dict[KeyType, str], t
             # If we run out of time, hide the window and move on without a choice.
             root.after(int(timeout * 1000), close_root)
 
+        # To make the dialog pop up over the terminal instead of behind it, we
+        # lift it and temporarily make it topmost. We don't keep it topmost
+        # because we want to let the user switch away from it.
+        root.attributes('-topmost', True)
+        root.after(10, lambda: root.attributes('-topmost', False))
+        root.lift()
+
         # Run the window's main loop
         root.mainloop()
 

toil/lib/io.py

@@ -6,15 +6,17 @@ import stat
 import sys
 import tempfile
 import uuid
-from collections.abc import Iterator
+from collections.abc import Iterator, Iterable
 from contextlib import contextmanager
 from io import BytesIO
 from typing import IO, Any, Callable, Optional, Protocol, Union
 
 from toil.lib.memoize import memoize
+from toil.lib.misc import StrPath
 
 logger = logging.getLogger(__name__)
 
+
 @memoize
 def get_toil_home() -> str:
     """
@@ -43,6 +45,9 @@ REMOTE_SCHEMES = STANDARD_SCHEMES + [TOIL_URI_SCHEME]
 ALL_SCHEMES = REMOTE_SCHEMES + ["file:"]
 
 def is_standard_url(filename: str) -> bool:
+    """
+    Return True if the given URL is a non-Toil, non-file: URL.
+    """
     return is_url_with_scheme(filename, STANDARD_SCHEMES)
 
 def is_remote_url(filename: str) -> bool:
@@ -70,16 +75,23 @@ def is_url_with_scheme(filename: str, schemes: list[str]) -> bool:
     return False
 
 def is_toil_url(filename: str) -> bool:
+    """
+    Return True if a URL is a toilfile: URL.
+    """
     return is_url_with_scheme(filename, [TOIL_URI_SCHEME])
 
 def is_file_url(filename: str) -> bool:
-    return is_url_with_scheme(filename, ["file:"])
+    """
+    Return True if a URL is a file: URL.
 
+    Will return False for bare paths.
+    """
+    return is_url_with_scheme(filename, ["file:"])
 
 def mkdtemp(
     suffix: Optional[str] = None,
     prefix: Optional[str] = None,
-    dir: Optional[str] = None,
+    dir: Optional[StrPath] = None,
 ) -> str:
     """
     Make a temporary directory like tempfile.mkdtemp, but with relaxed permissions.
@@ -174,28 +186,28 @@ def robust_rmtree(path: Union[str, bytes]) -> None:
             raise
 
 
-def atomic_tmp_file(final_path: str) -> str:
+def atomic_tmp_file(final_path: StrPath) -> str:
     """Return a tmp file name to use with atomic_install.  This will be in the
     same directory as final_path. The temporary file will have the same extension
     as finalPath.  It the final path is in /dev (/dev/null, /dev/stdout), it is
     returned unchanged and atomic_tmp_install will do nothing."""
     final_dir = os.path.dirname(os.path.normpath(final_path))  # can be empty
     if final_dir == "/dev":
-        return final_path
+        return str(final_path)
     final_basename = os.path.basename(final_path)
     final_ext = os.path.splitext(final_path)[1]
-    base_name = f"{final_basename}.{uuid.uuid4()}.tmp{final_ext}"
+    base_name = f"{final_basename}.{str(uuid.uuid4())}.tmp{final_ext}"
     return os.path.join(final_dir, base_name)
 
 
-def atomic_install(tmp_path, final_path) -> None:
+def atomic_install(tmp_path: StrPath, final_path: StrPath) -> None:
     """atomic install of tmp_path as final_path"""
     if os.path.dirname(os.path.normpath(final_path)) != "/dev":
         os.rename(tmp_path, final_path)
 
 
 @contextmanager
-def AtomicFileCreate(final_path: str, keep: bool = False) -> Iterator[str]:
+def AtomicFileCreate(final_path: StrPath, keep: bool = False) -> Iterator[str]:
     """Context manager to create a temporary file.  Entering returns path to
     the temporary file in the same directory as finalPath.  If the code in
     context succeeds, the file renamed to its actual name.  If an error
@@ -250,23 +262,23 @@ def make_public_dir(in_directory: str, suggested_name: Optional[str] = None) ->
     our old default.
     """
     if suggested_name is not None:
-        generated_dir_path: str = os.path.join(in_directory, suggested_name)
+        generated_dir_path1 = os.path.join(in_directory, suggested_name)
         try:
-            os.mkdir(generated_dir_path)
-            os.chmod(generated_dir_path, 0o777)
-            return generated_dir_path
+            os.mkdir(generated_dir_path1)
+            os.chmod(generated_dir_path1, 0o777)
+            return generated_dir_path1
         except FileExistsError:
             pass
     for i in range(
         4, 32 + 1
     ):  # make random uuids and truncate to lengths starting at 4 and working up to max 32
         for _ in range(10):  # make 10 attempts for each length
-            truncated_uuid: str = str(uuid.uuid4()).replace("-", "")[:i]
-            generated_dir_path: str = os.path.join(in_directory, truncated_uuid)
+            truncated_uuid = str(uuid.uuid4()).replace("-", "")[:i]
+            generated_dir_path2 = os.path.join(in_directory, truncated_uuid)
             try:
-                os.mkdir(generated_dir_path)
-                os.chmod(generated_dir_path, 0o777)
-                return generated_dir_path
+                os.mkdir(generated_dir_path2)
+                os.chmod(generated_dir_path2, 0o777)
+                return generated_dir_path2
             except FileExistsError:
                 pass
     this_should_never_happen: str = os.path.join(in_directory, str(uuid.uuid4()))
@@ -328,7 +340,7 @@ class WriteWatchingStream:
 
         self.backingStream = backingStream
         # We have no write listeners yet
-        self.writeListeners = []
+        self.writeListeners: list[Callable[[int], None]] = []
 
     def onWrite(self, listener: Callable[[int], None]) -> None:
         """
@@ -339,7 +351,7 @@ class WriteWatchingStream:
 
     # Implement the file API from https://docs.python.org/2.4/lib/bltin-file-objects.html
 
-    def write(self, data):
+    def write(self, data: bytes) -> None:
         """
         Write the given data to the file.
         """
@@ -351,7 +363,7 @@ class WriteWatchingStream:
             # Send out notifications
             listener(len(data))
 
-    def writelines(self, datas):
+    def writelines(self, datas: Iterable[bytes]) -> None:
         """
         Write each string from the given iterable, without newlines.
         """
@@ -359,14 +371,14 @@ class WriteWatchingStream:
         for data in datas:
             self.write(data)
 
-    def flush(self):
+    def flush(self) -> None:
         """
         Flush the backing stream.
         """
 
         self.backingStream.flush()
 
-    def close(self):
+    def close(self) -> None:
         """
         Close the backing stream.
         """

toil/lib/misc.py

@@ -9,10 +9,16 @@ import sys
 import time
 from collections.abc import Iterator
 from contextlib import closing
-from typing import Optional
+from typing import Optional, Union
+if sys.version_info >= (3, 10):
+    from typing import TypeAlias
+else:
+    from typing_extensions import TypeAlias
 
 logger = logging.getLogger(__name__)
 
+StrPath: TypeAlias = Union[str, os.PathLike[str]]
+FileDescriptorOrPath: TypeAlias = Union[int, bytes, os.PathLike[bytes], StrPath]
 
 def get_public_ip() -> str:
     """Get the IP that this machine uses to contact the internet.
@@ -21,7 +27,7 @@ def get_public_ip() -> str:
     try:
         # Try to get the internet-facing IP by attempting a connection
         # to a non-existent server and reading what IP was used.
-        ip = "127.0.0.1"
+        ip: str = "127.0.0.1"
         with closing(socket.socket(socket.AF_INET, socket.SOCK_DGRAM)) as sock:
             # 203.0.113.0/24 is reserved as TEST-NET-3 by RFC 5737, so
             # there is guaranteed to be no one listening on the other

toil/lib/plugins.py

@@ -0,0 +1,106 @@
+# Copyright (C) 2015-2025 Regents of the University of California
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Generic plugin system for Toil plugins.
+
+Plugins come in Python packages named::
+
+    toil_{PLUGIN_TYPE}_{WHATEVER}
+
+When looking for plugins, Toil will list all the Python packages with the right
+name prefix for the given type of plugin, and load them. The plugin modules
+then have an opportunity to import :meth:`register_plugin` and register
+themselves.
+"""
+
+import importlib
+from typing import Any, Literal, Union
+import pkgutil
+from toil.lib.memoize import memoize
+
+
+PluginType = Union[Literal["batch_system"], Literal["url_access"]]
+plugin_types: list[PluginType] = ["batch_system", "url_access"]
+
+_registry: dict[str, dict[str, Any]] = {k: {} for k in plugin_types}
+
+def register_plugin(
+    plugin_type: PluginType, plugin_name: str, plugin_being_registered: Any
+) -> None:
+    """
+    Adds a plugin to the registry for the given type of plugin.
+
+    :param plugin_name: For batch systems, this is the string the user will use
+        to select the batch system on the command line with ``--batchSystem``.
+        For URL access plugins, this is the URL scheme that the plugin
+        implements.
+    :param plugin_being_registered: This is a function that, when called,
+        imports and returns a plugin-provided class type. For batch systems,
+        the resulting type must extend
+        :class:`toil.batchSystems.abstractBatchSystem.AbstractBatchSystem`. For
+        URL access plugins, it must extend :class:`toil.lib.url.URLAccess`.
+        Note that the function used here should return the class itslef; it
+        should not construct an instance of the class.
+    """
+    _registry[plugin_type][plugin_name] = plugin_being_registered
+
+def remove_plugin(
+    plugin_type: PluginType, plugin_name: str) -> None:
+    """
+    Removes a plugin from the registry for the given type of plugin.
+    """
+    try:
+        del _registry[plugin_type][plugin_name]
+    except KeyError:
+        # If the plugin does not exist, it can be ignored
+        pass
+
+def get_plugin_names(plugin_type:PluginType) -> list[str]:
+    """
+    Get the names of all the available plugins of the given type.
+    """
+    _load_all_plugins(plugin_type)
+    return list(_registry[plugin_type].keys())
+
+def get_plugin(plugin_type: PluginType, plugin_name: str) -> Any:
+    """
+    Get a plugin class factory function by name.
+
+    :raises: KeyError if plugin_name is not the name of a plugin of the given
+        type.
+    """
+    _load_all_plugins(plugin_type)
+    return _registry[plugin_type][plugin_name]
+
+
+def _plugin_name_prefix(plugin_type: PluginType) -> str:
+    """
+    Get prefix for plugin type. 
+    
+    Any packages with prefix will count as toil plugins of that type.
+    """
+    return f"toil_{plugin_type}_"
+
+@memoize
+def _load_all_plugins(plugin_type: PluginType) -> None:
+    """
+    Load all the plugins of the given type that are installed.
+    """
+    prefix = _plugin_name_prefix(plugin_type)
+    for finder, name, is_pkg in pkgutil.iter_modules():
+        # For all installed packages
+        if name.startswith(prefix):
+            # If it is a Toil batch system plugin, import it
+            importlib.import_module(name)

toil/lib/resources.py

@@ -17,6 +17,7 @@ import os
 import resource
 import sys
 
+from toil.lib.misc import StrPath
 
 class ResourceMonitor:
     """
@@ -89,7 +90,7 @@ class ResourceMonitor:
         )
 
 
-def glob(glob_pattern: str, directoryname: str) -> list[str]:
+def glob(glob_pattern: str, directoryname: StrPath) -> list[str]:
     """
     Walks through a directory and its subdirectories looking for files matching
     the glob_pattern and returns a list=[].