toil 5.12.0__py3-none-any.whl → 6.1.0a1__py3-none-any.whl

toil/batchSystems/options.py

@@ -12,19 +12,18 @@
 # See the License for the specific language governing permissions and
 
 import logging
-import os
 import sys
 from argparse import ArgumentParser, _ArgumentGroup
-from typing import Any, Callable, List, Optional, TypeVar, Union, cast
+from typing import Any, Callable, List, Optional, TypeVar, Union
 
 if sys.version_info >= (3, 8):
     from typing import Protocol
 else:
     from typing_extensions import Protocol
 
-from toil.batchSystems.registry import (BATCH_SYSTEM_FACTORY_REGISTRY,
-                                        BATCH_SYSTEMS,
-                                        DEFAULT_BATCH_SYSTEM)
+from toil.batchSystems.registry import (DEFAULT_BATCH_SYSTEM,
+                                        get_batch_system,
+                                        get_batch_systems)
 from toil.lib.threading import cpu_count
 
 logger = logging.getLogger(__name__)
@@ -56,60 +55,62 @@ def set_batchsystem_options(batch_system: Optional[str], set_option: OptionSette
     """
     if batch_system is not None:
         # Use this batch system
-        batch_system_type = BATCH_SYSTEM_FACTORY_REGISTRY[batch_system]()
+        batch_system_type = get_batch_system(batch_system)
         batch_system_type.setOptions(set_option)
     else:
-        for factory in BATCH_SYSTEM_FACTORY_REGISTRY.values():
+        for name in get_batch_systems():
             # All the batch systems are responsible for setting their own options
             # with their setOptions() class methods.
             try:
-                batch_system_type = factory()
+                batch_system_type = get_batch_system(name)
             except ImportError:
                 # Skip anything we can't import
                 continue
             # Ask the batch system to tell us all its options.
             batch_system_type.setOptions(set_option)
     # Options shared between multiple batch systems
-    set_option("disableAutoDeployment", bool, default=False)
+    set_option("disableAutoDeployment")
     # Make limits maximum if set to 0
-    set_option("max_jobs", lambda x: int(x) or sys.maxsize)
-    set_option("max_local_jobs", lambda x: int(x) or sys.maxsize)
+    set_option("max_jobs")
+    set_option("max_local_jobs")
     set_option("manualMemArgs")
-    set_option("run_local_jobs_on_workers", bool, default=False)
+    set_option("run_local_jobs_on_workers")
     set_option("statePollingWait")
-    set_option("batch_logs_dir", env=["TOIL_BATCH_LOGS_DIR"])
+    set_option("batch_logs_dir")
 
 
 def add_all_batchsystem_options(parser: Union[ArgumentParser, _ArgumentGroup]) -> None:
+    from toil.options.common import SYS_MAX_SIZE
+
     # Do the global cross-batch-system arguments
     parser.add_argument(
         "--batchSystem",
         dest="batchSystem",
         default=DEFAULT_BATCH_SYSTEM,
-        choices=BATCH_SYSTEMS,
+        choices=get_batch_systems(),
         help=f"The type of batch system to run the job(s) with, currently can be one "
-        f"of {', '.join(BATCH_SYSTEMS)}. default={DEFAULT_BATCH_SYSTEM}",
+        f"of {', '.join(get_batch_systems())}. Default = {DEFAULT_BATCH_SYSTEM}",
     )
     parser.add_argument(
         "--disableHotDeployment",
         dest="disableAutoDeployment",
+        default=False,
         action="store_true",
-        default=None,
         help="Hot-deployment was renamed to auto-deployment.  Option now redirects to "
-        "--disableAutoDeployment.  Left in for backwards compatibility.",
+        "--disableAutoDeployment. Left in for backwards compatibility.",
     )
     parser.add_argument(
         "--disableAutoDeployment",
         dest="disableAutoDeployment",
-        action="store_true",
-        default=None,
-        help="Should auto-deployment of the user script be deactivated? If True, the user "
-        "script/package should be present at the same location on all workers.  Default = False.",
+        default=False,
+        help="Should auto-deployment of Toil Python workflows be deactivated? If True, the workflow's "
+        "Python code should be present at the same location on all workers. Default = False.",
     )
     parser.add_argument(
         "--maxJobs",
         dest="max_jobs",
-        default=sys.maxsize, # This is *basically* unlimited and saves a lot of Optional[]
+        default=SYS_MAX_SIZE, # This is *basically* unlimited and saves a lot of Optional[]
+        type=lambda x: int(x) or SYS_MAX_SIZE,
         help="Specifies the maximum number of jobs to submit to the "
              "backing scheduler at once. Not supported on Mesos or "
              "AWS Batch. Use 0 for unlimited. Defaults to unlimited.",
@@ -117,7 +118,8 @@ def add_all_batchsystem_options(parser: Union[ArgumentParser, _ArgumentGroup]) -
     parser.add_argument(
         "--maxLocalJobs",
         dest="max_local_jobs",
-        default=cpu_count(),
+        default=None,
+        type=lambda x: int(x) or 0,
         help=f"Specifies the maximum number of housekeeping jobs to "
              f"run sumultaneously on the local system. Use 0 for "
              f"unlimited. Defaults to the number of local cores ({cpu_count()}).",
@@ -125,18 +127,18 @@ def add_all_batchsystem_options(parser: Union[ArgumentParser, _ArgumentGroup]) -
     parser.add_argument(
         "--manualMemArgs",
         default=False,
-        action="store_true",
         dest="manualMemArgs",
+        action="store_true",
         help="Do not add the default arguments: 'hv=MEMORY' & 'h_vmem=MEMORY' to the qsub "
         "call, and instead rely on TOIL_GRIDGENGINE_ARGS to supply alternative arguments.  "
         "Requires that TOIL_GRIDGENGINE_ARGS be set.",
     )
     parser.add_argument(
-        "--runLocalJobsOnWorkers"
+        "--runLocalJobsOnWorkers",
         "--runCwlInternalJobsOnWorkers",
         dest="run_local_jobs_on_workers",
+        default=False,
         action="store_true",
-        default=None,
         help="Whether to run jobs marked as local (e.g. CWLScatter) on the worker nodes "
         "instead of the leader node. If false (default), then all such jobs are run on "
         "the leader node. Setting this to true can speed up CWL pipelines for very large "
@@ -166,6 +168,7 @@ def add_all_batchsystem_options(parser: Union[ArgumentParser, _ArgumentGroup]) -
         "--batchLogsDir",
         dest="batch_logs_dir",
         default=None,
+        env_var="TOIL_BATCH_LOGS_DIR",
         help="Directory to tell the backing batch system to log into. Should be available "
              "on both the leader and the workers, if the backing batch system writes logs "
              "to the worker machines' filesystems, as many HPC schedulers do. If unset, "
@@ -173,68 +176,14 @@ def add_all_batchsystem_options(parser: Union[ArgumentParser, _ArgumentGroup]) -
              "systems such as gridengine, htcondor, torque, slurm, and lsf."
     )
 
-    for factory in BATCH_SYSTEM_FACTORY_REGISTRY.values():
+    for name in get_batch_systems():
         # All the batch systems are responsible for adding their own options
         # with the add_options class method.
         try:
-            batch_system_type = factory()
+            batch_system_type = get_batch_system(name)
         except ImportError:
             # Skip anything we can't import
             continue
         # Ask the batch system to create its options in the parser
-        logger.debug('Add options for %s', batch_system_type)
+        logger.debug('Add options for %s batch system', name)
         batch_system_type.add_options(parser)
-
-def set_batchsystem_config_defaults(config) -> None:
-    """
-    Set default and environment-based options for builtin batch systems. This
-    is required if a Config object is not constructed from an Options object.
-    """
-
-    # Do the global options across batch systems
-    config.batchSystem = "single_machine"
-    config.disableAutoDeployment = False
-    config.max_jobs = sys.maxsize
-    config.max_local_jobs = cpu_count()
-    config.manualMemArgs = False
-    config.statePollingWait: Optional[Union[float, int]] = None  # Number of seconds to wait before querying job state
-
-    OptionType = TypeVar('OptionType')
-    def set_option(option_name: str,
-                   parsing_function: Optional[Callable[[Any], OptionType]] = None,
-                   check_function: Optional[Callable[[OptionType], None]] = None,
-                   default: Optional[OptionType] = None,
-                   env: Optional[List[str]] = None,
-                   old_names: Optional[List[str]] = None) -> None:
-        """
-        Function to set a batch-system-defined option to its default value, or
-        one from the environment.
-        """
-
-        # TODO: deduplicate with Config
-
-        option_value = default
-
-        if env is not None:
-            for env_var in env:
-                # Try all the environment variables
-                if option_value != default:
-                    break
-                option_value = os.environ.get(env_var, default)
-
-        if option_value is not None or not hasattr(config, option_name):
-            if parsing_function is not None:
-                # Parse whatever it is (string, argparse-made list, etc.)
-                option_value = parsing_function(option_value)
-            if check_function is not None:
-                try:
-                    check_function(option_value)
-                except AssertionError:
-                    raise RuntimeError(f"The {option_name} option has an invalid value: {option_value}")
-            setattr(config, option_name, option_value)
-
-    # Set up defaults from all the batch systems
-    set_batchsystem_options(None, cast(OptionSetter, set_option))
-
-
-

toil/batchSystems/registry.py

@@ -12,14 +12,58 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import importlib
 import logging
-from typing import TYPE_CHECKING, Callable, Dict, List, Tuple, Type
+import pkgutil
+import warnings
+from typing import TYPE_CHECKING, Callable, Dict, List, Sequence, Tuple, Type
+
+from toil.lib.compatibility import deprecated
+from toil.lib.memoize import memoize
 
 if TYPE_CHECKING:
     from toil.batchSystems.abstractBatchSystem import AbstractBatchSystem
 
 logger = logging.getLogger(__name__)
 
+#####
+# Plugin system/API
+#####
+
+def add_batch_system_factory(key: str, class_factory: Callable[[], Type['AbstractBatchSystem']]):
+    """
+    Adds a batch system to the registry for workflow or plugin-supplied batch systems.
+
+    :param class_factory: A function that returns a batch system class (NOT an instance), which implements :class:`toil.batchSystems.abstractBatchSystem.AbstractBatchSystem`.
+    """
+    _registry_keys.append(key)
+    _registry[key] = class_factory
+
+def get_batch_systems() -> Sequence[str]:
+    """
+    Get the names of all the availsble batch systems.
+    """
+    _load_all_plugins()
+
+    return _registry_keys
+
+def get_batch_system(key: str) -> Type['AbstractBatchSystem']:
+    """
+    Get a batch system class by name.
+
+    :raises: KeyError if the key is not the name of a batch system, and
+             ImportError if the batch system's class cannot be loaded.
+    """
+
+    return _registry[key]()
+
+
+DEFAULT_BATCH_SYSTEM = 'single_machine'
+
+#####
+# Built-in batch systems
+#####
+
 def aws_batch_batch_system_factory():
     from toil.batchSystems.awsBatch import AWSBatchBatchSystem
     return AWSBatchBatchSystem
@@ -29,11 +73,6 @@ def gridengine_batch_system_factory():
     return GridEngineBatchSystem
 
 
-def parasol_batch_system_factory():
-    from toil.batchSystems.parasol import ParasolBatchSystem
-    return ParasolBatchSystem
-
-
 def lsf_batch_system_factory():
     from toil.batchSystems.lsf import LSFBatchSystem
     return LSFBatchSystem
@@ -53,10 +92,6 @@ def slurm_batch_system_factory():
     from toil.batchSystems.slurm import SlurmBatchSystem
     return SlurmBatchSystem
 
-def tes_batch_system_factory():
-    from toil.batchSystems.tes import TESBatchSystem
-    return TESBatchSystem
-
 def torque_batch_system_factory():
     from toil.batchSystems.torque import TorqueBatchSystem
     return TorqueBatchSystem
@@ -71,29 +106,75 @@ def kubernetes_batch_system_factory():
     from toil.batchSystems.kubernetes import KubernetesBatchSystem
     return KubernetesBatchSystem
 
+#####
+# Registry implementation
+#####
 
-BATCH_SYSTEM_FACTORY_REGISTRY: Dict[str, Callable[[], Type["AbstractBatchSystem"]]] = {
+_registry: Dict[str, Callable[[], Type["AbstractBatchSystem"]]] = {
     'aws_batch'      : aws_batch_batch_system_factory,
-    'parasol'        : parasol_batch_system_factory,
     'single_machine' : single_machine_batch_system_factory,
     'grid_engine'    : gridengine_batch_system_factory,
     'lsf'            : lsf_batch_system_factory,
     'mesos'          : mesos_batch_system_factory,
     'slurm'          : slurm_batch_system_factory,
-    'tes'            : tes_batch_system_factory,
     'torque'         : torque_batch_system_factory,
     'htcondor'       : htcondor_batch_system_factory,
     'kubernetes'     : kubernetes_batch_system_factory
 }
-BATCH_SYSTEMS = list(BATCH_SYSTEM_FACTORY_REGISTRY.keys())
-DEFAULT_BATCH_SYSTEM = 'single_machine'
+_registry_keys = list(_registry.keys())
+
+# We will load any packages starting with this prefix and let them call
+# add_batch_system_factory()
+_PLUGIN_NAME_PREFIX = "toil_batch_system_"
+
+@memoize
+def _load_all_plugins() -> None:
+    """
+    Load all the batch system plugins that are installed.
+    """
+
+    for finder, name, is_pkg in pkgutil.iter_modules():
+        # For all installed packages
+        if name.startswith(_PLUGIN_NAME_PREFIX):
+            # If it is a Toil batch system plugin, import it
+            importlib.import_module(name)
+
+#####
+# Deprecated API
+#####
 
+# We used to directly access these constants, but now the Right Way to use this
+# module is add_batch_system_factory() to register and get_batch_systems() to
+# get the list/get_batch_system() to get a class by name.
+
+
+def __getattr__(name):
+    """
+    Implement a fallback attribute getter to handle deprecated constants.
+
+    See <https://stackoverflow.com/a/48242860>.
+    """
+    if name == "BATCH_SYSTEM_FACTORY_REGISTRY":
+        warnings.warn("BATCH_SYSTEM_FACTORY_REGISTRY is deprecated; use get_batch_system() or add_batch_system_factory()", DeprecationWarning)
+        return _registry
+    elif name == "BATCH_SYSTEMS":
+        warnings.warn("BATCH_SYSTEMS is deprecated; use get_batch_systems()", DeprecationWarning)
+        return _registry_keys
+    else:
+        raise AttributeError(f"Module {__name__} ahs no attribute {name}")
+
+
+@deprecated(new_function_name="add_batch_system_factory")
 def addBatchSystemFactory(key: str, batchSystemFactory: Callable[[], Type['AbstractBatchSystem']]):
     """
-    Adds a batch system to the registry for workflow-supplied batch systems.
+    Deprecated method to add a batch system.
     """
-    BATCH_SYSTEMS.append(key)
-    BATCH_SYSTEM_FACTORY_REGISTRY[key] = batchSystemFactory
+    return add_batch_system_factory(key, batchSystemFactory)
+
+
+#####
+# Testing utilities
+#####
 
 # We need a snapshot save/restore system for testing. We can't just tamper with
 # the globals because module-level globals are their own references, so we
@@ -106,7 +187,7 @@ def save_batch_system_plugin_state() -> Tuple[List[str], Dict[str, Callable[[],
     tests.
     """
 
-    snapshot = (list(BATCH_SYSTEMS), dict(BATCH_SYSTEM_FACTORY_REGISTRY))
+    snapshot = (list(_registry_keys), dict(_registry))
     return snapshot
 
 def restore_batch_system_plugin_state(snapshot: Tuple[List[str], Dict[str, Callable[[], Type['AbstractBatchSystem']]]]):
@@ -118,7 +199,7 @@ def restore_batch_system_plugin_state(snapshot: Tuple[List[str], Dict[str, Calla
     # We need to apply the snapshot without rebinding the names, because that
     # won't affect modules that imported the names.
     wanted_batch_systems, wanted_registry = snapshot
-    BATCH_SYSTEMS.clear()
-    BATCH_SYSTEMS.extend(wanted_batch_systems)
-    BATCH_SYSTEM_FACTORY_REGISTRY.clear()
-    BATCH_SYSTEM_FACTORY_REGISTRY.update(wanted_registry)
+    _registry_keys.clear()
+    _registry_keys.extend(wanted_batch_systems)
+    _registry.clear()
+    _registry.update(wanted_registry)

toil/batchSystems/singleMachine.py

@@ -22,23 +22,26 @@ import traceback
 from argparse import ArgumentParser, _ArgumentGroup
 from queue import Empty, Queue
 from threading import Event, Lock, Thread
-from typing import Dict, List, Optional, Set, Sequence, Tuple, Union
+from typing import Dict, List, Optional, Sequence, Set, Tuple, Union
 
 import toil
 from toil import worker as toil_worker
 from toil.batchSystems.abstractBatchSystem import (EXIT_STATUS_UNAVAILABLE_VALUE,
                                                    BatchSystemSupport,
+                                                   InsufficientSystemResources,
                                                    ResourcePool,
                                                    ResourceSet,
-                                                   UpdatedBatchJobInfo,
-                                                   InsufficientSystemResources)
-
-from toil.bus import ExternalBatchIdMessage
+                                                   UpdatedBatchJobInfo)
 from toil.batchSystems.options import OptionSetter
-
-from toil.common import SYS_MAX_SIZE, Config, Toil, fC
-from toil.job import JobDescription, AcceleratorRequirement, accelerator_satisfies, Requirer
-from toil.lib.accelerators import get_individual_local_accelerators, get_restrictive_environment_for_local_accelerators
+from toil.bus import ExternalBatchIdMessage
+from toil.common import Config, Toil
+from toil.options.common import SYS_MAX_SIZE, make_open_interval_action
+from toil.job import (AcceleratorRequirement,
+                      JobDescription,
+                      Requirer,
+                      accelerator_satisfies)
+from toil.lib.accelerators import (get_individual_local_accelerators,
+                                   get_restrictive_environment_for_local_accelerators)
 from toil.lib.threading import cpu_count
 
 logger = logging.getLogger(__name__)
@@ -104,7 +107,7 @@ class SingleMachineBatchSystem(BatchSystemSupport):
                 logger.warning('Not enough cores! User limited to %i but we only have %i.', maxCores, self.numCores)
             maxCores = self.numCores
         if maxMemory > self.physicalMemory:
-            if maxMemory != SYS_MAX_SIZE:
+            if maxMemory < SYS_MAX_SIZE:  # todo: looks like humans2bytes converts SYS_MAX_SIZE to SYS_MAX_SIZE+1
                 # We have an actually specified limit and not the default
                 logger.warning('Not enough memory! User limited to %i bytes but we only have %i bytes.', maxMemory, self.physicalMemory)
             maxMemory = self.physicalMemory
@@ -112,7 +115,7 @@ class SingleMachineBatchSystem(BatchSystemSupport):
         workdir = Toil.getLocalWorkflowDir(config.workflowID, config.workDir)  # config.workDir may be None; this sets a real directory
         self.physicalDisk = toil.physicalDisk(workdir)
         if maxDisk > self.physicalDisk:
-            if maxDisk != SYS_MAX_SIZE:
+            if maxDisk < SYS_MAX_SIZE:  # same as maxMemory logger.warning
                 # We have an actually specified limit and not the default
                 logger.warning('Not enough disk space! User limited to %i bytes but we only have %i bytes.', maxDisk, self.physicalDisk)
             maxDisk = self.physicalDisk
@@ -843,7 +846,7 @@ class SingleMachineBatchSystem(BatchSystemSupport):
 
     @classmethod
     def add_options(cls, parser: Union[ArgumentParser, _ArgumentGroup]) -> None:
-        parser.add_argument("--scale", dest="scale", default=1,
+        parser.add_argument("--scale", dest="scale", type=float, default=1, action=make_open_interval_action(0.0),
                             help="A scaling factor to change the value of all submitted tasks's submitted cores.  "
                                  "Used in the single_machine batch system. Useful for running workflows on "
                                  "smaller machines than they were designed for, by setting a value less than 1. "
@@ -851,7 +854,7 @@ class SingleMachineBatchSystem(BatchSystemSupport):
 
     @classmethod
     def setOptions(cls, setOption: OptionSetter):
-        setOption("scale", float, fC(0.0), default=1)
+        setOption("scale")
 
 
 class Info:

toil/batchSystems/slurm.py

@@ -16,13 +16,13 @@ import math
 import os
 from argparse import ArgumentParser, _ArgumentGroup
 from shlex import quote
-from typing import Callable, Dict, List, Optional, TypeVar, Union
+from typing import Dict, List, Optional, TypeVar, Union
 
 from toil.batchSystems.abstractGridEngineBatchSystem import \
     AbstractGridEngineBatchSystem
 from toil.batchSystems.options import OptionSetter
-from toil.lib.misc import CalledProcessErrorStderr, call_command
 from toil.job import Requirer
+from toil.lib.misc import CalledProcessErrorStderr, call_command
 
 logger = logging.getLogger(__name__)
 
@@ -387,5 +387,5 @@ class SlurmBatchSystem(AbstractGridEngineBatchSystem):
     OptionType = TypeVar('OptionType')
     @classmethod
     def setOptions(cls, setOption: OptionSetter) -> None:
-        setOption("allocate_mem", bool, default=False)
+        setOption("allocate_mem")
 

toil/batchSystems/torque.py

@@ -16,7 +16,6 @@ import math
 import os
 import shlex
 import tempfile
-import time
 from queue import Empty
 from shlex import quote
 from typing import Dict, List, Optional

toil/bus.py

@@ -61,15 +61,13 @@ MessageBus.connect_output_file() and MessageBus.scan_bus_messages().
 """
 
 import collections
-from dataclasses import dataclass
-import inspect
+import json
 import logging
-
 import os
 import queue
-import json
 import tempfile
 import threading
+from dataclasses import dataclass
 from typing import (IO,
                     Any,
                     Callable,
@@ -80,7 +78,6 @@ from typing import (IO,
                     Optional,
                     Type,
                     TypeVar,
-                    Union,
                     cast)
 
 from pubsub.core import Publisher
@@ -396,8 +393,8 @@ class MessageBus:
             given topic.
             """
             # There should always be a "message"
-            assert len(message_data) == 1
-            assert 'message' in message_data
+            if len(message_data) != 1 or 'message' not in message_data:
+                raise RuntimeError("Cannot log the bus message. The message is either empty/malformed or there are too many messages provided.")
             message = message_data['message']
             topic = topic_object.getName()
             stream.write(topic.encode('utf-8'))
@@ -572,7 +569,8 @@ class MessageInbox(MessageBusClient):
                 handled = False
                 try:
                     # Emit the message
-                    assert isinstance(message, message_type), f"Unacceptable message type {type(message)} in list for type {message_type}"
+                    if not isinstance(message, message_type):
+                        raise RuntimeError(f"Unacceptable message type {type(message)} in list for type {message_type}")
                     yield message
                     # If we get here it was handled without error.
                     handled = True