toil 8.1.0b1__py3-none-any.whl → 8.2.0__py3-none-any.whl

toil/__init__.py

@@ -480,38 +480,3 @@ def logProcessContext(config: "Config") -> None:
     log.info("Running Toil version %s on host %s.", version, socket.gethostname())
     log.debug("Configuration: %s", config.__dict__)
 
-
-try:
-    cache_path = "~/.cache/aws/cached_temporary_credentials"
-    datetime_format = (
-        "%Y-%m-%dT%H:%M:%SZ"  # incidentally the same as the format used by AWS
-    )
-    log = logging.getLogger(__name__)
-
-    # But in addition to our manual cache, we also are going to turn on boto3's
-    # new built-in caching layer.
-
-    def datetime_to_str(dt):
-        """
-        Convert a naive (implicitly UTC) datetime object into a string, explicitly UTC.
-
-        >>> datetime_to_str(datetime(1970, 1, 1, 0, 0, 0))
-        '1970-01-01T00:00:00Z'
-        """
-        return dt.strftime(datetime_format)
-
-    def str_to_datetime(s):
-        """
-        Convert a string, explicitly UTC into a naive (implicitly UTC) datetime object.
-
-        >>> str_to_datetime( '1970-01-01T00:00:00Z' )
-        datetime.datetime(1970, 1, 1, 0, 0)
-
-        Just to show that the constructor args for seconds and microseconds are optional:
-        >>> datetime(1970, 1, 1, 0, 0, 0)
-        datetime.datetime(1970, 1, 1, 0, 0)
-        """
-        return datetime.strptime(s, datetime_format)
-
-except ImportError:
-    pass

toil/batchSystems/abstractBatchSystem.py

@@ -310,7 +310,7 @@ class BatchSystemSupport(AbstractBatchSystem):
     """Partial implementation of AbstractBatchSystem, support methods."""
 
     def __init__(
-        self, config: Config, maxCores: float, maxMemory: int, maxDisk: int
+        self, config: Config, maxCores: float, maxMemory: float, maxDisk: int
     ) -> None:
         """
         Initialize initial state of the object.

toil/batchSystems/abstractGridEngineBatchSystem.py

@@ -421,7 +421,7 @@ class AbstractGridEngineBatchSystem(BatchSystemCleanupSupport):
             raise NotImplementedError()
 
     def __init__(
-        self, config: Config, maxCores: float, maxMemory: int, maxDisk: int
+        self, config: Config, maxCores: float, maxMemory: float, maxDisk: float
     ) -> None:
         super().__init__(config, maxCores, maxMemory, maxDisk)
         self.config = config

toil/batchSystems/awsBatch.py

@@ -83,7 +83,7 @@ class AWSBatchBatchSystem(BatchSystemCleanupSupport):
         return True
 
     def __init__(
-        self, config: Config, maxCores: float, maxMemory: int, maxDisk: int
+        self, config: Config, maxCores: float, maxMemory: float, maxDisk: int
     ) -> None:
         super().__init__(config, maxCores, maxMemory, maxDisk)
 

toil/batchSystems/cleanup_support.py

@@ -45,7 +45,7 @@ class BatchSystemCleanupSupport(BatchSystemLocalSupport):
         return contexts
 
     def __init__(
-        self, config: Config, maxCores: float, maxMemory: int, maxDisk: int
+        self, config: Config, maxCores: float, maxMemory: float, maxDisk: int
     ) -> None:
         super().__init__(config, maxCores, maxMemory, maxDisk)
 

toil/batchSystems/kubernetes.py

@@ -21,6 +21,7 @@ cannot yet be launched. That functionality will need to wait for user-mode
 Docker
 """
 import datetime
+import io
 import logging
 import math
 import os
@@ -50,7 +51,8 @@ else:
 from typing import Protocol, TypedDict, runtime_checkable
 
 import urllib3
-import yaml
+import ruamel.yaml as yaml
+import json
 
 # The Right Way to use the Kubernetes module is to `import kubernetes` and then you get all your stuff as like ApiClient. But this doesn't work for the stubs: the stubs seem to only support importing things from the internal modules in `kubernetes` where they are actually defined. See for example <https://github.com/MaterializeInc/kubernetes-stubs/issues/9 and <https://github.com/MaterializeInc/kubernetes-stubs/issues/10>. So we just import all the things we use into our global namespace here.
 from kubernetes.client import (
@@ -77,6 +79,7 @@ from kubernetes.client import (
     V1ResourceRequirements,
     V1SecretVolumeSource,
     V1SecurityContext,
+    V1PodSecurityContext,
     V1Toleration,
     V1Volume,
     V1VolumeMount,
@@ -148,7 +151,7 @@ class KubernetesBatchSystem(BatchSystemCleanupSupport):
         customObjects: NotRequired[CustomObjectsApi]
 
     def __init__(
-        self, config: Config, maxCores: int, maxMemory: int, maxDisk: int
+        self, config: Config, maxCores: int, maxMemory: float, maxDisk: int
     ) -> None:
         super().__init__(config, maxCores, maxMemory, maxDisk)
 
@@ -314,7 +317,10 @@ class KubernetesBatchSystem(BatchSystemCleanupSupport):
                 del here[k]
 
         drop_boring(root_dict)
-        return yaml.dump(root_dict)
+        s = io.StringIO()
+        YAML = yaml.YAML(typ='safe')
+        YAML.dump(root_dict, s)
+        return s.getvalue()
 
     @overload
     def _api(
@@ -803,6 +809,26 @@ class KubernetesBatchSystem(BatchSystemCleanupSupport):
                     ],
                 )
 
+    class FakeResponse:
+        data: str
+
+    T = TypeVar('T')
+    def _load_kubernetes_object(self, file: str, cls: type[T]) -> T:
+        """
+        Deserialize a YAML representation into a Kubernetes object
+        :param file: Path to YAML file
+        :param cls: Kubernetes API model type for deserialized object
+        :return: Deserialized object
+        """
+        YAML = yaml.YAML(typ='safe')
+        object_def = YAML.load(open('container.yaml').read())
+        # The kubernetes API does not have an actual deserializer, so this is a workaround
+        # See: https://github.com/kubernetes-client/python/issues/977
+        faked_response = self.FakeResponse()
+        faked_response.data = json.dumps(object_def)
+        return ApiClient().deserialize(faked_response, cls)
+
+
     def _create_pod_spec(
         self,
         command: str,
@@ -946,17 +972,24 @@ class KubernetesBatchSystem(BatchSystemCleanupSupport):
             volume_mounts=mounts,
         )
 
+        if self.config.kubernetes_security_context:
+            container.security_context = self._load_kubernetes_object(self.config.kubernetes_security_context, V1SecurityContext)
+
         # In case security context rules are not allowed to be set, we only apply
         # a security context at all if we need to turn on privileged mode.
         if self.config.kubernetes_privileged:
-            container.security_context = V1SecurityContext(
-                privileged=self.config.kubernetes_privileged
-            )
+            if container.security_context is None:
+                container.security_context = V1SecurityContext()
+            container.security_context.privileged = self.config.kubernetes_privileged
 
         # Wrap the container in a spec
         pod_spec = V1PodSpec(
             containers=[container], volumes=volumes, restart_policy="Never"
         )
+
+        if self.config.kubernetes_pod_security_context:
+            pod_spec.security_context = self._load_kubernetes_object(self.config.kubernetes_pod_security_context, V1PodSecurityContext)
+
         # Tell the spec where to land
         placement.apply(pod_spec)
 
@@ -2126,7 +2159,18 @@ class KubernetesBatchSystem(BatchSystemCleanupSupport):
             "privileged operations, such as FUSE. On Toil-managed clusters with --enableFuse, "
             "this is set to True. (default: %(default)s)",
         )
-
+        parser.add_argument("--kubernetesPodSecurityContext",
+                            dest='kubernetes_pod_security_context',
+                            type=str,
+                            env_var="TOIL_KUBERNETES_POD_SECURITY_CONTEXT",
+                            default=None,
+                            help="Path to a YAML defining a pod security context to apply to all pods.")
+        parser.add_argument("--kubernetesSecurityContext",
+                            dest='kubernetes_security_context',
+                            type=str,
+                            env_var="TOIL_KUBERNETES_SECURITY_CONTEXT",
+                            default=None,
+                            help="Path to a YAML defining a security context to apply to all containers.")
     OptionType = TypeVar("OptionType")
 
     @classmethod
@@ -2138,3 +2182,5 @@ class KubernetesBatchSystem(BatchSystemCleanupSupport):
         )
         setOption("kubernetes_pod_timeout")
         setOption("kubernetes_privileged")
+        setOption("kubernetes_pod_security_context")
+        setOption("kubernetes_security_context")

toil/batchSystems/local_support.py

@@ -30,7 +30,7 @@ class BatchSystemLocalSupport(BatchSystemSupport):
     """Adds a local queue for helper jobs, useful for CWL & others."""
 
     def __init__(
-        self, config: Config, maxCores: float, maxMemory: int, maxDisk: int
+        self, config: Config, maxCores: float, maxMemory: float, maxDisk: int
     ) -> None:
         super().__init__(config, maxCores, maxMemory, maxDisk)
         max_local_jobs = (

toil/batchSystems/mesos/batchSystem.py

@@ -40,6 +40,7 @@ from toil.batchSystems.abstractBatchSystem import (
 from toil.batchSystems.local_support import BatchSystemLocalSupport
 from toil.batchSystems.mesos import JobQueue, MesosShape, TaskData, ToilJob
 from toil.batchSystems.options import OptionSetter
+from toil.common import Config
 from toil.job import JobDescription
 from toil.lib.conversions import b_to_mib, mib_to_b
 from toil.lib.memoize import strict_bool
@@ -61,22 +62,26 @@ class MesosBatchSystem(BatchSystemLocalSupport, AbstractScalableBatchSystem, Sch
     """
 
     @classmethod
-    def supportsAutoDeployment(cls):
+    def supportsAutoDeployment(cls) -> bool:
         return True
 
     @classmethod
-    def supportsWorkerCleanup(cls):
+    def supportsWorkerCleanup(cls) -> bool:
         return True
 
     class ExecutorInfo:
-        def __init__(self, nodeAddress, agentId, nodeInfo, lastSeen):
+        def __init__(
+            self, nodeAddress: str, agentId: str, nodeInfo: str, lastSeen: str
+        ) -> None:
             super().__init__()
             self.nodeAddress = nodeAddress
             self.agentId = agentId
             self.nodeInfo = nodeInfo
             self.lastSeen = lastSeen
 
-    def __init__(self, config, maxCores, maxMemory, maxDisk):
+    def __init__(
+        self, config: Config, maxCores: float, maxMemory: float, maxDisk: int
+    ) -> None:
         super().__init__(config, maxCores, maxMemory, maxDisk)
 
         # The auto-deployed resource representing the user script. Will be passed along in every
@@ -165,13 +170,13 @@ class MesosBatchSystem(BatchSystemLocalSupport, AbstractScalableBatchSystem, Sch
 
         self._startDriver(config)
 
-    def setUserScript(self, userScript):
+    def setUserScript(self, userScript: str) -> None:
         self.userScript = userScript
 
-    def ignoreNode(self, nodeAddress):
+    def ignoreNode(self, nodeAddress: str) -> None:
         self.ignoredNodes.add(nodeAddress)
 
-    def unignoreNode(self, nodeAddress):
+    def unignoreNode(self, nodeAddress: str) -> None:
         self.ignoredNodes.remove(nodeAddress)
 
     def issueBatchJob(
@@ -179,7 +184,7 @@ class MesosBatchSystem(BatchSystemLocalSupport, AbstractScalableBatchSystem, Sch
         command: str,
         jobNode: JobDescription,
         job_environment: Optional[dict[str, str]] = None,
-    ):
+    ) -> str:
         """
         Issues the following command returning a unique jobID. Command is the string to run, memory
         is an int giving the number of bytes the job needs to run in and cores is the number of cpus

toil/batchSystems/mesos/test/__init__.py

@@ -7,6 +7,7 @@ from abc import ABCMeta, abstractmethod
 from contextlib import closing
 from shutil import which
 from urllib.request import urlopen
+from typing import Optional
 
 from toil.lib.retry import retry
 from toil.lib.threading import ExceptionalThread, cpu_count
@@ -25,7 +26,7 @@ class MesosTestSupport:
         with closing(urlopen("http://127.0.0.1:5050/version")) as content:
             content.read()
 
-    def _startMesos(self, numCores=None):
+    def _startMesos(self, numCores: Optional[int] = None) -> None:
         if numCores is None:
             numCores = cpu_count()
         shutil.rmtree("/tmp/mesos", ignore_errors=True)
@@ -52,7 +53,7 @@ class MesosTestSupport:
             log.warning("Forcibly killing child which ignored SIGTERM")
             process.kill()
 
-    def _stopMesos(self):
+    def _stopMesos(self) -> None:
         self._stopProcess(self.agent.popen)
         self.agent.join()
         self._stopProcess(self.master.popen)

toil/batchSystems/singleMachine.py

@@ -94,7 +94,7 @@ class SingleMachineBatchSystem(BatchSystemSupport):
         self,
         config: Config,
         maxCores: float,
-        maxMemory: int,
+        maxMemory: float,
         maxDisk: int,
         max_jobs: Optional[int] = None,
     ) -> None:

toil/batchSystems/slurm.py

@@ -101,6 +101,32 @@ def parse_slurm_time(slurm_time: str) -> int:
             total_seconds += multiplier * int(elapsed_split[index])
     return total_seconds
 
+# For parsing user-provided option overrides (or self-generated
+# options) for sbatch, we need a way to recognize long, long-with-equals, and
+# short forms.
+def option_detector(long: str, short: str | None = None) -> Callable[[str], bool]:
+    """
+    Get a function that returns true if it sees the long or short
+    option.
+    """
+    def is_match(option: str) -> bool:
+        return option == f"--{long}" or option.startswith(f"--{long}=") or (short is not None and option == f"-{short}")
+    return is_match
+
+def any_option_detector(options: list[str | tuple[str, str]]) -> Callable[[str], bool]:
+    """
+    Get a function that returns true if it sees any of the long
+    options or long or short option pairs.
+    """
+    detectors = [option_detector(o) if isinstance(o, str) else option_detector(*o) for o in options]
+    def is_match(option: str) -> bool:
+        for detector in detectors:
+            if detector(option):
+                return True
+        return False
+    return is_match
+
+
 
 class SlurmBatchSystem(AbstractGridEngineBatchSystem):
     class PartitionInfo(NamedTuple):
@@ -647,31 +673,6 @@ class SlurmBatchSystem(AbstractGridEngineBatchSystem):
             # Also any extra arguments from --slurmArgs or TOIL_SLURM_ARGS
             nativeConfig: str = self.boss.config.slurm_args  # type: ignore[attr-defined]
 
-            # For parsing user-provided option overrides (or self-generated
-            # options) we need a way to recognize long, long-with-equals, and
-            # short forms.
-            def option_detector(long: str, short: str | None = None) -> Callable[[str], bool]:
-                """
-                Get a function that returns true if it sees the long or short
-                option.
-                """
-                def is_match(option: str) -> bool:
-                    return option == f"--{long}" or option.startswith(f"--{long}=") or (short is not None and option == f"-{short}")
-                return is_match
-
-            def any_option_detector(options: list[str | tuple[str, str]]) -> Callable[[str], bool]:
-                """
-                Get a function that returns true if it sees any of the long
-                options or long or short option pairs.
-                """
-                detectors = [option_detector(o) if isinstance(o, str) else option_detector(*o) for o in options]
-                def is_match(option: str) -> bool:
-                    for detector in detectors:
-                        if detector(option):
-                            return True
-                    return False
-                return is_match
-
             is_any_mem_option = any_option_detector(["mem", "mem-per-cpu", "mem-per-gpu"])
             is_any_cpus_option = any_option_detector([("cpus-per-task", "c"), "cpus-per-gpu"])
             is_export_option = option_detector("export")
@@ -842,7 +843,7 @@ class SlurmBatchSystem(AbstractGridEngineBatchSystem):
             return sbatch_line
 
     def __init__(
-        self, config: Config, maxCores: float, maxMemory: int, maxDisk: int
+        self, config: Config, maxCores: float, maxMemory: float, maxDisk: float
     ) -> None:
         super().__init__(config, maxCores, maxMemory, maxDisk)
         self.partitions = SlurmBatchSystem.PartitionSet()

toil/bus.py

@@ -69,13 +69,15 @@ import tempfile
 import threading
 from collections.abc import Iterator
 from dataclasses import dataclass
-from typing import IO, Any, Callable, NamedTuple, Optional, TypeVar, cast
+from typing import IO, Any, Callable, NamedTuple, Optional, TypeVar, TYPE_CHECKING, cast
 
 from pubsub.core import Publisher
 from pubsub.core.listener import Listener
 from pubsub.core.topicobj import Topic
 from pubsub.core.topicutils import ALL_TOPICS
 
+from toil.lib.misc import FileDescriptorOrPath
+
 logger = logging.getLogger(__name__)
 
 # We define some ways to talk about jobs.
@@ -434,7 +436,7 @@ class MessageBus:
         connection._set_bus(self)
         return connection
 
-    def connect_output_file(self, file_path: str) -> Any:
+    def connect_output_file(self, file_path: FileDescriptorOrPath) -> Any:
         """
         Send copies of all messages to the given output file.
 
@@ -736,7 +738,7 @@ class JobStatus:
         )  # if the exit code is -1 and the job id is specified, we assume the job is running
 
 
-def replay_message_bus(path: str) -> dict[str, JobStatus]:
+def replay_message_bus(path: FileDescriptorOrPath) -> dict[str, JobStatus]:
     """
     Replay all the messages and work out what they mean for jobs.
 

toil/common.py

@@ -74,6 +74,7 @@ from toil.lib.compatibility import deprecated
 from toil.lib.history import HistoryManager
 from toil.lib.history_submission import ask_user_about_publishing_metrics, create_history_submission, create_current_submission
 from toil.lib.io import AtomicFileCreate, try_path, get_toil_home
+from toil.lib.misc import StrPath
 from toil.lib.memoize import memoize
 from toil.lib.retry import retry
 from toil.lib.threading import ensure_filesystem_lockable
@@ -126,6 +127,8 @@ class Config:
     kubernetes_service_account: Optional[str]
     kubernetes_pod_timeout: float
     kubernetes_privileged: bool
+    kubernetes_pod_security_context: Optional[str]
+    kubernetes_security_context: Optional[str]
     tes_endpoint: str
     tes_user: str
     tes_password: str
@@ -138,7 +141,7 @@ class Config:
     batch_logs_dir: Optional[str]
     """The backing scheduler will be instructed, if possible, to save logs
     to this directory, where the leader can read them."""
-    statePollingWait: int
+    statePollingWait: float
     state_polling_timeout: int
     disableAutoDeployment: bool
 
@@ -208,6 +211,7 @@ class Config:
 
     # Retrying/rescuing jobs
     retryCount: int
+    stop_on_first_failure: bool
     enableUnlimitedPreemptibleRetries: bool
     doubleMem: bool
     maxJobDuration: int
@@ -386,6 +390,7 @@ class Config:
 
         # Retrying/rescuing jobs
         set_option("retryCount")
+        set_option("stop_on_first_failure")
         set_option("enableUnlimitedPreemptibleRetries")
         set_option("doubleMem")
         set_option("maxJobDuration")
@@ -398,7 +403,7 @@ class Config:
         set_option("writeLogsGzip")
         set_option("writeLogsFromAllJobs")
         set_option("write_messages")
-        
+
         # Data Publishing Options
         set_option("publish_workflow_metrics")
 
@@ -653,7 +658,7 @@ def update_config(filepath: str, key: str, new_value: Union[str, bool, int, floa
     :param key: Setting to set. Must be the command-line option name, not the
         destination variable name.
     """
-    
+
     yaml = YAML(typ="rt")
     data = yaml.load(open(filepath))
 
@@ -678,6 +683,17 @@ def parser_with_common_options(
     prog: Optional[str] = None,
     default_log_level: Optional[int] = None,
 ) -> ArgParser:
+    """
+    Get a command-line option parser for a Toil subcommand.
+
+    The returned parser just has basic options (like version reporting and
+    logging) used by all Toil subcommands.
+
+    Toil Python workflows should use
+    :meth:`toil.job.Job.Runner.getDefaultArgumentParser` instead, which makes
+    sure to add all the important options for actually running a workflow.
+    """
+
     parser = ArgParser(
         prog=prog or "Toil", formatter_class=ArgumentDefaultsHelpFormatter
     )
@@ -781,7 +797,7 @@ def addOptions(
         :param typ: string of either "cwl" or "wdl" to specify which runner to check against
         :return: None, raise parser error if option is found
         """
-        check_parser = ArgParser()
+        check_parser = ArgParser(allow_abbrev=False)
         if typ == "wdl":
             add_cwl_options(check_parser)
         if typ == "cwl":
@@ -1489,21 +1505,33 @@ class Toil(ContextManager["Toil"]):
         self._jobStore.export_file(file_id, dst_uri)
 
     @staticmethod
-    def normalize_uri(uri: str, check_existence: bool = False) -> str:
+    def normalize_uri(uri: str, check_existence: bool = False, dir_path: Optional[str] = None) -> str:
         """
-        Given a URI, if it has no scheme, prepend "file:".
+        Given a URI, if it has no scheme, make it a properly quoted file: URI.
 
         :param check_existence: If set, raise FileNotFoundError if a URI points to
                a local file that does not exist.
+
+        :param dir_path: If specified, interpret relative paths relative to the
+            given directory path instead of the current one.
         """
-        if urlparse(uri).scheme == "file":
+
+        parsed = urlparse(uri)
+        if parsed.scheme == "file":
             uri = unquote(
-                urlparse(uri).path
+                parsed.path
             )  # this should strip off the local file scheme; it will be added back
+            parsed = urlparse(uri)
 
         # account for the scheme-less case, which should be coerced to a local absolute path
-        if urlparse(uri).scheme == "":
-            abs_path = os.path.abspath(uri)
+        if parsed.scheme == "":
+            if dir_path is not None:
+                # To support relative paths from a particular directory, join
+                # the directory on. If uri is already an abs path, join() will
+                # not do anything
+                abs_path = os.path.join(dir_path, uri)
+            else:
+                abs_path = os.path.abspath(uri)
             if not os.path.exists(abs_path) and check_existence:
                 raise FileNotFoundError(
                     f'Could not find local file "{abs_path}" when importing "{uri}".\n'
@@ -2019,7 +2047,7 @@ def cacheDirName(workflowID: str) -> str:
     return f"cache-{workflowID}"
 
 
-def getDirSizeRecursively(dirPath: str) -> int:
+def getDirSizeRecursively(dirPath: StrPath) -> int:
     """
     This method will return the cumulative number of bytes occupied by the files
     on disk in the directory and its subdirectories.

toil/cwl/cwltoil.py

@@ -2979,7 +2979,7 @@ def makeRootJob(
         # This will consist of files that we were not able to get a file size for
         leader_metadata = dict()
         for filename, file_data in metadata.items():
-            if file_data.size is None:
+            if file_data[2] is None:  # size
                 leader_metadata[filename] = file_data
             else:
                 worker_metadata[filename] = file_data