toil 8.0.0__py3-none-any.whl → 8.2.0__py3-none-any.whl

toil/__init__.py

@@ -19,7 +19,7 @@ import sys
 from datetime import datetime
 from typing import TYPE_CHECKING, Optional
 
-import requests
+from toil.lib.web import web_session
 
 from docker.errors import ImageNotFound
 from toil.lib.memoize import memoize
@@ -425,7 +425,7 @@ def requestCheckRegularDocker(
     ioURL = "https://{webhost}/v2/{pathName}/manifests/{tag}" "".format(
         webhost=registryName, pathName=imageName, tag=tag
     )
-    response = requests.head(ioURL)
+    response = web_session.head(ioURL)
     if not response.ok:
         raise ApplianceImageNotFound(origAppliance, ioURL, response.status_code)
     else:
@@ -459,10 +459,10 @@ def requestCheckDockerIo(origAppliance: str, imageName: str, tag: str) -> bool:
     )
     requests_url = f"https://registry-1.docker.io/v2/{imageName}/manifests/{tag}"
 
-    token = requests.get(token_url)
+    token = web_session.get(token_url)
     jsonToken = token.json()
     bearer = jsonToken["token"]
-    response = requests.head(
+    response = web_session.head(
         requests_url, headers={"Authorization": f"Bearer {bearer}"}
     )
     if not response.ok:
@@ -480,38 +480,3 @@ def logProcessContext(config: "Config") -> None:
     log.info("Running Toil version %s on host %s.", version, socket.gethostname())
     log.debug("Configuration: %s", config.__dict__)
 
-
-try:
-    cache_path = "~/.cache/aws/cached_temporary_credentials"
-    datetime_format = (
-        "%Y-%m-%dT%H:%M:%SZ"  # incidentally the same as the format used by AWS
-    )
-    log = logging.getLogger(__name__)
-
-    # But in addition to our manual cache, we also are going to turn on boto3's
-    # new built-in caching layer.
-
-    def datetime_to_str(dt):
-        """
-        Convert a naive (implicitly UTC) datetime object into a string, explicitly UTC.
-
-        >>> datetime_to_str(datetime(1970, 1, 1, 0, 0, 0))
-        '1970-01-01T00:00:00Z'
-        """
-        return dt.strftime(datetime_format)
-
-    def str_to_datetime(s):
-        """
-        Convert a string, explicitly UTC into a naive (implicitly UTC) datetime object.
-
-        >>> str_to_datetime( '1970-01-01T00:00:00Z' )
-        datetime.datetime(1970, 1, 1, 0, 0)
-
-        Just to show that the constructor args for seconds and microseconds are optional:
-        >>> datetime(1970, 1, 1, 0, 0, 0)
-        datetime.datetime(1970, 1, 1, 0, 0)
-        """
-        return datetime.strptime(s, datetime_format)
-
-except ImportError:
-    pass

toil/batchSystems/abstractBatchSystem.py

@@ -310,7 +310,7 @@ class BatchSystemSupport(AbstractBatchSystem):
     """Partial implementation of AbstractBatchSystem, support methods."""
 
     def __init__(
-        self, config: Config, maxCores: float, maxMemory: int, maxDisk: int
+        self, config: Config, maxCores: float, maxMemory: float, maxDisk: int
     ) -> None:
         """
         Initialize initial state of the object.

toil/batchSystems/abstractGridEngineBatchSystem.py

@@ -421,7 +421,7 @@ class AbstractGridEngineBatchSystem(BatchSystemCleanupSupport):
             raise NotImplementedError()
 
     def __init__(
-        self, config: Config, maxCores: float, maxMemory: int, maxDisk: int
+        self, config: Config, maxCores: float, maxMemory: float, maxDisk: float
     ) -> None:
         super().__init__(config, maxCores, maxMemory, maxDisk)
         self.config = config

toil/batchSystems/awsBatch.py

@@ -83,7 +83,7 @@ class AWSBatchBatchSystem(BatchSystemCleanupSupport):
         return True
 
     def __init__(
-        self, config: Config, maxCores: float, maxMemory: int, maxDisk: int
+        self, config: Config, maxCores: float, maxMemory: float, maxDisk: int
     ) -> None:
         super().__init__(config, maxCores, maxMemory, maxDisk)
 

toil/batchSystems/cleanup_support.py

@@ -45,7 +45,7 @@ class BatchSystemCleanupSupport(BatchSystemLocalSupport):
         return contexts
 
     def __init__(
-        self, config: Config, maxCores: float, maxMemory: int, maxDisk: int
+        self, config: Config, maxCores: float, maxMemory: float, maxDisk: int
     ) -> None:
         super().__init__(config, maxCores, maxMemory, maxDisk)
 

toil/batchSystems/kubernetes.py

@@ -21,6 +21,7 @@ cannot yet be launched. That functionality will need to wait for user-mode
 Docker
 """
 import datetime
+import io
 import logging
 import math
 import os
@@ -50,7 +51,8 @@ else:
 from typing import Protocol, TypedDict, runtime_checkable
 
 import urllib3
-import yaml
+import ruamel.yaml as yaml
+import json
 
 # The Right Way to use the Kubernetes module is to `import kubernetes` and then you get all your stuff as like ApiClient. But this doesn't work for the stubs: the stubs seem to only support importing things from the internal modules in `kubernetes` where they are actually defined. See for example <https://github.com/MaterializeInc/kubernetes-stubs/issues/9 and <https://github.com/MaterializeInc/kubernetes-stubs/issues/10>. So we just import all the things we use into our global namespace here.
 from kubernetes.client import (
@@ -77,6 +79,7 @@ from kubernetes.client import (
     V1ResourceRequirements,
     V1SecretVolumeSource,
     V1SecurityContext,
+    V1PodSecurityContext,
     V1Toleration,
     V1Volume,
     V1VolumeMount,
@@ -148,7 +151,7 @@ class KubernetesBatchSystem(BatchSystemCleanupSupport):
         customObjects: NotRequired[CustomObjectsApi]
 
     def __init__(
-        self, config: Config, maxCores: int, maxMemory: int, maxDisk: int
+        self, config: Config, maxCores: int, maxMemory: float, maxDisk: int
     ) -> None:
         super().__init__(config, maxCores, maxMemory, maxDisk)
 
@@ -314,7 +317,10 @@ class KubernetesBatchSystem(BatchSystemCleanupSupport):
                 del here[k]
 
         drop_boring(root_dict)
-        return yaml.dump(root_dict)
+        s = io.StringIO()
+        YAML = yaml.YAML(typ='safe')
+        YAML.dump(root_dict, s)
+        return s.getvalue()
 
     @overload
     def _api(
@@ -803,6 +809,26 @@ class KubernetesBatchSystem(BatchSystemCleanupSupport):
                     ],
                 )
 
+    class FakeResponse:
+        data: str
+
+    T = TypeVar('T')
+    def _load_kubernetes_object(self, file: str, cls: type[T]) -> T:
+        """
+        Deserialize a YAML representation into a Kubernetes object
+        :param file: Path to YAML file
+        :param cls: Kubernetes API model type for deserialized object
+        :return: Deserialized object
+        """
+        YAML = yaml.YAML(typ='safe')
+        object_def = YAML.load(open('container.yaml').read())
+        # The kubernetes API does not have an actual deserializer, so this is a workaround
+        # See: https://github.com/kubernetes-client/python/issues/977
+        faked_response = self.FakeResponse()
+        faked_response.data = json.dumps(object_def)
+        return ApiClient().deserialize(faked_response, cls)
+
+
     def _create_pod_spec(
         self,
         command: str,
@@ -946,17 +972,24 @@ class KubernetesBatchSystem(BatchSystemCleanupSupport):
             volume_mounts=mounts,
         )
 
+        if self.config.kubernetes_security_context:
+            container.security_context = self._load_kubernetes_object(self.config.kubernetes_security_context, V1SecurityContext)
+
         # In case security context rules are not allowed to be set, we only apply
         # a security context at all if we need to turn on privileged mode.
         if self.config.kubernetes_privileged:
-            container.security_context = V1SecurityContext(
-                privileged=self.config.kubernetes_privileged
-            )
+            if container.security_context is None:
+                container.security_context = V1SecurityContext()
+            container.security_context.privileged = self.config.kubernetes_privileged
 
         # Wrap the container in a spec
         pod_spec = V1PodSpec(
             containers=[container], volumes=volumes, restart_policy="Never"
         )
+
+        if self.config.kubernetes_pod_security_context:
+            pod_spec.security_context = self._load_kubernetes_object(self.config.kubernetes_pod_security_context, V1PodSecurityContext)
+
         # Tell the spec where to land
         placement.apply(pod_spec)
 
@@ -2126,7 +2159,18 @@ class KubernetesBatchSystem(BatchSystemCleanupSupport):
             "privileged operations, such as FUSE. On Toil-managed clusters with --enableFuse, "
             "this is set to True. (default: %(default)s)",
         )
-
+        parser.add_argument("--kubernetesPodSecurityContext",
+                            dest='kubernetes_pod_security_context',
+                            type=str,
+                            env_var="TOIL_KUBERNETES_POD_SECURITY_CONTEXT",
+                            default=None,
+                            help="Path to a YAML defining a pod security context to apply to all pods.")
+        parser.add_argument("--kubernetesSecurityContext",
+                            dest='kubernetes_security_context',
+                            type=str,
+                            env_var="TOIL_KUBERNETES_SECURITY_CONTEXT",
+                            default=None,
+                            help="Path to a YAML defining a security context to apply to all containers.")
     OptionType = TypeVar("OptionType")
 
     @classmethod
@@ -2138,3 +2182,5 @@ class KubernetesBatchSystem(BatchSystemCleanupSupport):
         )
         setOption("kubernetes_pod_timeout")
         setOption("kubernetes_privileged")
+        setOption("kubernetes_pod_security_context")
+        setOption("kubernetes_security_context")

toil/batchSystems/local_support.py

@@ -30,7 +30,7 @@ class BatchSystemLocalSupport(BatchSystemSupport):
     """Adds a local queue for helper jobs, useful for CWL & others."""
 
     def __init__(
-        self, config: Config, maxCores: float, maxMemory: int, maxDisk: int
+        self, config: Config, maxCores: float, maxMemory: float, maxDisk: int
     ) -> None:
         super().__init__(config, maxCores, maxMemory, maxDisk)
         max_local_jobs = (

toil/batchSystems/mesos/batchSystem.py

@@ -40,6 +40,7 @@ from toil.batchSystems.abstractBatchSystem import (
 from toil.batchSystems.local_support import BatchSystemLocalSupport
 from toil.batchSystems.mesos import JobQueue, MesosShape, TaskData, ToilJob
 from toil.batchSystems.options import OptionSetter
+from toil.common import Config
 from toil.job import JobDescription
 from toil.lib.conversions import b_to_mib, mib_to_b
 from toil.lib.memoize import strict_bool
@@ -61,22 +62,26 @@ class MesosBatchSystem(BatchSystemLocalSupport, AbstractScalableBatchSystem, Sch
     """
 
     @classmethod
-    def supportsAutoDeployment(cls):
+    def supportsAutoDeployment(cls) -> bool:
         return True
 
     @classmethod
-    def supportsWorkerCleanup(cls):
+    def supportsWorkerCleanup(cls) -> bool:
         return True
 
     class ExecutorInfo:
-        def __init__(self, nodeAddress, agentId, nodeInfo, lastSeen):
+        def __init__(
+            self, nodeAddress: str, agentId: str, nodeInfo: str, lastSeen: str
+        ) -> None:
             super().__init__()
             self.nodeAddress = nodeAddress
             self.agentId = agentId
             self.nodeInfo = nodeInfo
             self.lastSeen = lastSeen
 
-    def __init__(self, config, maxCores, maxMemory, maxDisk):
+    def __init__(
+        self, config: Config, maxCores: float, maxMemory: float, maxDisk: int
+    ) -> None:
         super().__init__(config, maxCores, maxMemory, maxDisk)
 
         # The auto-deployed resource representing the user script. Will be passed along in every
@@ -165,13 +170,13 @@ class MesosBatchSystem(BatchSystemLocalSupport, AbstractScalableBatchSystem, Sch
 
         self._startDriver(config)
 
-    def setUserScript(self, userScript):
+    def setUserScript(self, userScript: str) -> None:
         self.userScript = userScript
 
-    def ignoreNode(self, nodeAddress):
+    def ignoreNode(self, nodeAddress: str) -> None:
         self.ignoredNodes.add(nodeAddress)
 
-    def unignoreNode(self, nodeAddress):
+    def unignoreNode(self, nodeAddress: str) -> None:
         self.ignoredNodes.remove(nodeAddress)
 
     def issueBatchJob(
@@ -179,7 +184,7 @@ class MesosBatchSystem(BatchSystemLocalSupport, AbstractScalableBatchSystem, Sch
         command: str,
         jobNode: JobDescription,
         job_environment: Optional[dict[str, str]] = None,
-    ):
+    ) -> str:
         """
         Issues the following command returning a unique jobID. Command is the string to run, memory
         is an int giving the number of bytes the job needs to run in and cores is the number of cpus

toil/batchSystems/mesos/test/__init__.py

@@ -7,6 +7,7 @@ from abc import ABCMeta, abstractmethod
 from contextlib import closing
 from shutil import which
 from urllib.request import urlopen
+from typing import Optional
 
 from toil.lib.retry import retry
 from toil.lib.threading import ExceptionalThread, cpu_count
@@ -25,7 +26,7 @@ class MesosTestSupport:
         with closing(urlopen("http://127.0.0.1:5050/version")) as content:
             content.read()
 
-    def _startMesos(self, numCores=None):
+    def _startMesos(self, numCores: Optional[int] = None) -> None:
         if numCores is None:
             numCores = cpu_count()
         shutil.rmtree("/tmp/mesos", ignore_errors=True)
@@ -52,7 +53,7 @@ class MesosTestSupport:
             log.warning("Forcibly killing child which ignored SIGTERM")
             process.kill()
 
-    def _stopMesos(self):
+    def _stopMesos(self) -> None:
         self._stopProcess(self.agent.popen)
         self.agent.join()
         self._stopProcess(self.master.popen)

toil/batchSystems/options.py

@@ -185,6 +185,7 @@ def add_all_batchsystem_options(parser: Union[ArgumentParser, _ArgumentGroup]) -
         "systems such as gridengine, htcondor, torque, slurm, and lsf.",
     )
 
+    # TODO: Move this to Slurm specifically.
     parser.add_argument(
         "--memoryIsProduct",
         dest="memory_is_product",

toil/batchSystems/singleMachine.py

@@ -94,7 +94,7 @@ class SingleMachineBatchSystem(BatchSystemSupport):
         self,
         config: Config,
         maxCores: float,
-        maxMemory: int,
+        maxMemory: float,
         maxDisk: int,
         max_jobs: Optional[int] = None,
     ) -> None: