toil 6.1.0a1__py3-none-any.whl → 8.0.0__py3-none-any.whl

toil/batchSystems/cleanup_support.py

@@ -13,16 +13,16 @@
 # limitations under the License.
 import logging
 from types import TracebackType
-from typing import Any, ContextManager, List, Optional, Type
+from typing import Any, ContextManager, Optional
 
-from toil.batchSystems.abstractBatchSystem import (BatchSystemSupport,
-                                                   WorkerCleanupInfo)
+from toil.batchSystems.abstractBatchSystem import BatchSystemSupport, WorkerCleanupInfo
 from toil.batchSystems.local_support import BatchSystemLocalSupport
 from toil.common import Config, Toil
 from toil.lib.threading import LastProcessStandingArena
 
 logger = logging.getLogger(__name__)
 
+
 class BatchSystemCleanupSupport(BatchSystemLocalSupport):
     """
     Adds cleanup support when the last running job leaves a node, for batch
@@ -33,7 +33,7 @@ class BatchSystemCleanupSupport(BatchSystemLocalSupport):
     def supportsWorkerCleanup(cls) -> bool:
         return True
 
-    def getWorkerContexts(self) -> List[ContextManager[Any]]:
+    def getWorkerContexts(self) -> list[ContextManager[Any]]:
         # Tell worker to register for and invoke cleanup
 
         # Create a context manager that has a copy of our cleanup info
@@ -44,9 +44,12 @@ class BatchSystemCleanupSupport(BatchSystemLocalSupport):
         contexts.append(context)
         return contexts
 
-    def __init__(self, config: Config, maxCores: float, maxMemory: int, maxDisk: int) -> None:
+    def __init__(
+        self, config: Config, maxCores: float, maxMemory: int, maxDisk: int
+    ) -> None:
         super().__init__(config, maxCores, maxMemory, maxDisk)
 
+
 class WorkerCleanupContext:
     """
     Context manager used by :class:`BatchSystemCleanupSupport` to implement
@@ -63,31 +66,38 @@ class WorkerCleanupContext:
                                   the last to exit the context manager.
         """
 
-
         self.workerCleanupInfo = workerCleanupInfo
         # Don't set self.arena or MyPy will be upset that sometimes it doesn't have the right type.
 
     def __enter__(self) -> None:
         # Set up an arena so we know who is the last worker to leave
-        self.arena = LastProcessStandingArena(Toil.get_toil_coordination_dir(self.workerCleanupInfo.work_dir, self.workerCleanupInfo.coordination_dir),
-                                              self.workerCleanupInfo.workflow_id + '-cleanup')
-        logger.debug('Entering cleanup arena')
+        self.arena = LastProcessStandingArena(
+            Toil.get_toil_coordination_dir(
+                self.workerCleanupInfo.work_dir, self.workerCleanupInfo.coordination_dir
+            ),
+            Toil.get_workflow_path_component(self.workerCleanupInfo.workflow_id)
+            + "-cleanup",
+        )
+        logger.debug("Entering cleanup arena")
         self.arena.enter()
-        logger.debug('Cleanup arena entered')
+        logger.debug("Cleanup arena entered")
 
     # This is exactly the signature MyPy demands.
     # Also, it demands we not say we can return a bool if we return False
     # always, because it can be smarter about reachability if it knows what
     # context managers never eat exceptions. So it decides any context manager
     # that is always falsey but claims to return a bool is an error.
-    def __exit__(self, type: Optional[Type[BaseException]], value: Optional[BaseException], traceback: Optional[TracebackType]) -> None:
-        logger.debug('Leaving cleanup arena')
+    def __exit__(
+        self,
+        type: Optional[type[BaseException]],
+        value: Optional[BaseException],
+        traceback: Optional[TracebackType],
+    ) -> None:
+        logger.debug("Leaving cleanup arena")
         for _ in self.arena.leave():
             # We are the last concurrent worker to finish.
             # Do batch system cleanup.
-            logger.debug('Cleaning up worker')
+            logger.debug("Cleaning up worker")
             BatchSystemSupport.workerCleanup(self.workerCleanupInfo)
             # Now the coordination_dir is allowed to no longer exist on the node.
-        logger.debug('Cleanup arena left')
-
-
+        logger.debug("Cleanup arena left")

toil/batchSystems/contained_executor.py

@@ -22,21 +22,24 @@ import os
 import pickle
 import subprocess
 import sys
-from typing import Any, Dict, List, Optional
+from typing import Any, Optional
 
 from toil.batchSystems.abstractBatchSystem import EXIT_STATUS_UNAVAILABLE_VALUE
-from toil.job import JobDescription
 from toil.resource import Resource
 from toil.statsAndLogging import configure_root_logger, set_log_level
 
 logger = logging.getLogger(__name__)
 
 
-def pack_job(job_desc: JobDescription, user_script: Optional[Resource] = None, environment: Optional[Dict[str, str]] = None) -> List[str]:
+def pack_job(
+    command: str,
+    user_script: Optional[Resource] = None,
+    environment: Optional[dict[str, str]] = None,
+) -> list[str]:
     """
-    Create a command that, when run, will execute the given job.
+    Create a command that runs the given command in an environment.
 
-    :param job_desc: Job description for the job to run.
+    :param command: Worker command to run to run the job.
     :param user_script: User script that will be loaded before the job is run.
     :param environment: Environment variable dict that will be applied before
         the job is run.
@@ -46,19 +49,21 @@ def pack_job(job_desc: JobDescription, user_script: Optional[Resource] = None, e
     """
     # Make a job dict to send to the executor.
     # TODO: Factor out executor setup from here and Kubernetes and TES
-    job: Dict[str, Any] = {"command": job_desc.command}
+    job: dict[str, Any] = {"command": command}
     if user_script is not None:
         # If there's a user script resource be sure to send it along
-        job['userScript'] = user_script
+        job["userScript"] = user_script
     if environment is not None:
         # We also may have an environment to send.
-        job['environment'] = environment
+        job["environment"] = environment
     # Encode it in a form we can send in a command-line argument. Pickle in
     # the highest protocol to prevent mixed-Python-version workflows from
     # trying to work. Make sure it is text so we can ship it via JSON.
-    encoded_job = base64.b64encode(pickle.dumps(job, pickle.HIGHEST_PROTOCOL)).decode('utf-8')
+    encoded_job = base64.b64encode(pickle.dumps(job, pickle.HIGHEST_PROTOCOL)).decode(
+        "utf-8"
+    )
     # Make a command to run it in the executor
-    command_list = ['_toil_contained_executor', encoded_job]
+    command_list = ["_toil_contained_executor", encoded_job]
 
     return command_list
 
@@ -82,53 +87,51 @@ def executor() -> None:
     exit_code = EXIT_STATUS_UNAVAILABLE_VALUE
 
     if len(sys.argv) != 2:
-        logger.error('Executor requires exactly one base64-encoded argument')
+        logger.error("Executor requires exactly one base64-encoded argument")
         sys.exit(exit_code)
 
     # Take in a base64-encoded pickled dict as our first argument and decode it
     try:
         # Make sure to encode the text arguments to bytes before base 64 decoding
-        job = pickle.loads(base64.b64decode(sys.argv[1].encode('utf-8')))
+        job = pickle.loads(base64.b64decode(sys.argv[1].encode("utf-8")))
     except:
         exc_info = sys.exc_info()
-        logger.error('Exception while unpickling task: ', exc_info=exc_info)
+        logger.error("Exception while unpickling task: ", exc_info=exc_info)
         sys.exit(exit_code)
 
-    if 'environment' in job:
+    if "environment" in job:
         # Adopt the job environment into the executor.
         # This lets us use things like TOIL_WORKDIR when figuring out how to talk to other executors.
-        logger.debug('Adopting environment: %s', str(job['environment'].keys()))
-        for var, value in job['environment'].items():
+        logger.debug("Adopting environment: %s", str(job["environment"].keys()))
+        for var, value in job["environment"].items():
             os.environ[var] = value
 
     # Set JTRES_ROOT and other global state needed for resource
     # downloading/deployment to work.
     # TODO: Every worker downloads resources independently.
     # We should have a way to share a resource directory.
-    logger.debug('Preparing system for resource download')
+    logger.debug("Preparing system for resource download")
     Resource.prepareSystem()
     try:
-        if 'userScript' in job:
-            job['userScript'].register()
+        if "userScript" in job:
+            job["userScript"].register()
 
         # Start the child process
-        logger.debug("Invoking command: '%s'", job['command'])
-        child = subprocess.Popen(job['command'],
-                                 preexec_fn=lambda: os.setpgrp(),
-                                 shell=True)
+        logger.debug("Invoking command: '%s'", job["command"])
+        child = subprocess.Popen(
+            job["command"], preexec_fn=lambda: os.setpgrp(), shell=True
+        )
 
         # Reproduce child's exit code
         exit_code = child.wait()
     except:
         # This will print a traceback for us, since exit() in the finally
         # will bypass the normal way of getting one.
-        logger.exception('Encountered exception running child')
+        logger.exception("Encountered exception running child")
     finally:
-        logger.debug('Cleaning up resources')
+        logger.debug("Cleaning up resources")
         # TODO: Change resource system to use a shared resource directory for everyone.
         # Then move this into worker cleanup somehow
         Resource.cleanSystem()
-        logger.debug('Shutting down')
+        logger.debug("Shutting down")
         sys.exit(exit_code)
-
-

toil/batchSystems/gridengine.py

@@ -17,10 +17,11 @@ import os
 import shlex
 import time
 from shlex import quote
-from typing import Dict, List, Optional
+from typing import Optional
 
-from toil.batchSystems.abstractGridEngineBatchSystem import \
-    AbstractGridEngineBatchSystem
+from toil.batchSystems.abstractGridEngineBatchSystem import (
+    AbstractGridEngineBatchSystem,
+)
 from toil.lib.misc import CalledProcessErrorStderr, call_command
 
 logger = logging.getLogger(__name__)
@@ -28,37 +29,42 @@ logger = logging.getLogger(__name__)
 
 class GridEngineBatchSystem(AbstractGridEngineBatchSystem):
 
-    class Worker(AbstractGridEngineBatchSystem.Worker):
+    class GridEngineThread(AbstractGridEngineBatchSystem.GridEngineThread):
         """
         Grid Engine-specific AbstractGridEngineWorker methods
         """
+
         def getRunningJobIDs(self):
             times = {}
             with self.runningJobsLock:
                 currentjobs = {str(self.batchJobIDs[x][0]): x for x in self.runningJobs}
             stdout = call_command(["qstat"])
 
-            for currline in stdout.split('\n'):
+            for currline in stdout.split("\n"):
                 items = currline.strip().split()
                 if items:
-                    if items[0] in currentjobs and items[4] == 'r':
+                    if items[0] in currentjobs and items[4] == "r":
                         jobstart = " ".join(items[5:7])
-                        jobstart = time.mktime(time.strptime(jobstart, "%m/%d/%Y %H:%M:%S"))
+                        jobstart = time.mktime(
+                            time.strptime(jobstart, "%m/%d/%Y %H:%M:%S")
+                        )
                         times[currentjobs[items[0]]] = time.time() - jobstart
 
             return times
 
         def killJob(self, jobID):
-            call_command(['qdel', self.getBatchSystemID(jobID)])
-
-        def prepareSubmission(self,
-                              cpu: int,
-                              memory: int,
-                              jobID: int,
-                              command: str,
-                              jobName: str,
-                              job_environment: Optional[Dict[str, str]] = None,
-                              gpus: Optional[int] = None):
+            call_command(["qdel", self.getBatchSystemID(jobID)])
+
+        def prepareSubmission(
+            self,
+            cpu: int,
+            memory: int,
+            jobID: int,
+            command: str,
+            jobName: str,
+            job_environment: Optional[dict[str, str]] = None,
+            gpus: Optional[int] = None,
+        ):
             # POSIX qsub
             # <https://pubs.opengroup.org/onlinepubs/9699919799.2008edition/utilities/qsub.html>
             # expects a single script argument, which is supposed to be a file.
@@ -67,11 +73,13 @@ class GridEngineBatchSystem(AbstractGridEngineBatchSystem):
             # hope that the qsub we are using is clever enough to forward along
             # arguments. Otherwise, some qsubs will go looking for the full
             # Toil command string as a file.
-            return self.prepareQsub(cpu, memory, jobID, job_environment) + shlex.split(command)
+            return self.prepareQsub(cpu, memory, jobID, job_environment) + shlex.split(
+                command
+            )
 
         def submitJob(self, subLine):
             stdout = call_command(subLine)
-            output = stdout.split('\n')[0].strip()
+            output = stdout.split("\n")[0].strip()
             result = int(output)
             return result
 
@@ -84,8 +92,8 @@ class GridEngineBatchSystem(AbstractGridEngineBatchSystem):
             """
             # the task is set as part of the job ID if using getBatchSystemID()
             job, task = (sgeJobID, None)
-            if '.' in sgeJobID:
-                job, task = sgeJobID.split('.', 1)
+            if "." in sgeJobID:
+                job, task = sgeJobID.split(".", 1)
             assert task is None, "task ids not currently support by qstat logic below"
 
             # First try qstat to see if job is still running, if not get the
@@ -101,66 +109,94 @@ class GridEngineBatchSystem(AbstractGridEngineBatchSystem):
             if task is not None:
                 args.extend(["-t", str(task)])
             stdout = call_command(args)
-            for line in stdout.split('\n'):
+            for line in stdout.split("\n"):
                 if line.startswith("failed") and int(line.split()[1]) == 1:
                     return 1
                 elif line.startswith("exit_status"):
-                    logger.debug('Exit Status: %r', line.split()[1])
+                    logger.debug("Exit Status: %r", line.split()[1])
                     return int(line.split()[1])
             return None
 
         """
         Implementation-specific helper methods
         """
-        def prepareQsub(self,
-                        cpu: int,
-                        mem: int,
-                        jobID: int,
-                        job_environment: Optional[Dict[str, str]] = None) -> List[str]:
-            qsubline = ['qsub', '-V', '-b', 'y', '-terse', '-j', 'y', '-cwd',
-                        '-N', 'toil_job_' + str(jobID)]
+
+        def prepareQsub(
+            self,
+            cpu: int,
+            mem: int,
+            jobID: int,
+            job_environment: Optional[dict[str, str]] = None,
+        ) -> list[str]:
+            qsubline = [
+                "qsub",
+                "-V",
+                "-b",
+                "y",
+                "-terse",
+                "-j",
+                "y",
+                "-cwd",
+                "-N",
+                "toil_job_" + str(jobID),
+            ]
 
             environment = self.boss.environment.copy()
             if job_environment:
                 environment.update(job_environment)
 
             if environment:
-                qsubline.append('-v')
-                qsubline.append(','.join(k + '=' + quote(os.environ[k] if v is None else v)
-                                         for k, v in environment.items()))
+                qsubline.append("-v")
+                qsubline.append(
+                    ",".join(
+                        k + "=" + quote(os.environ[k] if v is None else v)
+                        for k, v in environment.items()
+                    )
+                )
 
             reqline = list()
-            sgeArgs = os.getenv('TOIL_GRIDENGINE_ARGS')
+            sgeArgs = os.getenv("TOIL_GRIDENGINE_ARGS")
             if mem is not None:
-                memStr = str(mem // 1024) + 'K'
+                memStr = str(mem // 1024) + "K"
                 if not self.boss.config.manualMemArgs:
                     # for UGE instead of SGE; see #2309
-                    reqline += ['vf=' + memStr, 'h_vmem=' + memStr]
+                    reqline += ["vf=" + memStr, "h_vmem=" + memStr]
                 elif self.boss.config.manualMemArgs and not sgeArgs:
-                    raise ValueError("--manualMemArgs set to True, but TOIL_GRIDGENGINE_ARGS is not set."
-                                     "Please set TOIL_GRIDGENGINE_ARGS to specify memory allocation for "
-                                     "your system.  Default adds the arguments: vf=<mem> h_vmem=<mem> "
-                                     "to qsub.")
+                    raise ValueError(
+                        "--manualMemArgs set to True, but TOIL_GRIDGENGINE_ARGS is not set."
+                        "Please set TOIL_GRIDGENGINE_ARGS to specify memory allocation for "
+                        "your system.  Default adds the arguments: vf=<mem> h_vmem=<mem> "
+                        "to qsub."
+                    )
             if len(reqline) > 0:
-                qsubline.extend(['-hard', '-l', ','.join(reqline)])
+                qsubline.extend(["-hard", "-l", ",".join(reqline)])
             if sgeArgs:
                 sgeArgs = sgeArgs.split()
                 for arg in sgeArgs:
                     if arg.startswith(("vf=", "h_vmem=", "-pe")):
-                        raise ValueError("Unexpected CPU, memory or pe specifications in TOIL_GRIDGENGINE_ARGs: %s" % arg)
+                        raise ValueError(
+                            "Unexpected CPU, memory or pe specifications in TOIL_GRIDGENGINE_ARGs: %s"
+                            % arg
+                        )
                 qsubline.extend(sgeArgs)
             # If cpu == 1 (or None) then don't add PE env variable to the qsub command.
             #               This will allow for use of the serial queue for these jobs.
-            if (os.getenv('TOIL_GRIDENGINE_PE') is not None) and (cpu is not None) and (cpu > 1) :
+            if (
+                (os.getenv("TOIL_GRIDENGINE_PE") is not None)
+                and (cpu is not None)
+                and (cpu > 1)
+            ):
                 peCpu = int(math.ceil(cpu))
-                qsubline.extend(['-pe', os.getenv('TOIL_GRIDENGINE_PE'), str(peCpu)])
+                qsubline.extend(["-pe", os.getenv("TOIL_GRIDENGINE_PE"), str(peCpu)])
             elif (cpu is not None) and (cpu > 1):
-                raise RuntimeError("must specify PE in TOIL_GRIDENGINE_PE environment variable when using multiple CPUs. "
-                                   "Run qconf -spl and your local documentation for possible values")
-
-            stdoutfile: str = self.boss.format_std_out_err_path(jobID, '$JOB_ID', 'out')
-            stderrfile: str = self.boss.format_std_out_err_path(jobID, '$JOB_ID', 'err')
-            qsubline.extend(['-o', stdoutfile, '-e', stderrfile])
+                raise RuntimeError(
+                    "must specify PE in TOIL_GRIDENGINE_PE environment variable when using multiple CPUs. "
+                    "Run qconf -spl and your local documentation for possible values"
+                )
+
+            stdoutfile: str = self.boss.format_std_out_err_path(jobID, "$JOB_ID", "out")
+            stderrfile: str = self.boss.format_std_out_err_path(jobID, "$JOB_ID", "err")
+            qsubline.extend(["-o", stdoutfile, "-e", stderrfile])
 
             return qsubline