toil 6.1.0a1__py3-none-any.whl → 8.0.0__py3-none-any.whl

toil/test/batchSystems/batch_system_plugin_test.py

@@ -0,0 +1,90 @@
+# Copyright (C) 2015-2021 Regents of the University of California
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+from typing import Optional
+
+from configargparse import ArgParser, ArgumentParser
+
+from toil.batchSystems.abstractBatchSystem import (
+    AbstractBatchSystem,
+    UpdatedBatchJobInfo,
+)
+from toil.batchSystems.cleanup_support import BatchSystemCleanupSupport
+from toil.batchSystems.options import OptionSetter
+from toil.batchSystems.registry import add_batch_system_factory
+from toil.common import Toil, addOptions
+from toil.job import JobDescription
+from toil.test import ToilTest
+
+logger = logging.getLogger(__name__)
+
+
+class FakeBatchSystem(BatchSystemCleanupSupport):
+    @classmethod
+    def supportsAutoDeployment(cls) -> bool:
+        pass
+
+    def issueBatchJob(
+        self,
+        command: str,
+        job_desc: JobDescription,
+        job_environment: Optional[dict[str, str]] = None,
+    ) -> int:
+        pass
+
+    def killBatchJobs(self, jobIDs: list[int]) -> None:
+        pass
+
+    def getIssuedBatchJobIDs(self) -> list[int]:
+        pass
+
+    def getRunningBatchJobIDs(self) -> dict[int, float]:
+        pass
+
+    def getUpdatedBatchJob(self, maxWait: int) -> Optional[UpdatedBatchJobInfo]:
+        pass
+
+    def shutdown(self) -> None:
+        pass
+
+    @classmethod
+    def add_options(cls, parser: ArgumentParser) -> None:
+        parser.add_argument("--fake_argument", default="exists")
+
+    @classmethod
+    def setOptions(cls, setOption: OptionSetter) -> None:
+        setOption("fake_argument")
+
+
+class BatchSystemPluginTest(ToilTest):
+    def test_batchsystem_plugin_installable(self):
+        """
+        Test that installing a batch system plugin works.
+        :return:
+        """
+
+        def fake_batch_system_factory() -> type[AbstractBatchSystem]:
+            return FakeBatchSystem
+
+        add_batch_system_factory("fake", fake_batch_system_factory)
+
+        parser = ArgParser()
+        addOptions(parser)
+
+        options = parser.parse_args(["test-jobstore", "--clean=always"])
+
+        # try to install a batchsystem plugin with some arguments
+        # if the arguments exists, the values should also exist in the config
+        with Toil(options) as toil:
+            self.assertEqual(toil.config.fake_argument == "exists", True)

toil/test/batchSystems/test_gridengine.py

@@ -0,0 +1,173 @@
+import textwrap
+from queue import Queue
+
+import pytest
+
+import toil.batchSystems.gridengine
+from toil.batchSystems.abstractGridEngineBatchSystem import ExceededRetryAttempts
+from toil.common import Config
+from toil.lib.misc import CalledProcessErrorStderr
+from toil.test import ToilTest
+
+
+class FakeBatchSystem:
+    """
+    Class that implements a minimal Batch System, needed to create a Worker (see below).
+    """
+
+    def __init__(self):
+        self.config = self.__fake_config()
+
+    def getWaitDuration(self):
+        return 10
+
+    def __fake_config(self):
+        """
+        Returns a dummy config for the batch system tests.  We need a workflowID to be set up
+        since we are running tests without setting up a jobstore. This is the class version
+        to be used when an instance is not available.
+
+        :rtype: toil.common.Config
+        """
+        config = Config()
+        from uuid import uuid4
+
+        config.workflowID = str(uuid4())
+        config.cleanWorkDir = "always"
+        return config
+
+    def with_retries(self, operation, *args, **kwargs):
+        """
+        The grid engine batch system needs a with_retries function when running the GridEngineThread, so fake one
+        """
+        return operation(*args, **kwargs)
+
+
+def call_qstat_or_qacct(args, **_):
+    # example outputs taken from https://2021.help.altair.com/2021.1/AltairGridEngine/8.7.0/UsersGuideGE.pdf
+    qacct_info = {}
+    job_id_info = {
+        1: {"failed": True, "exit_code": 0, "completed": True},
+        2: {"failed": True, "exit_code": 2, "completed": True},
+        3: {"failed": False, "exit_code": 0, "completed": True},
+        4: {"failed": False, "exit_code": 10, "completed": True},
+        5: {"failed": False, "exit_code": 0, "completed": False},
+    }
+    for job_id, status_info in job_id_info.items():
+        failed = 1 if status_info["failed"] else 0
+        exit_status = status_info["exit_code"]
+        qacct_info[job_id] = textwrap.dedent(
+            f"""\
+            ==============================================================
+            qname all.q
+            hostname kailua
+            group users
+            owner jondoe
+            project NONE
+            department defaultdepartment
+            jobname Sleeper
+            jobnumber 10
+            taskid undefined
+            account sge
+            priority 0
+            qsub_time Thu Mar 10 19:58:35 2011
+            start_time Thu Mar 10 19:58:42 2011
+            end_time Thu Mar 10 19:59:43 2011
+            granted_pe NONE
+            slots 1
+            failed {failed}
+            exit_status {exit_status}
+            ru_wallclock 61
+            ru_utime 0.070
+            ru_stime 0.050
+            ru_maxrss 1220
+            ru_ixrss 0
+            ru_ismrss 0
+            ru_idrss 0
+        """
+        )
+    if args[0] == "qstat":
+        # This is guess for what qstat will return given a job. I'm unable to find an example for qstat.
+        # This also assumes the second argument args[1] is -j, as that is what we try to use
+        job_id = int(args[2])
+        if job_id not in job_id_info.keys() or job_id_info[job_id]["completed"]:
+            stderr = f"Following jobs do not exist {job_id}"
+        else:
+            # This is not the output of qstat when the job is running, and is just a guess
+            # We test on the existence of the string "Following jobs do not exist", so this should be okay for now
+            stderr = f"Job exists {job_id}"
+        raise CalledProcessErrorStderr(2, args, stderr=stderr)
+    elif args[0] == "qacct":
+        if args[1] != "-j":
+            # Documentation for qacct says if -j is not found then all jobs are listed
+            # https://gridscheduler.sourceforge.net/htmlman/htmlman1/qacct.html
+            # This is a guess for the output of qacct. We don't have a SGE cluster and I can't find a bare qacct example output online
+            qacct_response = "\n".join(qacct_info.values())
+        else:
+            job_id = int(args[2])
+            if job_id not in job_id_info.keys():
+                # This is a guess of the behavior when the job does not exist. Since the behavior is unknown, this is not currently tested
+                return ""
+            qacct_response = qacct_info[job_id]
+
+        return qacct_response
+
+
+class GridEngineTest(ToilTest):
+    """
+    Class for unit-testing GridEngineBatchSystem
+    """
+
+    def setUp(self):
+        self.monkeypatch = pytest.MonkeyPatch()
+        self.worker = (
+            toil.batchSystems.gridengine.GridEngineBatchSystem.GridEngineThread(
+                newJobsQueue=Queue(),
+                updatedJobsQueue=Queue(),
+                killQueue=Queue(),
+                killedJobsQueue=Queue(),
+                boss=FakeBatchSystem(),
+            )
+        )
+
+    ###
+    ### Tests for coalesce_job_exit_codes for gridengine.
+    ###
+
+    def test_coalesce_job_exit_codes_one_exists(self):
+        self.monkeypatch.setattr(
+            toil.batchSystems.gridengine, "call_command", call_qstat_or_qacct
+        )
+        job_ids = ["1"]  # FAILED
+        expected_result = [1]
+        result = self.worker.coalesce_job_exit_codes(job_ids)
+        assert result == expected_result, f"{result} != {expected_result}"
+
+    def test_coalesce_job_exit_codes_one_still_running(self):
+        self.monkeypatch.setattr(
+            toil.batchSystems.gridengine, "call_command", call_qstat_or_qacct
+        )
+        job_ids = [
+            "5"
+        ]  # Still running. We currently raise an exception when this happens
+        try:
+            self.worker.coalesce_job_exit_codes(job_ids)
+        except ExceededRetryAttempts:
+            pass
+        else:
+            raise RuntimeError("Test did not raise an exception!")
+
+    def test_coalesce_job_exit_codes_many_all_exist(self):
+        self.monkeypatch.setattr(
+            toil.batchSystems.gridengine, "call_command", call_qstat_or_qacct
+        )
+        job_ids = [
+            "1",  # FAILED,
+            "2",  # FAILED (with exit code that we ignore),
+            "3",  # SUCCEEDED,
+            "4",
+        ]  # EXIT CODE 10
+        # RUNNING and PENDING jobs should return None
+        expected_result = [1, 1, 0, 10]
+        result = self.worker.coalesce_job_exit_codes(job_ids)
+        assert result == expected_result, f"{result} != {expected_result}"

toil/test/batchSystems/test_lsf_helper.py

@@ -1,4 +1,5 @@
 """lsfHelper.py shouldn't need a batch system and so the unit tests here should aim to run on any system."""
+
 from toil.batchSystems.lsfHelper import parse_mem_and_cmd_from_output
 from toil.test import ToilTest
 
@@ -6,69 +7,77 @@ from toil.test import ToilTest
 class LSFHelperTest(ToilTest):
     def test_parse_mem_and_cmd_from_output(self):
         # https://github.com/DataBiosphere/toil/pull/3475
-        output = ('\nJob <2924748>, Job Name <toil_job_64>, User <thiagogenez>, Project <default>, S'
-                  '\n                     tatus <RUN>, Queue <research-rh74>, Job Priority <50>, Com'
-                  '\n                     mand <_toil_worker CactusBarRecursion file:/hps/nobackup/p'
-                  '\n                     roduction/ensembl/thiagogenez/pairwises/arabidopsis/run/jo'
-                  '\n                     bstore/3 kind-CactusBarRecursion/instance-iu6wo56x --conte'
-                  '\n                     xt gAShortenedh32xqlE51Yi4=>, Share group charged </thiago'
-                  '\n                     genez>, Esub <esub>'
-                  '\nThu Mar 18 02:06:32: Submitted from host <noah-login-01>, CWD </hps/nobackup/pr'
-                  '\n                     oduction/ensembl/thiagogenez/pairwises/arabidopsis/run>, S'
-                  '\n                     pecified CWD </hps/nobackup/production/ensembl/thiagogenez'
-                  '\n                     /pairwises/arabidopsis/run/.>, Output File </tmp/toil_work'
-                  '\n                     flow_10e83102-2e4b-4093-9128-2a52f4bbc65a_job_64_batch_lsf'
-                  '\n                     _2924748_std_output.log>, Error File </tmp/toil_workflow_1'
-                  '\n                     0e83102-2e4b-4093-9128-2a52f4bbc65a_job_64_batch_lsf_29247'
-                  '\n                     48_std_error.log>, Requested Resources <select[(mem>4000)]'
-                  '\n                      rusage[mem=4000:duration=480, numcpus=1:duration=480]>;'
-                  '\nThu Mar 18 02:06:33: Started on <hx-noah-31-02>, Execution Home </homes/thiagog'
-                  '\n                     enez>, Execution CWD </hps/nobackup/production/ensembl/thi'
-                  '\n                     agogenez/pairwises/arabidopsis/run/.>;'
-                  '\nThu Mar 18 17:07:47: Resource usage collected.'
-                  '\n                     The CPU time used is 53936 seconds.'
-                  '\n                     MEM: 344 Mbytes;  SWAP: 1.3 Gbytes;  NTHREAD: 5'
-                  '\n                     PGID: 433168;  PIDs: 433168 433177 433179 444026 '
-                  '\n'
-                  '\n RUNLIMIT                '
-                  '\n 10085.0 min'
-                  '\n'
-                  '\n CORELIMIT MEMLIMIT'
-                  '\n      0 M     3.9 G '
-                  '\n'
-                  '\n MEMORY USAGE:'
-                  '\n MAX MEM: 2.5 Gbytes;  AVG MEM: 343 Mbytes'
-                  '\n'
-                  '\n SCHEDULING PARAMETERS:'
-                  '\n           r15s   r1m  r15m   ut      pg    io   ls    it    tmp    swp    mem'
-                  '\n loadSched   -     -     -     -    10.0     -    -     -   500M     -   1000M '
-                  '\n loadStop    -     -     -     -       -     -    -     -     -      -      -  '
-                  '\n'
-                  '\n          availcpus '
-                  '\n loadSched      1.0 '
-                  '\n loadStop        -  '
-                  '\n'
-                  '\n RESOURCE REQUIREMENT DETAILS:'
-                  '\n Combined: select[((mem>4000)) && (type == local)] order[r15s:pg] rusage[mem=40'
-                  '\n                     00.00:duration=8h:decay=0,numcpus=1.00:duration=8h:decay=0'
-                  '\n                     ] span[hosts=1]'
-                  '\n Effective: select[(((mem>4000))) && (type == local)] order[r15s:pg] rusage[mem'
-                  '\n                     =4000.00:duration=8h:decay=0,numcpus=1.00:duration=8h:deca'
-                  '\n                     y=0] span[hosts=1] '
-                  '\n'
-                  '\n')
+        output = (
+            "\nJob <2924748>, Job Name <toil_job_64>, User <thiagogenez>, Project <default>, S"
+            "\n                     tatus <RUN>, Queue <research-rh74>, Job Priority <50>, Com"
+            "\n                     mand <_toil_worker CactusBarRecursion file:/hps/nobackup/p"
+            "\n                     roduction/ensembl/thiagogenez/pairwises/arabidopsis/run/jo"
+            "\n                     bstore/3 kind-CactusBarRecursion/instance-iu6wo56x --conte"
+            "\n                     xt gAShortenedh32xqlE51Yi4=>, Share group charged </thiago"
+            "\n                     genez>, Esub <esub>"
+            "\nThu Mar 18 02:06:32: Submitted from host <noah-login-01>, CWD </hps/nobackup/pr"
+            "\n                     oduction/ensembl/thiagogenez/pairwises/arabidopsis/run>, S"
+            "\n                     pecified CWD </hps/nobackup/production/ensembl/thiagogenez"
+            "\n                     /pairwises/arabidopsis/run/.>, Output File </tmp/toil_work"
+            "\n                     flow_10e83102-2e4b-4093-9128-2a52f4bbc65a_job_64_batch_lsf"
+            "\n                     _2924748_std_output.log>, Error File </tmp/toil_workflow_1"
+            "\n                     0e83102-2e4b-4093-9128-2a52f4bbc65a_job_64_batch_lsf_29247"
+            "\n                     48_std_error.log>, Requested Resources <select[(mem>4000)]"
+            "\n                      rusage[mem=4000:duration=480, numcpus=1:duration=480]>;"
+            "\nThu Mar 18 02:06:33: Started on <hx-noah-31-02>, Execution Home </homes/thiagog"
+            "\n                     enez>, Execution CWD </hps/nobackup/production/ensembl/thi"
+            "\n                     agogenez/pairwises/arabidopsis/run/.>;"
+            "\nThu Mar 18 17:07:47: Resource usage collected."
+            "\n                     The CPU time used is 53936 seconds."
+            "\n                     MEM: 344 Mbytes;  SWAP: 1.3 Gbytes;  NTHREAD: 5"
+            "\n                     PGID: 433168;  PIDs: 433168 433177 433179 444026 "
+            "\n"
+            "\n RUNLIMIT                "
+            "\n 10085.0 min"
+            "\n"
+            "\n CORELIMIT MEMLIMIT"
+            "\n      0 M     3.9 G "
+            "\n"
+            "\n MEMORY USAGE:"
+            "\n MAX MEM: 2.5 Gbytes;  AVG MEM: 343 Mbytes"
+            "\n"
+            "\n SCHEDULING PARAMETERS:"
+            "\n           r15s   r1m  r15m   ut      pg    io   ls    it    tmp    swp    mem"
+            "\n loadSched   -     -     -     -    10.0     -    -     -   500M     -   1000M "
+            "\n loadStop    -     -     -     -       -     -    -     -     -      -      -  "
+            "\n"
+            "\n          availcpus "
+            "\n loadSched      1.0 "
+            "\n loadStop        -  "
+            "\n"
+            "\n RESOURCE REQUIREMENT DETAILS:"
+            "\n Combined: select[((mem>4000)) && (type == local)] order[r15s:pg] rusage[mem=40"
+            "\n                     00.00:duration=8h:decay=0,numcpus=1.00:duration=8h:decay=0"
+            "\n                     ] span[hosts=1]"
+            "\n Effective: select[(((mem>4000))) && (type == local)] order[r15s:pg] rusage[mem"
+            "\n                     =4000.00:duration=8h:decay=0,numcpus=1.00:duration=8h:deca"
+            "\n                     y=0] span[hosts=1] "
+            "\n"
+            "\n"
+        )
         max_mem, command = parse_mem_and_cmd_from_output(output=output)
         assert len(max_mem.groups()) == 1
-        expected_mem = '2.5 Gbytes'
-        assert max_mem.group(1) == expected_mem, f'Actual: {max_mem.group(1)}, Expected: "{expected_mem}"'
+        expected_mem = "2.5 Gbytes"
+        assert (
+            max_mem.group(1) == expected_mem
+        ), f'Actual: {max_mem.group(1)}, Expected: "{expected_mem}"'
 
         assert len(command.groups()) == 1
-        expected_command = ('_toil_worker CactusBarRecursion file:/hps/nobackup/production/ensembl/thiagogenez/'
-                            'pairwises/arabidopsis/run/jobstore/3 kind-CactusBarRecursion/instance-iu6wo56x '
-                            '--context gAShortenedh32xqlE51Yi4=')
-        assert command.group(1) == expected_command, f'Actual: {command.group(1)}, Expected: "{expected_command}"'
+        expected_command = (
+            "_toil_worker CactusBarRecursion file:/hps/nobackup/production/ensembl/thiagogenez/"
+            "pairwises/arabidopsis/run/jobstore/3 kind-CactusBarRecursion/instance-iu6wo56x "
+            "--context gAShortenedh32xqlE51Yi4="
+        )
+        assert (
+            command.group(1) == expected_command
+        ), f'Actual: {command.group(1)}, Expected: "{expected_command}"'
         print(command)
 
-        max_mem, command = parse_mem_and_cmd_from_output(output='')
+        max_mem, command = parse_mem_and_cmd_from_output(output="")
         assert max_mem == None
         assert command == None