toil 6.0.0__py3-none-any.whl → 6.1.0__py3-none-any.whl

toil/test/docs/scriptsTest.py

@@ -69,8 +69,8 @@ class ToilDocumentationTest(ToilTest):
     def testCwlexample(self):
         self.checkExitCode("tutorial_cwlexample.py")
 
-    def testDiscoverfiles(self):
-        self.checkExitCode("tutorial_discoverfiles.py")
+    def testStats(self):
+        self.checkExitCode("tutorial_stats.py")
 
     def testDynamic(self):
         self.checkExitCode("tutorial_dynamic.py")

toil/test/jobStores/jobStoreTest.py

@@ -40,7 +40,6 @@ from toil.job import Job, JobDescription, TemporaryID
 from toil.jobStores.abstractJobStore import (NoSuchFileException,
                                              NoSuchJobException)
 from toil.jobStores.fileJobStore import FileJobStore
-from toil.lib.aws.utils import create_s3_bucket, get_object_for_url
 from toil.lib.io import mkdtemp
 from toil.lib.memoize import memoize
 from toil.lib.retry import retry
@@ -548,14 +547,16 @@ class AbstractJobStoreTest:
             jobNames = ['testStatsAndLogging_writeLogFiles']
             jobLogList = ['string', b'bytes', '', b'newline\n']
             config = self._createConfig()
-            setattr(config, 'writeLogs', '.')
+            setattr(config, 'writeLogs', self._createTempDir())
             setattr(config, 'writeLogsGzip', None)
             StatsAndLogging.writeLogFiles(jobNames, jobLogList, config)
-            jobLogFile = os.path.join(config.writeLogs, jobNames[0] + '000.log')
+            jobLogFile = os.path.join(config.writeLogs, jobNames[0] + '_000.log')
+            # The log directory should get exactly one file, names after this
+            # easy job name with no replacements needed.
+            self.assertEqual(os.listdir(config.writeLogs), [os.path.basename(jobLogFile)])
             self.assertTrue(os.path.isfile(jobLogFile))
             with open(jobLogFile) as f:
                 self.assertEqual(f.read(), 'string\nbytes\n\nnewline\n')
-            os.remove(jobLogFile)
 
         def testBatchCreate(self):
             """Test creation of many jobs."""
@@ -1463,6 +1464,7 @@ class AWSJobStoreTest(AbstractJobStoreTest.Test):
 
     def _hashTestFile(self, url: str) -> str:
         from toil.jobStores.aws.jobStore import AWSJobStore
+        from toil.lib.aws.utils import get_object_for_url
         str(AWSJobStore)  # to prevent removal of that import
         key = get_object_for_url(urlparse.urlparse(url), existing=True)
         contents = key.get().get('Body').read()
@@ -1471,7 +1473,7 @@ class AWSJobStoreTest(AbstractJobStoreTest.Test):
     def _createExternalStore(self):
         """A S3.Bucket instance is returned"""
         from toil.jobStores.aws.jobStore import establish_boto3_session
-        from toil.lib.aws.utils import retry_s3
+        from toil.lib.aws.utils import retry_s3, create_s3_bucket
 
         resource = establish_boto3_session().resource(
             "s3", region_name=self.awsRegion()

toil/test/lib/test_ec2.py

@@ -20,7 +20,6 @@ from toil.lib.aws.ami import (aws_marketplace_flatcar_ami_search,
                               feed_flatcar_ami_release,
                               flatcar_release_feed_amis,
                               get_flatcar_ami)
-from toil.lib.aws.session import establish_boto3_session
 from toil.test import ToilTest, needs_aws_ec2, needs_online
 
 logger = logging.getLogger(__name__)
@@ -59,6 +58,7 @@ class FlatcarFeedTest(ToilTest):
 class AMITest(ToilTest):
     @classmethod
     def setUpClass(cls):
+        from toil.lib.aws.session import establish_boto3_session
         session = establish_boto3_session(region_name='us-west-2')
         cls.ec2_client = session.client('ec2')
 

toil/test/options/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (C) 2015-2021 Regents of the University of California
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

toil/test/options/options.py

@@ -0,0 +1,37 @@
+from configargparse import ArgParser
+
+from toil.common import addOptions, Toil
+from toil.test import ToilTest
+
+
+class OptionsTest(ToilTest):
+    """
+    Class to test functionality of all Toil options
+    """
+    def test_default_caching_slurm(self):
+        """
+        Test to ensure that caching will be set to false when running on Slurm
+        :return:
+        """
+        parser = ArgParser()
+        addOptions(parser, jobstore_as_flag=True, wdl=False, cwl=False)
+        test_args = ["--jobstore=example-jobstore", "--batchSystem=slurm"]
+        options = parser.parse_args(test_args)
+        with Toil(options) as toil:
+            caching_value = toil.config.caching
+        self.assertEqual(caching_value, False)
+
+    def test_caching_option_priority(self):
+        """
+        Test to ensure that the --caching option takes priority over the default_caching() return value
+        :return:
+        """
+        parser = ArgParser()
+        addOptions(parser, jobstore_as_flag=True, wdl=False, cwl=False)
+        # the kubernetes batchsystem (and I think all batchsystems including singlemachine) return False
+        # for default_caching
+        test_args = ["--jobstore=example-jobstore", "--batchSystem=kubernetes", "--caching=True"]
+        options = parser.parse_args(test_args)
+        with Toil(options) as toil:
+            caching_value = toil.config.caching
+        self.assertEqual(caching_value, True)

toil/test/provisioners/clusterTest.py

@@ -17,6 +17,7 @@ import os
 import subprocess
 import time
 from uuid import uuid4
+from typing import Optional, List
 
 from toil.lib.aws import zone_to_region
 from toil.lib.retry import retry
@@ -28,7 +29,7 @@ log = logging.getLogger(__name__)
 @needs_aws_ec2
 @needs_fetchable_appliance
 class AbstractClusterTest(ToilTest):
-    def __init__(self, methodName):
+    def __init__(self, methodName: str) -> None:
         super().__init__(methodName=methodName)
         self.keyName = os.getenv('TOIL_AWS_KEYNAME').strip() or 'id_rsa'
         self.clusterName = 'aws-provisioner-test-' + str(uuid4())
@@ -43,13 +44,13 @@ class AbstractClusterTest(ToilTest):
         # Where should we put our virtualenv?
         self.venvDir = '/tmp/venv'
 
-    def python(self):
+    def python(self) -> str:
         """
         Return the full path to the venv Python on the leader.
         """
         return os.path.join(self.venvDir, 'bin/python')
 
-    def pip(self):
+    def pip(self) -> str:
         """
         Return the full path to the venv pip on the leader.
         """
@@ -63,7 +64,7 @@ class AbstractClusterTest(ToilTest):
         """
         subprocess.check_call(['toil', 'destroy-cluster', '-p=aws', '-z', self.zone, self.clusterName])
 
-    def setUp(self):
+    def setUp(self) -> None:
         """
         Set up for the test.
         Must be overridden to call this method and set self.jobStore.
@@ -73,13 +74,13 @@ class AbstractClusterTest(ToilTest):
         # If this fails, no tests will run.
         self.destroyCluster()
 
-    def tearDown(self):
+    def tearDown(self) -> None:
         # Note that teardown will run even if the test crashes.
         super().tearDown()
         self.destroyCluster()
         subprocess.check_call(['toil', 'clean', self.jobStore])
 
-    def sshUtil(self, command):
+    def sshUtil(self, command: List[str]) -> None:
         """
         Run the given command on the cluster.
         Raise subprocess.CalledProcessError if it fails.
@@ -155,7 +156,7 @@ class AbstractClusterTest(ToilTest):
         subprocess.check_call(cmd)
 
     @retry(errors=[subprocess.CalledProcessError], intervals=[1, 1])
-    def createClusterUtil(self, args=None):
+    def createClusterUtil(self, args: Optional[List[str]]=None) -> None:
         args = [] if args is None else args
 
         command = ['toil', 'launch-cluster', '-p=aws', '-z', self.zone, f'--keyPairName={self.keyName}',
@@ -167,5 +168,5 @@ class AbstractClusterTest(ToilTest):
         subprocess.check_call(command)
         # If we fail, tearDown will destroy the cluster.
 
-    def launchCluster(self):
+    def launchCluster(self) -> None:
         self.createClusterUtil()

toil/test/utils/toilDebugTest.py

@@ -142,7 +142,7 @@ class DebugJobTest(ToilTest):
                 os.path.abspath("src/toil/test/docs/scripts/example_alwaysfail.py"),
                 "--retryCount=0",
                 "--logCritical",
-                "--disableProgress=True",
+                "--disableProgress",
                 job_store
             ], stderr=subprocess.DEVNULL)
             raise RuntimeError("Failing workflow succeeded!")

toil/test/utils/utilsTest.py

@@ -39,7 +39,7 @@ from toil.test import (ToilTest,
                        needs_rsync3,
                        slow)
 from toil.test.sort.sortTest import makeFileToSort
-from toil.utils.toilStats import getStats, processData
+from toil.utils.toilStats import get_stats, process_data
 from toil.utils.toilStatus import ToilStatus
 from toil.version import python
 
@@ -298,8 +298,8 @@ class UtilsTest(ToilTest):
         config = Config()
         config.setOptions(options)
         jobStore = Toil.resumeJobStore(config.jobStore)
-        stats = getStats(jobStore)
-        collatedStats = processData(jobStore.config, stats)
+        stats = get_stats(jobStore)
+        collatedStats = process_data(jobStore.config, stats)
         self.assertTrue(len(collatedStats.job_types) == 2, "Some jobs are not represented in the stats.")
 
     def check_status(self, status, status_fn, seconds=20):

toil/test/wdl/wdltoil_test.py

@@ -3,27 +3,31 @@ import os
 import shutil
 import subprocess
 import unittest
-import uuid
-from typing import Any, Dict, List, Optional, Set
-from unittest.mock import patch
+from uuid import uuid4
+from typing import Optional
 
 from unittest.mock import patch
 from typing import Any, Dict, List, Set
 
+import pytest
+
+from toil.provisioners import cluster_factory
 from toil.test import (ToilTest,
                        needs_docker_cuda,
                        needs_google_storage,
                        needs_singularity_or_docker,
-                       slow)
+                       slow, integrative)
+from toil.test.provisioners.clusterTest import AbstractClusterTest
 from toil.version import exactPython
 from toil.wdl.wdltoil import WDLSectionJob, WDLWorkflowGraph
 
 
 class BaseWDLTest(ToilTest):
     """Base test class for WDL tests."""
+
     def setUp(self) -> None:
         """Runs anew before each test to create farm fresh temp dirs."""
-        self.output_dir = os.path.join('/tmp/', 'toil-wdl-test-' + str(uuid.uuid4()))
+        self.output_dir = os.path.join('/tmp/', 'toil-wdl-test-' + str(uuid4()))
         os.makedirs(self.output_dir)
 
     def tearDown(self) -> None:
@@ -36,6 +40,7 @@ class WDLConformanceTests(BaseWDLTest):
     WDL conformance tests for Toil.
     """
     wdl_dir = "wdl-conformance-tests"
+
     @classmethod
     def setUpClass(cls) -> None:
 
@@ -87,6 +92,7 @@ class WDLConformanceTests(BaseWDLTest):
 
 class WDLTests(BaseWDLTest):
     """Tests for Toil's MiniWDL-based implementation."""
+
     @classmethod
     def setUpClass(cls) -> None:
         """Runs once for all tests."""
@@ -101,7 +107,8 @@ class WDLTests(BaseWDLTest):
         wdl = os.path.abspath('src/toil/test/wdl/md5sum/md5sum.1.0.wdl')
         json_file = os.path.abspath('src/toil/test/wdl/md5sum/md5sum.json')
 
-        result_json = subprocess.check_output(self.base_command + [wdl, json_file, '-o', self.output_dir, '--logDebug', '--retryCount=0'])
+        result_json = subprocess.check_output(
+            self.base_command + [wdl, json_file, '-o', self.output_dir, '--logDebug', '--retryCount=0'])
         result = json.loads(result_json)
 
         assert 'ga4ghMd5.value' in result
@@ -115,7 +122,9 @@ class WDLTests(BaseWDLTest):
         wdl_file = os.path.abspath('src/toil/test/wdl/miniwdl_self_test/self_test.wdl')
         json_file = os.path.abspath('src/toil/test/wdl/miniwdl_self_test/inputs.json')
 
-        result_json = subprocess.check_output(self.base_command + [wdl_file, json_file, '--logDebug', '-o', self.output_dir, '--outputDialect', 'miniwdl'] + (extra_args or []))
+        result_json = subprocess.check_output(
+            self.base_command + [wdl_file, json_file, '--logDebug', '-o', self.output_dir, '--outputDialect',
+                                 'miniwdl'] + (extra_args or []))
         result = json.loads(result_json)
 
         # Expect MiniWDL-style output with a designated "dir"
@@ -172,7 +181,8 @@ class WDLTests(BaseWDLTest):
                 "GiraffeDeepVariant.runDeepVariantCallVariants.in_dv_gpu_container": "google/deepvariant:1.3.0-gpu"
             })
 
-        result_json = subprocess.check_output(self.base_command + [wdl_file, json_file, '-o', self.output_dir, '--outputDialect', 'miniwdl'])
+        result_json = subprocess.check_output(
+            self.base_command + [wdl_file, json_file, '-o', self.output_dir, '--outputDialect', 'miniwdl'])
         result = json.loads(result_json)
 
         # Expect MiniWDL-style output with a designated "dir"
@@ -200,7 +210,9 @@ class WDLTests(BaseWDLTest):
         wdl_file = f"{base_uri}/workflows/giraffe.wdl"
         json_file = f"{base_uri}/params/giraffe.json"
 
-        result_json = subprocess.check_output(self.base_command + [wdl_file, json_file, '-o', self.output_dir, '--outputDialect', 'miniwdl', '--scale', '0.1'])
+        result_json = subprocess.check_output(
+            self.base_command + [wdl_file, json_file, '-o', self.output_dir, '--outputDialect', 'miniwdl', '--scale',
+                                 '0.1'])
         result = json.loads(result_json)
 
         # Expect MiniWDL-style output with a designated "dir"
@@ -265,7 +277,6 @@ class WDLTests(BaseWDLTest):
         # worth extracting a base type for this interface.
         with patch.object(WDLWorkflowGraph, 'is_decl', mock_is_decl):
             with patch.object(WDLWorkflowGraph, 'get_transitive_dependencies', mock_get_transitive_dependencies):
-
                 with self.subTest(msg="Two unrelated decls can coalesce"):
                     # Set up two unrelated decls
                     all_decls = {"decl1", "decl2"}
@@ -275,7 +286,7 @@ class WDLTests(BaseWDLTest):
                     }
 
                     result = WDLSectionJob.coalesce_nodes(["decl1", "decl2"], WDLWorkflowGraph([]))
-                    
+
                     # Make sure they coalesced
                     assert len(result) == 1
                     assert "decl1" in result[0]
@@ -289,12 +300,11 @@ class WDLTests(BaseWDLTest):
                     }
 
                     result = WDLSectionJob.coalesce_nodes(["decl", "nondecl"], WDLWorkflowGraph([]))
-                    
+
                     assert len(result) == 2
                     assert len(result[0]) == 1
                     assert len(result[1]) == 1
 
-
                 with self.subTest(msg="Two adjacent decls with a common dependency can coalesce"):
                     all_decls = {"decl1", "decl2"}
                     all_deps = {
@@ -304,7 +314,7 @@ class WDLTests(BaseWDLTest):
                     }
 
                     result = WDLSectionJob.coalesce_nodes(["base", "decl1", "decl2"], WDLWorkflowGraph([]))
-                    
+
                     assert len(result) == 2
                     assert "base" in result[0]
                     assert "decl1" in result[1]
@@ -319,7 +329,7 @@ class WDLTests(BaseWDLTest):
                     }
 
                     result = WDLSectionJob.coalesce_nodes(["base", "decl1", "decl2"], WDLWorkflowGraph([]))
-                    
+
                     assert len(result) == 3
                     assert "base" in result[0]
 
@@ -332,12 +342,77 @@ class WDLTests(BaseWDLTest):
                     }
 
                     result = WDLSectionJob.coalesce_nodes(["decl1", "decl2", "successor"], WDLWorkflowGraph([]))
-                    
+
                     assert len(result) == 2
                     assert "decl1" in result[0]
                     assert "decl2" in result[0]
                     assert "successor" in result[1]
 
 
+@integrative
+@slow
+@pytest.mark.timeout(600)
+class WDLKubernetesClusterTest(AbstractClusterTest):
+    """
+    Ensure WDL works on the Kubernetes batchsystem.
+    """
+
+    def __init__(self, name):
+        super().__init__(name)
+        self.clusterName = 'wdl-integration-test-' + str(uuid4())
+        # t2.medium is the minimum t2 instance that permits Kubernetes
+        self.leaderNodeType = "t2.medium"
+        self.instanceTypes = ["t2.medium"]
+        self.clusterType = "kubernetes"
+
+    def setUp(self) -> None:
+        super().setUp()
+        self.jobStore = f'aws:{self.awsRegion()}:wdl-test-{uuid4()}'
+
+    def launchCluster(self) -> None:
+        self.createClusterUtil(args=['--leaderStorage', str(self.requestedLeaderStorage),
+                                     '--nodeTypes', ",".join(self.instanceTypes),
+                                     '-w', ",".join(self.numWorkers),
+                                     '--nodeStorage', str(self.requestedLeaderStorage)])
+
+    def test_wdl_kubernetes_cluster(self):
+        """
+        Test that a wdl workflow works on a kubernetes cluster. Launches a cluster with 1 worker. This runs a wdl
+        workflow that performs an image pull on the worker.
+        :return:
+        """
+        self.numWorkers = "1"
+        self.requestedLeaderStorage = 30
+        # create the cluster
+        self.launchCluster()
+        # get leader
+        self.cluster = cluster_factory(
+            provisioner="aws", zone=self.zone, clusterName=self.clusterName
+        )
+        self.leader = self.cluster.getLeader()
+
+        url = "https://github.com/DataBiosphere/wdl-conformance-tests.git"
+        commit = "09b9659cd01473e836738a2e0dd205df0adb49c5"
+        wdl_dir = "wdl_conformance_tests"
+
+        # get the wdl-conformance-tests repo to get WDL tasks to run
+        self.sshUtil([
+            "bash",
+            "-c",
+            f"git clone {url} {wdl_dir} && cd {wdl_dir} && git checkout {commit}"
+        ])
+
+        # run on kubernetes batchsystem
+        toil_options = ['--batchSystem=kubernetes',
+                        f"--jobstore={self.jobStore}"]
+
+        # run WDL workflow that will run singularity
+        test_options = [f"tests/md5sum/md5sum.wdl", f"tests/md5sum/md5sum.json"]
+        self.sshUtil([
+            "bash",
+            "-c",
+            f"cd {wdl_dir} && toil-wdl-runner {' '.join(test_options)} {' '.join(toil_options)}"])
+
+
 if __name__ == "__main__":
     unittest.main()  # run all tests

toil/utils/toilDebugFile.py

@@ -17,11 +17,11 @@ import logging
 import os.path
 import sys
 from typing import Optional
-from distutils.util import strtobool
 
 from toil.common import Config, Toil, parser_with_common_options
 from toil.jobStores.fileJobStore import FileJobStore
 from toil.lib.resources import glob
+from toil.lib.conversions import strtobool
 from toil.statsAndLogging import set_logging_from_options
 
 logger = logging.getLogger(__name__)