toil 7.0.0__py3-none-any.whl → 8.1.0b1__py3-none-any.whl

toil/test/lib/test_misc.py

@@ -32,6 +32,7 @@ class UserNameAvailableTest(ToilTest):
         apparent_user_name = get_user_name()
         self.assertEqual(apparent_user_name, real_user_name)
 
+
 class UserNameUnvailableTest(ToilTest):
     """
     Make sure we can get something for a user name when user names are not
@@ -42,9 +43,12 @@ class UserNameUnvailableTest(ToilTest):
         super().setUp()
         # Monkey patch getpass.getuser to fail
         self.original_getuser = getpass.getuser
+
         def fake_getuser():
-            raise KeyError('Fake key error')
+            raise KeyError("Fake key error")
+
         getpass.getuser = fake_getuser
+
     def tearDown(self):
         # Fix the module we hacked up
         getpass.getuser = self.original_getuser
@@ -54,7 +58,8 @@ class UserNameUnvailableTest(ToilTest):
         apparent_user_name = get_user_name()
         # Make sure we got something
         self.assertTrue(isinstance(apparent_user_name, str))
-        self.assertNotEqual(apparent_user_name, '')
+        self.assertNotEqual(apparent_user_name, "")
+
 
 class UserNameVeryBrokenTest(ToilTest):
     """
@@ -66,9 +71,12 @@ class UserNameVeryBrokenTest(ToilTest):
         super().setUp()
         # Monkey patch getpass.getuser to fail
         self.original_getuser = getpass.getuser
+
         def fake_getuser():
-            raise RuntimeError('Fake error that we did not anticipate')
+            raise RuntimeError("Fake error that we did not anticipate")
+
         getpass.getuser = fake_getuser
+
     def tearDown(self):
         # Fix the module we hacked up
         getpass.getuser = self.original_getuser
@@ -78,5 +86,4 @@ class UserNameVeryBrokenTest(ToilTest):
         apparent_user_name = get_user_name()
         # Make sure we got something
         self.assertTrue(isinstance(apparent_user_name, str))
-        self.assertNotEqual(apparent_user_name, '')
-
+        self.assertNotEqual(apparent_user_name, "")

toil/test/lib/test_trs.py

@@ -0,0 +1,161 @@
+# Copyright (C) 2015-2024 Regents of the University of California
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import io
+import logging
+import pytest
+from typing import IO
+import urllib.request
+from urllib.error import URLError
+
+from toil.lib.retry import retry
+from toil.lib.trs import find_workflow, fetch_workflow
+from toil.test import ToilTest, needs_online
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.DEBUG)
+
+@pytest.mark.integrative
+@needs_online
+class DockstoreLookupTest(ToilTest):
+    """
+    Make sure we can look up workflows on Dockstore.
+    """
+
+    @retry(errors=[URLError, RuntimeError])
+    def read_result(self, url_or_path: str) -> IO[bytes]:
+        """
+        Read a file or URL.
+
+        Binary mode to allow testing for binary file support.
+
+        This lets us test that we have the right workflow contents and not care
+        how we are being shown them.
+        """
+        if url_or_path.startswith("http://") or url_or_path.startswith("https://"):
+            response = urllib.request.urlopen(url_or_path)
+            if response.status != 200:
+                raise RuntimeError(f"HTTP error response: {response}")
+            return response
+        else:
+            return open(url_or_path, "rb")
+
+    # TODO: Tests that definitely test a clear cache
+
+    def test_lookup_from_page_url(self) -> None:
+        PAGE_URL = "https://dockstore.org/workflows/github.com/dockstore/bcc2020-training/HelloWorld:master?tab=info"
+        trs_id, trs_version, language = find_workflow(PAGE_URL)
+
+        self.assertEqual(trs_id, "#workflow/github.com/dockstore/bcc2020-training/HelloWorld")
+        self.assertEqual(trs_version, "master")
+        self.assertEqual(language, "WDL")
+
+    def test_lookup_from_trs_with_version(self) -> None:
+        TRS_ID = "#workflow/github.com/dockstore-testing/md5sum-checker"
+        TRS_VERSION = "master"
+        trs_id, trs_version, language = find_workflow(f"{TRS_ID}:{TRS_VERSION}")
+        
+        self.assertEqual(trs_id, TRS_ID)
+        self.assertEqual(trs_version, TRS_VERSION)
+        self.assertEqual(language, "CWL")
+
+    def test_lookup_from_trs_no_version(self) -> None:
+        TRS_ID = "#workflow/github.com/dockstore-testing/md5sum-checker"
+        with pytest.raises(ValueError):
+            # We don't yet have a way to read Dockstore's default version info,
+            # so it's not safe to apply any default version when multiple
+            # versions exist.
+            trs_id, trs_version, language = find_workflow(TRS_ID)
+
+    # TODO: Add a test with a workflow that we know has and will only ever
+    # have one version, to test version auto-detection in that case.
+
+    def test_get(self) -> None:
+        TRS_ID = "#workflow/github.com/dockstore-testing/md5sum-checker"
+        TRS_VERSION = "master"
+        LANGUAGE = "CWL"
+        # Despite "-checker" in the ID, this actually refers to the base md5sum
+        # workflow that just happens to have a checker *available*, not to the
+        # checker workflow itself.
+        WORKFLOW_URL = "https://raw.githubusercontent.com/dockstore-testing/md5sum-checker/master/md5sum/md5sum-workflow.cwl"
+        looked_up = fetch_workflow(TRS_ID, TRS_VERSION, LANGUAGE)
+        
+        data_from_lookup = self.read_result(looked_up).read()
+        data_from_source = self.read_result(WORKFLOW_URL).read()
+        self.assertEqual(data_from_lookup, data_from_source)
+
+    def test_get_from_trs_cached(self) -> None:
+        TRS_ID = "#workflow/github.com/dockstore-testing/md5sum-checker"
+        TRS_VERSION = "master"
+        LANGUAGE = "CWL"
+        WORKFLOW_URL = "https://raw.githubusercontent.com/dockstore-testing/md5sum-checker/master/md5sum/md5sum-workflow.cwl"
+        # This lookup may or may not be cached
+        fetch_workflow(TRS_ID, TRS_VERSION, LANGUAGE)
+        # This lookup is definitely cached
+        looked_up = fetch_workflow(TRS_ID, TRS_VERSION, LANGUAGE)
+        
+        data_from_lookup = self.read_result(looked_up).read()
+        data_from_source = self.read_result(WORKFLOW_URL).read()
+        self.assertEqual(data_from_lookup, data_from_source)
+
+    def test_lookup_from_trs_with_version(self) -> None:
+        TRS_VERSIONED_ID = "#workflow/github.com/dockstore-testing/md5sum-checker:workflowWithHTTPImport"
+        trs_id, trs_version, language = find_workflow(TRS_VERSIONED_ID)
+        
+        parts = TRS_VERSIONED_ID.split(":")
+
+        self.assertEqual(trs_id, parts[0])
+        self.assertEqual(trs_version, parts[1])
+        self.assertEqual(language, "CWL")
+
+    def test_lookup_from_trs_nonexistent_workflow(self) -> None:
+        TRS_VERSIONED_ID = "#workflow/github.com/adamnovak/veryfakerepo:notARealVersion"
+        with self.assertRaises(FileNotFoundError):
+            looked_up = find_workflow(TRS_VERSIONED_ID)
+
+    def test_lookup_from_trs_nonexistent_workflow_bad_format(self) -> None:
+        TRS_VERSIONED_ID = "#workflow/AbsoluteGarbage:notARealVersion"
+        with self.assertRaises(FileNotFoundError):
+            looked_up = find_workflow(TRS_VERSIONED_ID)
+
+    def test_lookup_from_trs_nonexistent_version(self) -> None:
+        TRS_VERSIONED_ID = "#workflow/github.com/dockstore-testing/md5sum-checker:notARealVersion"
+        with self.assertRaises(FileNotFoundError):
+            looked_up = find_workflow(TRS_VERSIONED_ID)
+    
+    def test_get_nonexistent_workflow(self) -> None:
+        TRS_ID = "#workflow/github.com/adamnovak/veryfakerepo"
+        TRS_VERSION = "notARealVersion"
+        LANGUAGE = "CWL"
+        with self.assertRaises(FileNotFoundError):
+            looked_up = fetch_workflow(TRS_ID, TRS_VERSION, LANGUAGE)
+
+    def test_get_nonexistent_version(self) -> None:
+        TRS_ID = "#workflow/github.com/dockstore-testing/md5sum-checker"
+        TRS_VERSION = "notARealVersion"
+        LANGUAGE = "CWL"
+        with self.assertRaises(FileNotFoundError):
+            looked_up = fetch_workflow(TRS_ID, TRS_VERSION, LANGUAGE)
+
+    def test_get_nonexistent_workflow_bad_format(self) -> None:
+        # Dockstore enforces an ID pattern and blames your request if you ask
+        # about something that doesn't follow it. So don't follow it.
+        TRS_ID = "#workflow/AbsoluteGarbage"
+        TRS_VERSION = "notARealVersion"
+        LANGUAGE = "CWL"
+        with self.assertRaises(FileNotFoundError):
+            looked_up = fetch_workflow(TRS_ID, TRS_VERSION, LANGUAGE)
+
+
+

toil/test/mesos/MesosDataStructuresTest.py

@@ -21,15 +21,19 @@ class DataStructuresTest(ToilTest):
     def _getJob(self, cores=1, memory=1000, disk=5000, preemptible=True):
         from toil.batchSystems.mesos import MesosShape, ToilJob
 
-        resources = MesosShape(wallTime=0, cores=cores, memory=memory, disk=disk, preemptible=preemptible)
+        resources = MesosShape(
+            wallTime=0, cores=cores, memory=memory, disk=disk, preemptible=preemptible
+        )
 
-        job = ToilJob(jobID=str(uuid.uuid4()),
-                      name=str(uuid.uuid4()),
-                      resources=resources,
-                      command="do nothing",
-                      userScript=None,
-                      environment=None,
-                      workerCleanupInfo=None)
+        job = ToilJob(
+            jobID=str(uuid.uuid4()),
+            name=str(uuid.uuid4()),
+            resources=resources,
+            command="do nothing",
+            userScript=None,
+            environment=None,
+            workerCleanupInfo=None,
+        )
         return job
 
     def testJobQueue(self, testJobs=1000):
@@ -39,15 +43,24 @@ class DataStructuresTest(ToilTest):
         non-preemptible jobs groups first, with priority given to large jobs.
         """
         from toil.batchSystems.mesos import JobQueue
+
         jobQueue = JobQueue()
 
         for jobNum in range(0, testJobs):
-            testJob = self._getJob(cores=random.choice(list(range(10))), preemptible=random.choice([True, False]))
+            testJob = self._getJob(
+                cores=random.choice(list(range(10))),
+                preemptible=random.choice([True, False]),
+            )
             jobQueue.insertJob(testJob, testJob.resources)
 
         sortedTypes = jobQueue.sortedTypes
         self.assertGreaterEqual(20, len(sortedTypes))
-        self.assertTrue(all(sortedTypes[i] <= sortedTypes[i + 1] for i in range(len(sortedTypes) - 1)))
+        self.assertTrue(
+            all(
+                sortedTypes[i] <= sortedTypes[i + 1]
+                for i in range(len(sortedTypes) - 1)
+            )
+        )
 
         preemptible = sortedTypes.pop(0).preemptible
         for jtype in sortedTypes:

toil/test/mesos/helloWorld.py

@@ -23,14 +23,15 @@ from toil.job import Job
 childMessage = "The child job is now running!"
 parentMessage = "The parent job is now running!"
 
+
 def hello_world(job):
 
     job.fileStore.log_to_leader(parentMessage)
-    with open('foo_bam.txt', 'w') as handle:
-        handle.write('\nThis is a triumph...\n')
+    with open("foo_bam.txt", "w") as handle:
+        handle.write("\nThis is a triumph...\n")
 
     # Assign FileStoreID to a given file
-    foo_bam = job.fileStore.writeGlobalFile('foo_bam.txt')
+    foo_bam = job.fileStore.writeGlobalFile("foo_bam.txt")
 
     # Spawn child
     job.addChildJobFn(hello_world_child, foo_bam, memory=100, cores=0.5, disk="3G")
@@ -43,13 +44,13 @@ def hello_world_child(job, hw):
     # NOTE: path and the udpated file are stored to /tmp
     # If we want to SAVE our changes to this tmp file, we must write it out.
     with open(path) as r:
-        with open('bar_bam.txt', 'w') as handle:
+        with open("bar_bam.txt", "w") as handle:
             for line in r.readlines():
                 handle.write(line)
 
     # Assign FileStoreID to a given file
     # can also use:  job.updateGlobalFile() given the FileStoreID instantiation.
-    job.fileStore.writeGlobalFile('bar_bam.txt')
+    job.fileStore.writeGlobalFile("bar_bam.txt")
 
 
 def main():
@@ -67,5 +68,5 @@ def main():
         toil.start(i)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

toil/test/mesos/stress.py

@@ -17,58 +17,63 @@ from configargparse import ArgumentParser
 from toil.job import Job
 
 
-def touchFile( fileStore ):
+def touchFile(fileStore):
     with fileStore.writeGlobalFileStream() as (f, id):
-        f.write( "This is a triumph" )
+        f.write("This is a triumph")
+
 
 class LongTestJob(Job):
     def __init__(self, numJobs):
-        Job.__init__(self,  memory=100000, cores=0.01)
+        Job.__init__(self, memory=100000, cores=0.01)
         self.numJobs = numJobs
 
     def run(self, fileStore):
-        for i in range(0,self.numJobs):
+        for i in range(0, self.numJobs):
             self.addChild(HelloWorldJob(i))
         self.addFollowOn(LongTestFollowOn())
 
+
 class LongTestFollowOn(Job):
 
     def __init__(self):
-        Job.__init__(self,  memory=1000000, cores=0.01)
+        Job.__init__(self, memory=1000000, cores=0.01)
 
     def run(self, fileStore):
-        touchFile( fileStore )
+        touchFile(fileStore)
 
-class HelloWorldJob(Job):
 
-    def __init__(self,i):
-        Job.__init__(self,  memory=100000, cores=0.01)
-        self.i=i
+class HelloWorldJob(Job):
 
+    def __init__(self, i):
+        Job.__init__(self, memory=100000, cores=0.01)
+        self.i = i
 
     def run(self, fileStore):
-        touchFile( fileStore )
+        touchFile(fileStore)
         self.addFollowOn(HelloWorldFollowOn(self.i))
 
+
 class HelloWorldFollowOn(Job):
 
-    def __init__(self,i):
-        Job.__init__(self,  memory=200000, cores=0.01)
+    def __init__(self, i):
+        Job.__init__(self, memory=200000, cores=0.01)
         self.i = i
 
     def run(self, fileStore):
-        touchFile( fileStore)
+        touchFile(fileStore)
+
 
 def main(numJobs):
     # Boilerplate -- startToil requires options
     parser = ArgumentParser()
     Job.Runner.addToilOptions(parser)
-    options = parser.parse_args( args=['./toilTest'] )
-    options.batchSystem="mesos"
-    options.mesos_endpoint="localhost:5050"
+    options = parser.parse_args(args=["./toilTest"])
+    options.batchSystem = "mesos"
+    options.mesos_endpoint = "localhost:5050"
     # Launch first toil Job
-    i = LongTestJob( numJobs )
-    Job.Runner.startToil(i,  options )
+    i = LongTestJob(numJobs)
+    Job.Runner.startToil(i, options)
+
 
-if __name__=="__main__":
+if __name__ == "__main__":
     main(numJobs=5)

toil/test/options/options.py

@@ -1,6 +1,6 @@
 from configargparse import ArgParser
 
-from toil.common import addOptions, Toil
+from toil.common import Toil, addOptions
 from toil.test import ToilTest
 
 
@@ -8,6 +8,7 @@ class OptionsTest(ToilTest):
     """
     Class to test functionality of all Toil options
     """
+
     def test_default_caching_slurm(self):
         """
         Test to ensure that caching will be set to false when running on Slurm
@@ -30,7 +31,11 @@ class OptionsTest(ToilTest):
         addOptions(parser, jobstore_as_flag=True, wdl=False, cwl=False)
         # the kubernetes batchsystem (and I think all batchsystems including singlemachine) return False
         # for default_caching
-        test_args = ["--jobstore=example-jobstore", "--batchSystem=kubernetes", "--caching=True"]
+        test_args = [
+            "--jobstore=example-jobstore",
+            "--batchSystem=kubernetes",
+            "--caching=True",
+        ]
         options = parser.parse_args(test_args)
         with Toil(options) as toil:
             caching_value = toil.config.caching