toil 8.1.0b1__py3-none-any.whl → 9.0.0__py3-none-any.whl

toil/test/docs/scripts/tutorial_debugging_works.wdl

@@ -0,0 +1,129 @@
+version 1.1
+
+workflow TutorialDebugging {
+
+    input {
+        Array[String] messages = ["Uh-oh!", "Oh dear", "Oops"]
+    }
+
+    scatter(message in messages) {
+
+        call WhaleSay {
+            input:
+                message = message
+        }
+
+        call CountLines {
+            input:
+                to_count = WhaleSay.result
+        }
+    }
+
+    Array[File] to_compress = flatten([CountLines.result, WhaleSay.result])
+
+    call CompressFiles {
+        input:
+            files = to_compress
+    }
+
+    output {
+        File compressed = CompressFiles.result
+    }
+
+}
+
+# Draw ASCII art
+task WhaleSay {
+    input {
+        String message
+    }
+
+    command <<<
+        cowsay "~{message}"
+    >>>
+
+    output {
+        File result = stdout()
+    }
+
+    runtime {
+        container: "docker/whalesay"
+    }
+}
+
+# Count the lines in a file
+task CountLines {
+    input {
+        File to_count
+    }
+
+    command <<<
+        wc -l ~{to_count}
+    >>>
+
+    output {
+        File result = stdout()
+    }
+
+    runtime {
+        container: ["ubuntu:latest", "https://gcr.io/standard-images/ubuntu:latest"]
+    }
+}
+
+# Compress files into a ZIP
+task CompressFiles {
+    input {
+        Array[File] files
+    }
+
+    command <<<
+    set -e
+    cat >script.py <<'EOF'
+    import sys
+    from zipfile import ZipFile
+    import os
+
+    # Interpret command line arguments
+    to_compress = list(reversed(sys.argv[1:]))
+
+    with ZipFile("compressed.zip", "w") as z:
+        while to_compress != []:
+            # Grab the file to add off the end of the list
+            input_filename = to_compress[-1]
+            # Now we need to write this to the zip file.
+            # What internal filename should we use?
+            basename = os.path.basename(input_filename)
+            disambiguation_number = 0
+            while True:
+                target_filename = str(disambiguation_number) + basename
+                try:
+                    z.getinfo(target_filename)
+                except KeyError:
+                    # Filename is free
+                    break
+                # Otherwise try another name
+                disambiguation_number += 1
+            # Now we can actually make the compressed file
+            with z.open(target_filename, 'w') as out_stream:
+                with open(input_filename) as in_stream:
+                    for line in in_stream:
+                        # Prefix each line of text with the original input file
+                        # it came from.
+                        # Also remember to encode the text as the zip file
+                        # stream is in binary mode.
+                        out_stream.write(f"{basename}: {line}".encode("utf-8"))
+            # Even though we got distracted by zip file manipulation, remember
+            # to pop off the file we just did.
+            to_compress.pop()
+    EOF
+    python script.py ~{sep(" ", files)}
+    >>>
+
+    output {
+        File result = "compressed.zip"
+    }
+
+    runtime {
+        container: "python:3.11"
+    }
+}

toil/test/docs/scripts/tutorial_docker.py

@@ -0,0 +1,20 @@
+import os
+
+from toil.common import Toil
+from toil.job import Job
+from toil.lib.docker import apiDockerCall
+from toil.lib.io import mkdtemp
+
+align = Job.wrapJobFn(
+    apiDockerCall, image="ubuntu", working_dir=os.getcwd(), parameters=["ls", "-lha"]
+)
+
+if __name__ == "__main__":
+    jobstore: str = mkdtemp("tutorial_docker")
+    os.rmdir(jobstore)
+    options = Job.Runner.getDefaultOptions(jobstore)
+    options.logLevel = "INFO"
+    options.clean = "always"
+
+    with Toil(options) as toil:
+        toil.start(align)

toil/test/docs/scripts/tutorial_dynamic.py

@@ -0,0 +1,24 @@
+import os
+
+from toil.common import Toil
+from toil.job import Job
+from toil.lib.io import mkdtemp
+
+
+def binaryStringFn(job, depth, message=""):
+    if depth > 0:
+        job.addChildJobFn(binaryStringFn, depth - 1, message + "0")
+        job.addChildJobFn(binaryStringFn, depth - 1, message + "1")
+    else:
+        job.log(f"Binary string: {message}")
+
+
+if __name__ == "__main__":
+    jobstore: str = mkdtemp("tutorial_dynamic")
+    os.rmdir(jobstore)
+    options = Job.Runner.getDefaultOptions(jobstore)
+    options.logLevel = "INFO"
+    options.clean = "always"
+
+    with Toil(options) as toil:
+        toil.start(Job.wrapJobFn(binaryStringFn, depth=5))

toil/test/docs/scripts/tutorial_encapsulation.py

@@ -0,0 +1,28 @@
+import os
+
+from toil.common import Toil
+from toil.job import Job
+from toil.lib.io import mkdtemp
+
+if __name__ == "__main__":
+    # A is a job with children and follow-ons, for example:
+    A = Job()
+    A.addChild(Job())
+    A.addFollowOn(Job())
+
+    # B is a job which needs to run after A and its successors
+    B = Job()
+
+    # The way to do this without encapsulation is to make a parent of A, Ap, and make B a follow-on of Ap.
+    Ap = Job()
+    Ap.addChild(A)
+    Ap.addFollowOn(B)
+
+    jobstore: str = mkdtemp("tutorial_encapsulations")
+    os.rmdir(jobstore)
+    options = Job.Runner.getDefaultOptions(jobstore)
+    options.logLevel = "INFO"
+    options.clean = "always"
+
+    with Toil(options) as toil:
+        print(toil.start(Ap))

toil/test/docs/scripts/tutorial_encapsulation2.py

@@ -0,0 +1,29 @@
+import os
+
+from toil.common import Toil
+from toil.job import Job
+from toil.lib.io import mkdtemp
+
+if __name__ == "__main__":
+    # A
+    A = Job()
+    A.addChild(Job())
+    A.addFollowOn(Job())
+
+    # Encapsulate A
+    A = A.encapsulate()
+
+    # B is a job which needs to run after A and its successors
+    B = Job()
+
+    # With encapsulation A and its successor subgraph appear to be a single job, hence:
+    A.addChild(B)
+
+    jobstore: str = mkdtemp("tutorial_encapsulations2")
+    os.rmdir(jobstore)
+    options = Job.Runner.getDefaultOptions(jobstore)
+    options.logLevel = "INFO"
+    options.clean = "always"
+
+    with Toil(options) as toil:
+        print(toil.start(A))

toil/test/docs/scripts/tutorial_helloworld.py

@@ -0,0 +1,15 @@
+from toil.common import Toil
+from toil.job import Job
+
+
+def helloWorld(message, memory="1G", cores=1, disk="1G"):
+    return f"Hello, world!, here's a message: {message}"
+
+
+if __name__ == "__main__":
+    parser = Job.Runner.getDefaultArgumentParser()
+    options = parser.parse_args()
+    options.clean = "always"
+    with Toil(options) as toil:
+        output = toil.start(Job.wrapFn(helloWorld, "You did it!"))
+    print(output)

toil/test/docs/scripts/tutorial_invokeworkflow.py

@@ -0,0 +1,27 @@
+import os
+
+from toil.common import Toil
+from toil.job import Job
+from toil.lib.io import mkdtemp
+
+
+class HelloWorld(Job):
+    def __init__(self, message):
+        Job.__init__(self)
+        self.message = message
+
+    def run(self, fileStore):
+        return f"Hello, world!, here's a message: {self.message}"
+
+
+if __name__ == "__main__":
+    jobstore: str = mkdtemp("tutorial_invokeworkflow")
+    os.rmdir(jobstore)
+    options = Job.Runner.getDefaultOptions(jobstore)
+    options.logLevel = "OFF"
+    options.clean = "always"
+
+    hello_job = HelloWorld("Woot")
+
+    with Toil(options) as toil:
+        print(toil.start(hello_job))

toil/test/docs/scripts/tutorial_invokeworkflow2.py

@@ -0,0 +1,30 @@
+import os
+
+from toil.common import Toil
+from toil.job import Job
+from toil.lib.io import mkdtemp
+
+
+class HelloWorld(Job):
+    def __init__(self, message):
+        Job.__init__(self)
+        self.message = message
+
+    def run(self, fileStore):
+        return f"Hello, world!, I have a message: {self.message}"
+
+
+if __name__ == "__main__":
+    jobstore: str = mkdtemp("tutorial_invokeworkflow2")
+    os.rmdir(jobstore)
+    options = Job.Runner.getDefaultOptions(jobstore)
+    options.logLevel = "INFO"
+    options.clean = "always"
+
+    with Toil(options) as toil:
+        if not toil.options.restart:
+            job = HelloWorld("Woot!")
+            output = toil.start(job)
+        else:
+            output = toil.restart()
+    print(output)

toil/test/docs/scripts/tutorial_jobfunctions.py

@@ -0,0 +1,22 @@
+import os
+
+from toil.common import Toil
+from toil.job import Job
+from toil.lib.io import mkdtemp
+
+
+def helloWorld(job, message):
+    job.log(f"Hello world, I have a message: {message}")
+
+
+if __name__ == "__main__":
+    jobstore: str = mkdtemp("tutorial_jobfunctions")
+    os.rmdir(jobstore)
+    options = Job.Runner.getDefaultOptions(jobstore)
+    options.logLevel = "INFO"
+    options.clean = "always"
+
+    hello_job = Job.wrapJobFn(helloWorld, "Woot!")
+
+    with Toil(options) as toil:
+        toil.start(hello_job)

toil/test/docs/scripts/tutorial_managing.py

@@ -0,0 +1,29 @@
+import os
+
+from toil.common import Toil
+from toil.job import Job
+from toil.lib.io import mkdtemp
+
+
+class LocalFileStoreJob(Job):
+    def run(self, fileStore):
+        # self.tempDir will always contain the name of a directory within the allocated disk space reserved for the job
+        scratchDir = self.tempDir
+
+        # Similarly create a temporary file.
+        scratchFile = fileStore.getLocalTempFile()
+
+
+if __name__ == "__main__":
+    jobstore: str = mkdtemp("tutorial_managing")
+    os.rmdir(jobstore)
+    options = Job.Runner.getDefaultOptions(jobstore)
+    options.logLevel = "INFO"
+    options.clean = "always"
+
+    # Create an instance of FooJob which will have at least 2 gigabytes of storage space.
+    j = LocalFileStoreJob(disk="2G")
+
+    # Run the workflow
+    with Toil(options) as toil:
+        toil.start(j)

toil/test/docs/scripts/tutorial_managing2.py

@@ -0,0 +1,56 @@
+import os
+
+from toil.common import Toil
+from toil.job import Job
+from toil.lib.io import mkdtemp
+
+
+def globalFileStoreJobFn(job):
+    job.log(
+        "The following example exercises all the methods provided "
+        "by the toil.fileStores.abstractFileStore.AbstractFileStore class"
+    )
+
+    # Create a local temporary file.
+    scratchFile = job.fileStore.getLocalTempFile()
+
+    # Write something in the scratch file.
+    with open(scratchFile, "w") as fH:
+        fH.write("What a tangled web we weave")
+
+    # Write a copy of the file into the file-store; fileID is the key that can be used to retrieve the file.
+    # This write is asynchronous by default
+    fileID = job.fileStore.writeGlobalFile(scratchFile)
+
+    # Write another file using a stream; fileID2 is the
+    # key for this second file.
+    with job.fileStore.writeGlobalFileStream(cleanup=True) as (fH, fileID2):
+        fH.write(b"Out brief candle")
+
+    # Now read the first file; scratchFile2 is a local copy of the file that is read-only by default.
+    scratchFile2 = job.fileStore.readGlobalFile(fileID)
+
+    # Read the second file to a desired location: scratchFile3.
+    scratchFile3 = os.path.join(job.tempDir, "foo.txt")
+    job.fileStore.readGlobalFile(fileID2, userPath=scratchFile3)
+
+    # Read the second file again using a stream.
+    with job.fileStore.readGlobalFileStream(fileID2) as fH:
+        print(fH.read())  # This prints "Out brief candle"
+
+    # Delete the first file from the global file-store.
+    job.fileStore.deleteGlobalFile(fileID)
+
+    # It is unnecessary to delete the file keyed by fileID2 because we used the cleanup flag,
+    # which removes the file after this job and all its successors have run (if the file still exists)
+
+
+if __name__ == "__main__":
+    jobstore: str = mkdtemp("tutorial_managing2")
+    os.rmdir(jobstore)
+    options = Job.Runner.getDefaultOptions(jobstore)
+    options.logLevel = "INFO"
+    options.clean = "always"
+
+    with Toil(options) as toil:
+        toil.start(Job.wrapJobFn(globalFileStoreJobFn))

toil/test/docs/scripts/tutorial_multiplejobs.py

@@ -0,0 +1,25 @@
+from toil.common import Toil
+from toil.job import Job
+
+
+def helloWorld(job, message):
+    job.log(f"Hello world, I have a message: {message}")
+
+
+if __name__ == "__main__":
+    parser = Job.Runner.getDefaultArgumentParser()
+    options = parser.parse_args()
+    options.logLevel = "INFO"
+    options.clean = "always"
+
+    j1 = Job.wrapJobFn(helloWorld, "first")
+    j2 = Job.wrapJobFn(helloWorld, "second or third")
+    j3 = Job.wrapJobFn(helloWorld, "second or third")
+    j4 = Job.wrapJobFn(helloWorld, "last")
+
+    j1.addChild(j2)
+    j1.addChild(j3)
+    j1.addFollowOn(j4)
+
+    with Toil(options) as toil:
+        toil.start(j1)

toil/test/docs/scripts/tutorial_multiplejobs2.py

@@ -0,0 +1,21 @@
+from toil.common import Toil
+from toil.job import Job
+
+
+def helloWorld(job, message):
+    job.log(f"Hello world, I have a message: {message}")
+
+
+if __name__ == "__main__":
+    parser = Job.Runner.getDefaultArgumentParser()
+    options = parser.parse_args()
+    options.logLevel = "INFO"
+    options.clean = "always"
+
+    j1 = Job.wrapJobFn(helloWorld, "first")
+    j2 = j1.addChildJobFn(helloWorld, "second or third")
+    j3 = j1.addChildJobFn(helloWorld, "second or third")
+    j4 = j1.addFollowOnJobFn(helloWorld, "last")
+
+    with Toil(options) as toil:
+        toil.start(j1)

toil/test/docs/scripts/tutorial_multiplejobs3.py

@@ -0,0 +1,22 @@
+from toil.common import Toil
+from toil.job import Job
+
+
+def helloWorld(job, message):
+    job.log(f"Hello world, I have a message: {message}")
+
+
+if __name__ == "__main__":
+    parser = Job.Runner.getDefaultArgumentParser()
+    options = parser.parse_args()
+    options.logLevel = "INFO"
+    options.clean = "always"
+
+    j1 = Job.wrapJobFn(helloWorld, "first")
+    j2 = j1.addChildJobFn(helloWorld, "second or third")
+    j3 = j1.addChildJobFn(helloWorld, "second or third")
+    j4 = j2.addChildJobFn(helloWorld, "last")
+    j3.addChild(j4)
+
+    with Toil(options) as toil:
+        toil.start(j1)

toil/test/docs/scripts/tutorial_promises.py

@@ -0,0 +1,25 @@
+import os
+
+from toil.common import Toil
+from toil.job import Job
+from toil.lib.io import mkdtemp
+
+
+def fn(job, i):
+    job.log("i is: %s" % i, level=100)
+    return i + 1
+
+
+if __name__ == "__main__":
+    jobstore: str = mkdtemp("tutorial_promises")
+    os.rmdir(jobstore)
+    options = Job.Runner.getDefaultOptions(jobstore)
+    options.logLevel = "INFO"
+    options.clean = "always"
+
+    j1 = Job.wrapJobFn(fn, 1)
+    j2 = j1.addChildJobFn(fn, j1.rv())
+    j3 = j1.addFollowOnJobFn(fn, j2.rv())
+
+    with Toil(options) as toil:
+        toil.start(j1)

toil/test/docs/scripts/tutorial_promises2.py

@@ -0,0 +1,30 @@
+import os
+
+from toil.common import Toil
+from toil.job import Job
+from toil.lib.io import mkdtemp
+
+
+def binaryStrings(job, depth, message=""):
+    if depth > 0:
+        s = [
+            job.addChildJobFn(binaryStrings, depth - 1, message + "0").rv(),
+            job.addChildJobFn(binaryStrings, depth - 1, message + "1").rv(),
+        ]
+        return job.addFollowOnFn(merge, s).rv()
+    return [message]
+
+
+def merge(strings):
+    return strings[0] + strings[1]
+
+
+if __name__ == "__main__":
+    jobstore: str = mkdtemp("tutorial_promises2")
+    os.rmdir(jobstore)
+    options = Job.Runner.getDefaultOptions(jobstore)
+    options.loglevel = "OFF"
+    options.clean = "always"
+
+    with Toil(options) as toil:
+        print(toil.start(Job.wrapJobFn(binaryStrings, depth=5)))

toil/test/docs/scripts/tutorial_quickstart.py

@@ -0,0 +1,22 @@
+import os
+
+from toil.common import Toil
+from toil.job import Job
+from toil.lib.io import mkdtemp
+
+
+def helloWorld(message):
+    return f"Hello, world!, here's a message: {message}"
+
+
+if __name__ == "__main__":
+    jobstore: str = mkdtemp("tutorial_quickstart")
+    os.rmdir(jobstore)
+    options = Job.Runner.getDefaultOptions(jobstore)
+    options.logLevel = "OFF"
+    options.clean = "always"
+
+    hello_job = Job.wrapFn(helloWorld, "Woot")
+
+    with Toil(options) as toil:
+        print(toil.start(hello_job))  # prints "Hello, world!, ..."

toil/test/docs/scripts/tutorial_requirements.py

@@ -0,0 +1,44 @@
+import os
+
+from toil.common import Toil
+from toil.job import Job, PromisedRequirement
+from toil.lib.io import mkdtemp
+
+
+def parentJob(job):
+    downloadJob = Job.wrapJobFn(
+        stageFn,
+        "file://" + os.path.realpath(__file__),
+        cores=0.1,
+        memory="32M",
+        disk="1M",
+    )
+    job.addChild(downloadJob)
+
+    analysis = Job.wrapJobFn(
+        analysisJob,
+        fileStoreID=downloadJob.rv(0),
+        disk=PromisedRequirement(downloadJob.rv(1)),
+    )
+    job.addFollowOn(analysis)
+
+
+def stageFn(job, url):
+    importedFile = job.fileStore.import_file(url)
+    return importedFile, importedFile.size
+
+
+def analysisJob(job, fileStoreID):
+    # now do some analysis on the file
+    pass
+
+
+if __name__ == "__main__":
+    jobstore: str = mkdtemp("tutorial_requirements")
+    os.rmdir(jobstore)
+    options = Job.Runner.getDefaultOptions(jobstore)
+    options.logLevel = "INFO"
+    options.clean = "always"
+
+    with Toil(options) as toil:
+        toil.start(Job.wrapJobFn(parentJob))

toil/test/docs/scripts/tutorial_services.py

@@ -0,0 +1,45 @@
+import os
+
+from toil.common import Toil
+from toil.job import Job
+from toil.lib.io import mkdtemp
+
+
+class DemoService(Job.Service):
+    def start(self, fileStore):
+        # Start up a database/service here
+        # Return a value that enables another process to connect to the database
+        return "loginCredentials"
+
+    def check(self):
+        # A function that if it returns False causes the service to quit
+        # If it raises an exception the service is killed and an error is reported
+        return True
+
+    def stop(self, fileStore):
+        # Cleanup the database here
+        pass
+
+
+j = Job()
+s = DemoService()
+loginCredentialsPromise = j.addService(s)
+
+
+def dbFn(loginCredentials):
+    # Use the login credentials returned from the service's start method to connect to the service
+    pass
+
+
+j.addChildFn(dbFn, loginCredentialsPromise)
+
+
+if __name__ == "__main__":
+    jobstore: str = mkdtemp("tutorial_services")
+    os.rmdir(jobstore)
+    options = Job.Runner.getDefaultOptions(jobstore)
+    options.logLevel = "INFO"
+    options.clean = "always"
+
+    with Toil(options) as toil:
+        toil.start(j)