toil 8.0.0__py3-none-any.whl → 8.2.0__py3-none-any.whl

toil/test/cwl/directory/directory/file.txt

@@ -0,0 +1,15 @@
+hello
+hello 
+not yellow
+not yellow
+not yellow
+not yellow
+not yellow
+hello
+not yellow
+not yellow
+not yellow
+hello
+not yellow
+not yellow
+not yellow

toil/test/cwl/download_directory_file.json

@@ -0,0 +1,4 @@
+{
+    "input_directory": {"class": "Directory", "location": "directory/directory"},
+    "input_filename": "file.txt"
+}

toil/test/cwl/download_directory_s3.json

@@ -0,0 +1,4 @@
+{
+    "input_directory": {"class": "Directory", "location": "s3://toil-datasets/images/logos/"},
+    "input_filename": "toil-logo-draft.png"
+}

toil/test/cwl/download_file.json

@@ -0,0 +1,6 @@
+{
+  "input": {
+    "class": "File",
+    "location": "directory/directory/file.txt"
+  }
+}

toil/test/cwl/download_http.json

@@ -0,0 +1,6 @@
+{
+  "input": {
+    "class": "File",
+    "location": "http://toil-datasets.s3.amazonaws.com/hello.txt"
+  }
+}

toil/test/cwl/download_https.json

@@ -0,0 +1,6 @@
+{
+  "input": {
+    "class": "File",
+    "location": "https://toil-datasets.s3.amazonaws.com/hello.txt"
+  }
+}

toil/test/cwl/download_s3.json

@@ -0,0 +1,6 @@
+{
+  "input": {
+    "class": "File",
+    "location": "s3://toil-datasets/hello.txt"
+  }
+}

toil/test/cwl/download_subdirectory_file.json

@@ -0,0 +1,5 @@
+{
+    "input_parent_directory": {"class": "Directory", "location": "directory"},
+    "input_subdirname": "directory",
+    "input_filename": "file.txt"
+}

toil/test/cwl/download_subdirectory_s3.json

@@ -0,0 +1,5 @@
+{
+    "input_parent_directory": {"class": "Directory", "location": "s3://toil-datasets/images/"},
+    "input_subdirname": "logos",
+    "input_filename": "toil-logo-draft.png"
+}

toil/test/cwl/empty.json

@@ -0,0 +1 @@
+{}

toil/test/cwl/mock_mpi/fake_mpi.yml

@@ -0,0 +1,8 @@
+runner: fake_mpi_run.py
+nproc_flag: --num
+extra_flags:
+- --no-fail
+env_set:
+  TEST_MPI_FOO: bar
+env_pass:
+- USER

toil/test/cwl/mock_mpi/fake_mpi_run.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+import argparse
+import subprocess
+import sys
+
+from configargparse import ArgumentParser
+
+
+def make_parser() -> ArgumentParser:
+    p = ArgumentParser()
+    p.add_argument(
+        "progargs", nargs=argparse.REMAINDER, help="The program and its arguments"
+    )
+    p.add_argument("--num", type=int, help="number of times to run the application")
+    p.add_argument(
+        "--no-fail", help="add this flag to actually work", action="store_true"
+    )
+    return p
+
+
+class Runner:
+    def __init__(self) -> None:
+        if sys.stdin.isatty():
+            self.indata = None
+        else:
+            self.indata = sys.stdin.read().encode(sys.stdin.encoding)
+
+    def run_once(self, args: list[str]) -> None:
+        subprocess.run(
+            args, input=self.indata, stdout=sys.stdout, stderr=sys.stderr
+        ).check_returncode()
+
+    def run_many(self, n: int, args: list[str]) -> None:
+        for i in range(n):
+            self.run_once(args)
+
+
+if __name__ == "__main__":
+    args = make_parser().parse_args()
+    assert args.no_fail is True, "Didn't set the --no-fail flag"
+    r = Runner()
+    r.run_many(args.num, args.progargs)

toil/test/cwl/optional-file-exists.json

@@ -0,0 +1,6 @@
+{
+  "inputFile": {
+    "class": "File",
+    "path": "s3://toil-datasets/wdl_templates.zip"
+  }
+}

toil/test/cwl/optional-file-missing.json

@@ -0,0 +1,6 @@
+{
+  "inputFile": {
+    "class": "File",
+    "path": "s3://toil-datasets/hello.txt"
+  }
+}

toil/test/cwl/optional-file.cwl

@@ -0,0 +1,18 @@
+#!/usr/bin/env cwl-runner
+class: Workflow
+cwlVersion: v1.2
+
+inputs:
+    inputFile:
+      type: File
+      secondaryFiles:
+        - ^_old.zip?
+
+outputs:
+  out_file:
+    type: File
+    outputSource:
+      - inputFile
+
+steps:
+  []

toil/test/cwl/preemptible_expression.json

@@ -0,0 +1 @@
+{"preemptible": true}

toil/test/cwl/revsort-job-missing.json

@@ -0,0 +1,6 @@
+{
+  "input": {
+    "class": "File",
+    "path": "missing.txt"
+  }
+}

toil/test/cwl/revsort-job.json

@@ -0,0 +1,6 @@
+{
+  "input": {
+    "class": "File",
+    "path": "whale.txt"
+  }
+}

toil/test/cwl/s3_secondary_file.json

@@ -0,0 +1,16 @@
+{
+  "ref": {
+        "class": "File",
+        "location": "whale.txt",
+        "size": 1111,
+        "checksum": "sha1$hash",
+        "secondaryFiles": [
+          {
+            "class": "File",
+            "location": "s3://toil-cwl-infra-test-bucket-dont-delete/whale.txt.changedOnPorpoise",
+            "size": 24,
+            "checksum": "sha1$hash"
+          }
+        ]
+  }
+}

toil/test/cwl/seqtk_seq_job.json

@@ -0,0 +1,6 @@
+{
+  "input1": {
+    "class": "File",
+    "location": "2.fastq"
+  }
+}

toil/test/cwl/stream.json

@@ -0,0 +1,6 @@
+{
+  "input": {
+    "class": "File",
+    "location": "s3://toil-cwl-infra-test-bucket-dont-delete/whale.txt.changedOnPorpoise"
+  }
+}

toil/test/cwl/whale.txt

@@ -0,0 +1,16 @@
+Call me Ishmael. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.

toil/test/docs/scripts/example_alwaysfail.py

@@ -0,0 +1,38 @@
+import sys
+
+from configargparse import ArgumentParser
+
+from toil.common import Toil
+from toil.job import Job
+
+
+def main():
+    """
+    This workflow always fails.
+
+    Invoke like:
+
+        python examples/example_alwaysfail.py ./jobstore
+
+    Then you can inspect the job store with tools like `toil status`:
+
+        toil status --logs ./jobstore
+
+    """
+    parser = ArgumentParser(description=main.__doc__)
+    Job.Runner.addToilOptions(parser)
+    options = parser.parse_args(sys.argv[1:])
+
+    hello_job = Job.wrapJobFn(explode)
+
+    with Toil(options) as toil:
+        toil.start(hello_job)
+
+
+def explode(job):
+    sys.stderr.write("Something somewhere has gone terribly wrong\n")
+    raise RuntimeError("Boom!")
+
+
+if __name__ == "__main__":
+    main()

toil/test/docs/scripts/example_alwaysfail_with_files.wdl

@@ -0,0 +1,33 @@
+version 1.0
+workflow test {
+    call make_file as f1
+    call make_file as f2
+    call hello {
+        input:
+            name_file=f1.out,
+            unused_file=f2.out
+    }
+}
+task make_file {
+    input {
+    }
+    command <<<
+        echo "These are the contents" >test.txt
+    >>>
+    output {
+        File out = "test.txt"
+    }
+}
+task hello {
+    input {
+        File name_file
+        File? unused_file
+    }
+    command <<<
+        set -e
+        echoo "Hello" "$(cat ~{name_file})"
+    >>>
+    output {
+        File out = stdout()
+    }
+}

toil/test/docs/scripts/example_cachingbenchmark.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+# example_cachingbenchmark.py: Empirically inspect Toil caching
+
+"""
+This workflow collects statistics about caching.
+
+Invoke like:
+
+    python examples/example_cachingbenchmark.py ./jobstore --realTimeLogging --logInfo
+
+    python examples/example_cachingbenchmark.py ./jobstore --realTimeLogging --logInfo --disableCaching
+
+    python examples/example_cachingbenchmark.py aws:us-west-2:cachingjobstore --realTimeLogging --logInfo
+
+    python examples/example_cachingbenchmark.py aws:us-west-2:cachingjobstore --realTimeLogging --logInfo --disableCaching
+
+"""
+
+import argparse
+import collections
+import os
+import random
+import socket
+import sys
+import time
+
+from configargparse import ArgumentParser
+
+from toil.common import Toil
+from toil.job import Job
+from toil.realtimeLogger import RealtimeLogger
+
+
+def main():
+    parser = ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+
+    parser.add_argument(
+        "--minSleep", type=int, default=1, help="Minimum seconds to sleep"
+    )
+
+    Job.Runner.addToilOptions(parser)
+
+    options = parser.parse_args(sys.argv[1:])
+
+    root_job = Job.wrapJobFn(root, options)
+
+    with Toil(options) as toil:
+        results = toil.start(root_job)
+
+    print("Caching results:")
+    print(results)
+
+
+def root(job, options):
+    # Make a file
+    with job.fileStore.writeGlobalFileStream() as (stream, file_id):
+        stream.write(
+            b"This is a test of the Toil file caching system. "
+            b"Had this been an actual file, its contents would have been more interesting."
+        )
+
+    child_rvs = []
+    for i in range(100):
+        # Make lots of child jobs that read and report on the file
+        child_rvs.append(job.addChildJobFn(poll, options, file_id, i).rv())
+
+    # Collect all their views into a report
+    return job.addFollowOnJobFn(report, child_rvs).rv()
+
+
+def poll(job, options, file_id, number, cores=0.1, disk="200M", memory="512M"):
+
+    # Wait a random amount of time before grabbing the file for others to cache it
+    time.sleep(random.randint(options.minSleep, options.minSleep + 10))
+
+    # Read the file. Don't accept a symlink because then we might just have the
+    # filestore's copy, even if caching is not happening.
+    local_file = job.fileStore.readGlobalFile(
+        file_id, cache=True, mutable=False, symlink=False
+    )
+
+    # Wait a random amount of after before grabbing the file for others to use it
+    time.sleep(random.randint(options.minSleep, options.minSleep + 10))
+
+    # Stat the file (reads through links)
+    stats = os.stat(local_file)
+
+    # Check what machine we are
+    hostname = socket.gethostname()
+
+    RealtimeLogger.info(
+        f"Job {number} on host {hostname} sees file at device {stats.st_dev} inode {stats.st_ino}"
+    )
+
+    # Return a tuple representing our view of the file.
+    # Drop hostname since hostnames are unique per pod.
+    return stats.st_dev, stats.st_ino
+
+
+def report(job, views):
+    # Count the distinct views
+    counts = collections.Counter()
+    for v in views:
+        counts[v] += 1
+
+    report = [f"{len(counts)} distinct views, most frequent:"]
+
+    for view, count in counts.most_common(10):
+        report.append(f"{view}: {count}")
+
+    return "\n".join(report)
+
+
+if __name__ == "__main__":
+    main()

toil/test/docs/scripts/stagingExampleFiles/in.txt

@@ -0,0 +1 @@
+Hello,

toil/test/docs/scripts/stagingExampleFiles/out.txt

@@ -0,0 +1,2 @@
+Hello,
+World!

toil/test/docs/scripts/tutorial_arguments.py

@@ -0,0 +1,23 @@
+from toil.common import Toil
+from toil.job import Job
+
+
+class HelloWorld(Job):
+    def __init__(self, message):
+        Job.__init__(self)
+        self.message = message
+
+    def run(self, fileStore):
+        return "Hello, world!, here's a message: %s" % self.message
+
+
+if __name__ == "__main__":
+    parser = Job.Runner.getDefaultArgumentParser()
+    options = parser.parse_args()
+    options.logLevel = "OFF"
+    options.clean = "always"
+
+    hello_job = HelloWorld("Woot")
+
+    with Toil(options) as toil:
+        print(toil.start(hello_job))

toil/test/docs/scripts/tutorial_debugging.patch

@@ -0,0 +1,12 @@
+--- tutorial_debugging_works.wdl	2024-06-18 12:03:32
++++ tutorial_debugging_hangs.wdl	2024-06-18 12:03:53
+@@ -112,9 +112,6 @@
+                         # Also remember to encode the text as the zip file
+                         # stream is in binary mode.
+                         out_stream.write(f"{basename}: {line}".encode("utf-8"))
+-            # Even though we got distracted by zip file manipulation, remember
+-            # to pop off the file we just did.
+-            to_compress.pop()
+     EOF
+     python script.py ~{sep(" ", files)}
+     >>>

toil/test/docs/scripts/tutorial_debugging_hangs.wdl

@@ -0,0 +1,126 @@
+version 1.1
+
+workflow TutorialDebugging {
+
+    input {
+        Array[String] messages = ["Uh-oh!", "Oh dear", "Oops"]
+    }
+
+    scatter(message in messages) {
+
+        call WhaleSay {
+            input:
+                message = message
+        }
+
+        call CountLines {
+            input:
+                to_count = WhaleSay.result
+        }
+    }
+
+    Array[File] to_compress = flatten([CountLines.result, WhaleSay.result])
+
+    call CompressFiles {
+        input:
+            files = to_compress
+    }
+
+    output {
+        File compressed = CompressFiles.result
+    }
+
+}
+
+# Draw ASCII art
+task WhaleSay {
+    input {
+        String message
+    }
+
+    command <<<
+        cowsay "~{message}"
+    >>>
+
+    output {
+        File result = stdout()
+    }
+
+    runtime {
+        container: "docker/whalesay"
+    }
+}
+
+# Count the lines in a file
+task CountLines {
+    input {
+        File to_count
+    }
+
+    command <<<
+        wc -l ~{to_count}
+    >>>
+
+    output {
+        File result = stdout()
+    }
+
+    runtime {
+        container: ["ubuntu:latest", "https://gcr.io/standard-images/ubuntu:latest"]
+    }
+}
+
+# Compress files into a ZIP
+task CompressFiles {
+    input {
+        Array[File] files
+    }
+
+    command <<<
+    set -e
+    cat >script.py <<'EOF'
+    import sys
+    from zipfile import ZipFile
+    import os
+
+    # Interpret command line arguments
+    to_compress = list(reversed(sys.argv[1:]))
+
+    with ZipFile("compressed.zip", "w") as z:
+        while to_compress != []:
+            # Grab the file to add off the end of the list
+            input_filename = to_compress[-1]
+            # Now we need to write this to the zip file.
+            # What internal filename should we use?
+            basename = os.path.basename(input_filename)
+            disambiguation_number = 0
+            while True:
+                target_filename = str(disambiguation_number) + basename
+                try:
+                    z.getinfo(target_filename)
+                except KeyError:
+                    # Filename is free
+                    break
+                # Otherwise try another name
+                disambiguation_number += 1
+            # Now we can actually make the compressed file
+            with z.open(target_filename, 'w') as out_stream:
+                with open(input_filename) as in_stream:
+                    for line in in_stream:
+                        # Prefix each line of text with the original input file
+                        # it came from.
+                        # Also remember to encode the text as the zip file
+                        # stream is in binary mode.
+                        out_stream.write(f"{basename}: {line}".encode("utf-8"))
+    EOF
+    python script.py ~{sep(" ", files)}
+    >>>
+
+    output {
+        File result = "compressed.zip"
+    }
+
+    runtime {
+        container: "python:3.11"
+    }
+}