toil 6.1.0a1__py3-none-any.whl → 8.0.0__py3-none-any.whl

toil/options/cwl.py

@@ -15,45 +15,66 @@ def add_cwl_options(parser: ArgumentParser, suppress: bool = True) -> None:
     :return: None
     """
     suppress_help = SUPPRESS if suppress else None
+
+    # These are options that we have to match cwltool
+    # TODO: Are there still any Toil-specific options in here?
     parser.add_argument("--not-strict", action="store_true", help=suppress_help)
     parser.add_argument(
         "--enable-dev",
         action="store_true",
-        help=suppress_help or suppress_help or "Enable loading and running development versions of CWL",
+        help=suppress_help
+        or suppress_help
+        or "Enable loading and running development versions of CWL",
     )
     parser.add_argument(
         "--enable-ext",
         action="store_true",
-        help=suppress_help or "Enable loading and running 'cwltool:' extensions to the CWL standards.",
+        help=suppress_help
+        or "Enable loading and running 'cwltool:' extensions to the CWL standards.",
         default=False,
     )
-    parser.add_argument("--quiet", dest="quiet", action="store_true", default=False, help=suppress_help)
-    parser.add_argument("--basedir", type=str, help=suppress_help)  # TODO: Might be hard-coded?
+    parser.add_argument(
+        "--quiet", dest="quiet", action="store_true", default=False, help=suppress_help
+    )
+    parser.add_argument(
+        "--basedir", type=str, help=suppress_help
+    )  # TODO: Might be hard-coded?
     parser.add_argument("--outdir", type=str, default=None, help=suppress_help)
-    parser.add_argument("--version", action="version", version=baseVersion,
-                        help=suppress_help or "show program's version number and exit")
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=baseVersion,
+        help=suppress_help or "show program's version number and exit",
+    )
     parser.add_argument(
         "--log-dir",
         type=str,
         default="",
-        help=suppress_help or "Log your tools stdout/stderr to this location outside of container",
+        help=suppress_help
+        or "Log your tools stdout/stderr to this location outside of container",
     )
     # this is as a result of suppressed help statements not working well with mutually_exclusive_groups, which will
     # cause an assertion error
     # https://github.com/python/cpython/issues/62090
-    dockergroup = parser.add_mutually_exclusive_group() if not suppress_help else parser.add_argument_group()
+    dockergroup = (
+        parser.add_mutually_exclusive_group()
+        if not suppress_help
+        else parser.add_argument_group()
+    )
     dockergroup.add_argument(
         "--user-space-docker-cmd",
-        help=suppress_help or "(Linux/OS X only) Specify a user space docker command (like "
-                              "udocker or dx-docker) that will be used to call 'pull' and 'run'",
+        help=suppress_help
+        or "(Linux/OS X only) Specify a user space docker command (like "
+        "udocker or dx-docker) that will be used to call 'pull' and 'run'",
     )
     dockergroup.add_argument(
         "--singularity",
         action="store_true",
         default=False,
-        help=suppress_help or "Use Singularity runtime for running containers. "
-                              "Requires Singularity v2.6.1+ and Linux with kernel version v3.18+ or "
-                              "with overlayfs support backported.",
+        help=suppress_help
+        or "Use Singularity runtime for running containers. "
+        "Requires Singularity v2.6.1+ and Linux with kernel version v3.18+ or "
+        "with overlayfs support backported.",
     )
     dockergroup.add_argument(
         "--podman",
@@ -64,21 +85,23 @@ def add_cwl_options(parser: ArgumentParser, suppress: bool = True) -> None:
     dockergroup.add_argument(
         "--no-container",
         action="store_true",
-        help=suppress_help or "Do not execute jobs in a "
-                              "Docker container, even when `DockerRequirement` "
-                              "is specified under `hints`.",
+        help=suppress_help
+        or "Do not execute jobs in a "
+        "Docker container, even when `DockerRequirement` "
+        "is specified under `hints`.",
     )
     dockergroup.add_argument(
         "--leave-container",
         action="store_false",
         default=True,
-        help=suppress_help or "Do not delete Docker container used by jobs after they exit",
+        help=suppress_help
+        or "Do not delete Docker container used by jobs after they exit",
         dest="rm_container",
     )
-    extra_dockergroup = parser.add_argument_group()
-    extra_dockergroup.add_argument(
+    parser.add_argument(
         "--custom-net",
-        help=suppress_help or "Specify docker network name to pass to docker run command",
+        help=suppress_help
+        or "Specify docker network name to pass to docker run command",
     )
     cidgroup = parser.add_argument_group(
         "Options for recording the Docker container identifier into a file."
@@ -95,7 +118,8 @@ def add_cwl_options(parser: ArgumentParser, suppress: bool = True) -> None:
     cidgroup.add_argument(
         "--cidfile-dir",
         type=str,
-        help=suppress_help or "Store the Docker container ID into a file in the specified directory.",
+        help=suppress_help
+        or "Store the Docker container ID into a file in the specified directory.",
         default=None,
         dest="cidfile_dir",
     )
@@ -103,19 +127,28 @@ def add_cwl_options(parser: ArgumentParser, suppress: bool = True) -> None:
     cidgroup.add_argument(
         "--cidfile-prefix",
         type=str,
-        help=suppress_help or "Specify a prefix to the container ID filename. "
-                              "Final file name will be followed by a timestamp. "
-                              "The default is no prefix.",
+        help=suppress_help
+        or "Specify a prefix to the container ID filename. "
+        "Final file name will be followed by a timestamp. "
+        "The default is no prefix.",
         default=None,
         dest="cidfile_prefix",
     )
 
+    parser.add_argument(
+        "--no-prepull",
+        action="store_true",
+        default=False,
+        help=suppress_help
+        or "Do not prepull the container prior to running the workflow",
+    )
+
     parser.add_argument(
         "--preserve-environment",
         type=str,
         nargs="+",
-        help=suppress_help or "Preserve specified environment variables when running"
-                              " CommandLineTools",
+        help=suppress_help
+        or "Preserve specified environment variables when running" " CommandLineTools",
         metavar=("VAR1 VAR2"),
         default=("PATH",),
         dest="preserve_environment",
@@ -123,19 +156,29 @@ def add_cwl_options(parser: ArgumentParser, suppress: bool = True) -> None:
     parser.add_argument(
         "--preserve-entire-environment",
         action="store_true",
-        help=suppress_help or "Preserve all environment variable when running CommandLineTools.",
+        help=suppress_help
+        or "Preserve all environment variable when running CommandLineTools.",
         default=False,
         dest="preserve_entire_environment",
     )
     parser.add_argument(
-        "--destBucket",
-        type=str,
-        help=suppress_help or "Specify a cloud bucket endpoint for output files.",
+        "--beta-dependency-resolvers-configuration", default=None, help=suppress_help
+    )
+    parser.add_argument(
+        "--beta-dependencies-directory", default=None, help=suppress_help
+    )
+    parser.add_argument(
+        "--beta-use-biocontainers",
+        default=None,
+        action="store_true",
+        help=suppress_help,
+    )
+    parser.add_argument(
+        "--beta-conda-dependencies",
+        default=None,
+        action="store_true",
+        help=suppress_help,
     )
-    parser.add_argument("--beta-dependency-resolvers-configuration", default=None, help=suppress_help)
-    parser.add_argument("--beta-dependencies-directory", default=None, help=suppress_help)
-    parser.add_argument("--beta-use-biocontainers", default=None, action="store_true", help=suppress_help)
-    parser.add_argument("--beta-conda-dependencies", default=None, action="store_true", help=suppress_help)
     parser.add_argument(
         "--tmpdir-prefix",
         type=str,
@@ -170,32 +213,35 @@ def add_cwl_options(parser: ArgumentParser, suppress: bool = True) -> None:
     parser.add_argument(
         "--strict-memory-limit",
         action="store_true",
-        help=suppress_help or "When running with "
-                              "software containers and the Docker engine, pass either the "
-                              "calculated memory allocation from ResourceRequirements or the "
-                              "default of 1 gigabyte to Docker's --memory option.",
+        help=suppress_help
+        or "When running with "
+        "software containers and the Docker engine, pass either the "
+        "calculated memory allocation from ResourceRequirements or the "
+        "default of 1 gigabyte to Docker's --memory option.",
     )
     parser.add_argument(
         "--strict-cpu-limit",
         action="store_true",
-        help=suppress_help or "When running with "
-                              "software containers and the Docker engine, pass either the "
-                              "calculated cpu allocation from ResourceRequirements or the "
-                              "default of 1 core to Docker's --cpu option. "
-                              "Requires docker version >= v1.13.",
+        help=suppress_help
+        or "When running with "
+        "software containers and the Docker engine, pass either the "
+        "calculated cpu allocation from ResourceRequirements or the "
+        "default of 1 core to Docker's --cpu option. "
+        "Requires docker version >= v1.13.",
     )
     parser.add_argument(
         "--relax-path-checks",
         action="store_true",
         default=False,
-        help=suppress_help or "Relax requirements on path names to permit "
-                              "spaces and hash characters.",
+        help=suppress_help
+        or "Relax requirements on path names to permit " "spaces and hash characters.",
         dest="relax_path_checks",
     )
     parser.add_argument(
         "--default-container",
-        help=suppress_help or "Specify a default docker container that will be "
-                              "used if the workflow fails to specify one.",
+        help=suppress_help
+        or "Specify a default docker container that will be "
+        "used if the workflow fails to specify one.",
     )
     parser.add_argument(
         "--disable-validate",
@@ -212,7 +258,11 @@ def add_cwl_options(parser: ArgumentParser, suppress: bool = True) -> None:
         help=suppress_help or SUPPRESS,
     )
     # same workaround as dockergroup
-    checkgroup = parser.add_mutually_exclusive_group() if not suppress_help else parser.add_argument_group()
+    checkgroup = (
+        parser.add_mutually_exclusive_group()
+        if not suppress_help
+        else parser.add_argument_group()
+    )
     checkgroup.add_argument(
         "--compute-checksum",
         action="store_true",
@@ -223,14 +273,16 @@ def add_cwl_options(parser: ArgumentParser, suppress: bool = True) -> None:
     checkgroup.add_argument(
         "--no-compute-checksum",
         action="store_false",
-        help=suppress_help or "Do not compute checksum of contents while collecting outputs",
+        help=suppress_help
+        or "Do not compute checksum of contents while collecting outputs",
         dest="compute_checksum",
     )
 
     parser.add_argument(
         "--eval-timeout",
-        help=suppress_help or "Time to wait for a Javascript expression to evaluate before giving "
-                              "an error, default 20s.",
+        help=suppress_help
+        or "Time to wait for a Javascript expression to evaluate before giving "
+        "an error, default 20s.",
         type=float,
         default=20,
     )
@@ -245,34 +297,11 @@ def add_cwl_options(parser: ArgumentParser, suppress: bool = True) -> None:
         "--mpi-config-file",
         type=str,
         default=None,
-        help=suppress_help or "Platform specific configuration for MPI (parallel "
-                              "launcher, its flag etc). See the cwltool README "
-                              "section 'Running MPI-based tools' for details of the format: "
-                              "https://github.com/common-workflow-language/cwltool#running-mpi-based-tools-that-need-to-be-launched",
-    )
-    parser.add_argument(
-        "--bypass-file-store",
-        action="store_true",
-        default=False,
-        help=suppress_help or "Do not use Toil's file store and assume all "
-                              "paths are accessible in place from all nodes.",
-        dest="bypass_file_store",
-    )
-    parser.add_argument(
-        "--reference-inputs",
-        action="store_true",
-        default=False,
-        help=suppress_help or "Do not copy remote inputs into Toil's file "
-        "store and assume they are accessible in place from "
-        "all nodes.",
-        dest="reference_inputs",
-    )
-    parser.add_argument(
-        "--disable-streaming",
-        action="store_true",
-        default=False,
-        help=suppress_help or "Disable file streaming for files that have 'streamable' flag True",
-        dest="disable_streaming",
+        help=suppress_help
+        or "Platform specific configuration for MPI (parallel "
+        "launcher, its flag etc). See the cwltool README "
+        "section 'Running MPI-based tools' for details of the format: "
+        "https://github.com/common-workflow-language/cwltool#running-mpi-based-tools-that-need-to-be-launched",
     )
 
     provgroup = parser.add_argument_group(
@@ -280,9 +309,10 @@ def add_cwl_options(parser: ArgumentParser, suppress: bool = True) -> None:
     )
     provgroup.add_argument(
         "--provenance",
-        help=suppress_help or "Save provenance to specified folder as a "
-                              "Research Object that captures and aggregates "
-                              "workflow execution and data products.",
+        help=suppress_help
+        or "Save provenance to specified folder as a "
+        "Research Object that captures and aggregates "
+        "workflow execution and data products.",
         type=str,
     )
 
@@ -316,21 +346,76 @@ def add_cwl_options(parser: ArgumentParser, suppress: bool = True) -> None:
     )
     provgroup.add_argument(
         "--orcid",
-        help=suppress_help or "Record user ORCID identifier as part of "
-                              "provenance, e.g. https://orcid.org/0000-0002-1825-0097 "
-                              "or 0000-0002-1825-0097. Alternatively the environment variable "
-                              "ORCID may be set.",
+        help=suppress_help
+        or "Record user ORCID identifier as part of "
+        "provenance, e.g. https://orcid.org/0000-0002-1825-0097 "
+        "or 0000-0002-1825-0097. Alternatively the environment variable "
+        "ORCID may be set.",
         dest="orcid",
         default=os.environ.get("ORCID", ""),
         type=str,
     )
     provgroup.add_argument(
         "--full-name",
-        help=suppress_help or "Record full name of user as part of provenance, "
-                              "e.g. Josiah Carberry. You may need to use shell quotes to preserve "
-                              "spaces. Alternatively the environment variable CWL_FULL_NAME may "
-                              "be set.",
+        help=suppress_help
+        or "Record full name of user as part of provenance, "
+        "e.g. Josiah Carberry. You may need to use shell quotes to preserve "
+        "spaces. Alternatively the environment variable CWL_FULL_NAME may "
+        "be set.",
         dest="cwl_full_name",
         default=os.environ.get("CWL_FULL_NAME", ""),
         type=str,
     )
+
+    # These are Toil-specific options
+    parser.add_argument(
+        "--bypass-file-store",
+        action="store_true",
+        default=False,
+        help=suppress_help
+        or "Do not use Toil's file store and assume all "
+        "paths are accessible in place from all nodes.",
+        dest="bypass_file_store",
+    )
+    parser.add_argument(
+        "--reference-inputs",
+        action="store_true",
+        default=False,
+        help=suppress_help
+        or "Do not copy remote inputs into Toil's file "
+        "store and assume they are accessible in place from "
+        "all nodes.",
+        dest="reference_inputs",
+    )
+    parser.add_argument(
+        "--disable-streaming",
+        action="store_true",
+        default=False,
+        help=suppress_help
+        or "Disable file streaming for files that have 'streamable' flag True.",
+        dest="disable_streaming",
+    )
+    ram_group = (
+        parser.add_mutually_exclusive_group()
+        if not suppress_help
+        else parser.add_argument_group()
+    )
+    ram_group.add_argument(
+        "--cwl-default-ram",
+        action="store_true",
+        default=True,
+        help=suppress_help or "Apply CWL specification default ramMin.",
+        dest="cwl_default_ram",
+    )
+    ram_group.add_argument(
+        "--no-cwl-default-ram",
+        action="store_false",
+        help=suppress_help
+        or "Do not apply CWL specification default ramMin, so that Toil --defaultMemory applies.",
+        dest="cwl_default_ram",
+    )
+    parser.add_argument(
+        "--destBucket",
+        type=str,
+        help=suppress_help or "Specify a cloud bucket endpoint for output files.",
+    )

toil/options/runner.py

@@ -0,0 +1,50 @@
+from argparse import ArgumentParser
+
+from toil.lib.conversions import human2bytes
+
+
+def add_runner_options(
+    parser: ArgumentParser, cwl: bool = False, wdl: bool = False
+) -> None:
+    """
+    Add to the WDL or CWL runners options that are shared or the same between runners
+    :param parser: parser to add arguments to
+    :param cwl: bool
+    :param wdl: bool
+    :return: None
+    """
+    # This function should be constructed so that even when wdl and cwl are false, the "default" options are still added
+    run_imports_on_workers_arguments = ["--runImportsOnWorkers"]
+    if cwl:
+        run_imports_on_workers_arguments.append("--run-imports-on-workers")
+    parser.add_argument(
+        *run_imports_on_workers_arguments,
+        action="store_true",
+        default=False,
+        dest="run_imports_on_workers",
+        help="Run the file imports on a worker instead of the leader. This is useful if the leader is not optimized for high network performance. "
+        "If set to true, the argument --importWorkersDisk must also be set."
+    )
+    import_workers_threshold_argument = ["--importWorkersThreshold"]
+    if cwl:
+        import_workers_threshold_argument.append("--import-workers-threshold")
+    parser.add_argument(
+        *import_workers_threshold_argument,
+        dest="import_workers_threshold",
+        type=lambda x: human2bytes(str(x)),
+        default="1 GiB",
+        help="Specify the file size threshold that determines how many files go into a batched import. As many files will go into a batch import job until this threshold "
+             "is reached. This should be set in conjunction with the argument --runImportsOnWorkers."
+    )
+    import_workers_disk_argument = ["--importWorkersDisk"]
+    if cwl:
+        import_workers_disk_argument.append("--import-workers-disk")
+    parser.add_argument(
+        *import_workers_disk_argument,
+        dest="import_workers_disk",
+        type=lambda x: human2bytes(str(x)),
+        default="1 MiB",
+        help="Specify the disk size each import worker will get. This may be necessary when file streaming is not possible. For example, downloading from AWS to a GCE "
+             "job store. If specified, this should be set to the largest file size of all files to import. This should be set in conjunction with the arguments "
+             "--runImportsOnWorkers and --importWorkersThreshold."
+    )

toil/options/wdl.py

@@ -2,6 +2,8 @@ from argparse import ArgumentParser
 
 from configargparse import SUPPRESS
 
+from toil.lib.conversions import strtobool
+
 
 def add_wdl_options(parser: ArgumentParser, suppress: bool = True) -> None:
     """
@@ -13,20 +15,74 @@ def add_wdl_options(parser: ArgumentParser, suppress: bool = True) -> None:
     suppress_help = SUPPRESS if suppress else None
     # include arg names without a wdl specifier if suppress is False
     # this is to avoid possible duplicate options in custom toil scripts, ex outputFile can be a common argument name
-    output_dialect_arguments = ["--wdlOutputDialect"] + (["--outputDialect"] if not suppress else [])
-    parser.add_argument(*output_dialect_arguments, dest="output_dialect", type=str, default='cromwell',
-                        choices=['cromwell', 'miniwdl'],
-                        help=suppress_help or ("JSON output format dialect. 'cromwell' just returns the workflow's "
-                                               "output values as JSON, while 'miniwdl' nests that under an 'outputs' "
-                                               "key, and includes a 'dir' key where files are written."))
-    output_directory_arguments = ["--wdlOutputDirectory"] + (["--outputDirectory", "-o"] if not suppress else [])
-    parser.add_argument(*output_directory_arguments, dest="output_directory", type=str, default=None,
-                        help=suppress_help or (
-                            "Directory or URI prefix to save output files at. By default a new directory is created "
-                            "in the current directory."))
-    output_file_arguments = ["--wdlOutputFile"] + (["--outputFile", "-m"] if not suppress else [])
-    parser.add_argument(*output_file_arguments, dest="output_file", type=str, default=None,
-                        help=suppress_help or "File or URI to save output JSON to.")
-    reference_inputs_arguments = ["--wdlReferenceInputs"] + (["--referenceInputs"] if not suppress else [])
-    parser.add_argument(*reference_inputs_arguments, dest="reference_inputs", type=bool, default=False,
-                        help=suppress_help or "Pass input files by URL")
+    # TODO: Why do we even need them at all in other Toil scripts? Do we have to worry about dest= collisions?
+    # TODO: Can the better option name be first?
+    output_dialect_arguments = ["--wdlOutputDialect"] + (
+        ["--outputDialect"] if not suppress else []
+    )
+    parser.add_argument(
+        *output_dialect_arguments,
+        dest="output_dialect",
+        type=str,
+        default="cromwell",
+        choices=["cromwell", "miniwdl"],
+        help=suppress_help
+        or (
+            "JSON output format dialect. 'cromwell' just returns the workflow's "
+            "output values as JSON, while 'miniwdl' nests that under an 'outputs' "
+            "key, and includes a 'dir' key where files are written."
+        )
+    )
+    output_directory_arguments = ["--wdlOutputDirectory"] + (
+        ["--outputDirectory", "-o"] if not suppress else []
+    )
+    parser.add_argument(
+        *output_directory_arguments,
+        dest="output_directory",
+        type=str,
+        default=None,
+        help=suppress_help
+        or (
+            "Directory or URI prefix to save output files at. By default a new directory is created "
+            "in the current directory."
+        )
+    )
+    output_file_arguments = ["--wdlOutputFile"] + (
+        ["--outputFile", "-m"] if not suppress else []
+    )
+    parser.add_argument(
+        *output_file_arguments,
+        dest="output_file",
+        type=str,
+        default=None,
+        help=suppress_help or "File or URI to save output JSON to."
+    )
+    reference_inputs_arguments = ["--wdlReferenceInputs"] + (
+        ["--referenceInputs"] if not suppress else []
+    )
+    parser.add_argument(
+        *reference_inputs_arguments,
+        dest="reference_inputs",
+        type=strtobool,
+        default=False,
+        help=suppress_help or "Pass input files by URL"
+    )
+    container_arguments = ["--wdlContainer"] + (["--container"] if not suppress else [])
+    parser.add_argument(
+        *container_arguments,
+        dest="container",
+        type=str,
+        choices=["singularity", "docker", "auto"],
+        default="auto",
+        help=suppress_help or "Container engine to use to run WDL tasks"
+    )
+    all_call_outputs_arguments = ["--wdlAllCallOutputs"] + (
+        ["--allCallOutputs"] if not suppress else []
+    )
+    parser.add_argument(
+        *all_call_outputs_arguments,
+        dest="all_call_outputs",
+        type=strtobool,
+        default=None,
+        help=suppress_help or "Keep and return all call outputs as workflow outputs"
+    )