toil 5.12.0__py3-none-any.whl → 6.1.0a1__py3-none-any.whl

toil/utils/toilDebugFile.py

@@ -15,17 +15,19 @@
 import argparse
 import logging
 import os.path
+import sys
 from typing import Optional
+from distutils.util import strtobool
 
 from toil.common import Config, Toil, parser_with_common_options
-from toil.jobStores.abstractJobStore import AbstractJobStore
+from toil.jobStores.fileJobStore import FileJobStore
 from toil.lib.resources import glob
 from toil.statsAndLogging import set_logging_from_options
 
 logger = logging.getLogger(__name__)
 
 
-def fetchJobStoreFiles(jobStore: AbstractJobStore, options: argparse.Namespace) -> None:
+def fetchJobStoreFiles(jobStore: FileJobStore, options: argparse.Namespace) -> None:
     """
     Takes a list of file names as glob patterns, searches for these within a
     given directory, and attempts to take all of the files found and copy them
@@ -37,6 +39,10 @@ def fetchJobStoreFiles(jobStore: AbstractJobStore, options: argparse.Namespace)
     :param options.localFilePath: Local directory to copy files into.
     :param options.jobStore: The path to the jobStore directory.
     """
+
+    # TODO: Implement the necessary methods in the job store class and stop
+    # globbing around inside it. Does this even work?
+
     for jobStoreFile in options.fetch:
         jobStoreHits = glob(directoryname=options.jobStore,
                             glob_pattern=jobStoreFile)
@@ -48,40 +54,42 @@ def fetchJobStoreFiles(jobStore: AbstractJobStore, options: argparse.Namespace)
                                symlink=options.useSymlinks)
 
 
-def printContentsOfJobStore(jobStorePath: str, nameOfJob: Optional[str] = None) -> None:
+def printContentsOfJobStore(job_store: FileJobStore, job_id: Optional[str] = None) -> None:
     """
-    Fetch a list of all files contained in the jobStore directory input if
-    nameOfJob is not declared, otherwise it only prints out the names of files
-    for that specific job for which it can find a match.  Also creates a logFile
-    containing this same record of job files in the working directory.
-
-    :param jobStorePath: Directory path to recursively look for files.
-    :param nameOfJob: Default is None, which prints out all files in the jobStore.
-    If specified, it will print all jobStore files that have been written to the
-    jobStore by that job.
+    Fetch a list of all files contained in the job store if nameOfJob is not
+    declared, otherwise it only prints out the names of files for that specific
+    job for which it can find a match.  Also creates a log file of these file
+    names in the current directory.
+
+    :param job_store: Job store to ask for files from.
+    :param job_id: Default is None, which prints out all files in the jobStore.
+        If specified, it will print all jobStore files that have been written
+        to the jobStore by that job.
     """
 
-    if nameOfJob:
-        glob_pattern = "*" + nameOfJob + "*"
-        logFile = nameOfJob + "_fileset.txt"
+    # TODO: Implement the necessary methods for job stores other than
+    # FileJobStore.
+
+    if job_id:
+        logFile = job_id.replace("/", "_") + "_fileset.txt"
     else:
-        glob_pattern = "*"
         logFile = "jobstore_files.txt"
-        nameOfJob = ""
 
-    list_of_files = glob(directoryname=jobStorePath, glob_pattern=glob_pattern)
+    list_of_files = job_store.list_all_file_names(for_job=job_id)
     if os.path.exists(logFile):
         os.remove(logFile)
     for gfile in sorted(list_of_files):
-        if not gfile.endswith('.new'):
-            logger.debug(f"{nameOfJob} File: {os.path.basename(gfile)}")
-            with open(logFile, "a+") as f:
-                f.write(os.path.basename(gfile))
-                f.write("\n")
+        if job_id:
+            logger.debug(f"{job_id} File: {os.path.basename(gfile)}")
+        else:
+            logger.debug(f"File: {os.path.basename(gfile)}")
+        with open(logFile, "a+") as f:
+            f.write(os.path.basename(gfile))
+            f.write("\n")
 
 
 def main() -> None:
-    parser = parser_with_common_options(jobstore_option=True)
+    parser = parser_with_common_options(jobstore_option=True, prog="toil debug-file")
     parser.add_argument("--localFilePath",
                         nargs=1,
                         help="Location to which to copy job store files.")
@@ -90,11 +98,11 @@ def main() -> None:
                         help="List of job-store files to be copied locally."
                              "Use either explicit names (i.e. 'data.txt'), or "
                              "specify glob patterns (i.e. '*.txt')")
-    parser.add_argument("--listFilesInJobStore",
+    parser.add_argument("--listFilesInJobStore", type=strtobool,
                         help="Prints a list of the current files in the jobStore.")
-    parser.add_argument("--fetchEntireJobStore",
+    parser.add_argument("--fetchEntireJobStore", type=strtobool,
                         help="Copy all job store files into a local directory.")
-    parser.add_argument("--useSymlinks",
+    parser.add_argument("--useSymlinks", type=strtobool,
                         help="Creates symlink 'shortcuts' of files in the localFilePath"
                              " instead of hardlinking or copying, where possible.  If this is"
                              " not possible, it will copy the files (shutil.copyfile()).")
@@ -109,18 +117,37 @@ def main() -> None:
 
     if options.fetch:
         # Copy only the listed files locally
-        logger.debug("Fetching local files: %s", options.fetch)
-        fetchJobStoreFiles(jobStore=jobStore, options=options)
+
+        if isinstance(jobStore, FileJobStore):
+            logger.debug("Fetching local files: %s", options.fetch)
+            fetchJobStoreFiles(jobStore=jobStore, options=options)
+        else:
+            # The user asked for something we can't do yet.
+            # Tell them no but don't stack trace.
+            logger.critical("Can only fetch by name or glob from file-based job stores")
+            sys.exit(1)
 
     elif options.fetchEntireJobStore:
         # Copy all jobStore files locally
-        logger.debug("Fetching all local files.")
-        options.fetch = "*"
-        fetchJobStoreFiles(jobStore=jobStore, options=options)
+
+        if isinstance(jobStore, FileJobStore):
+            logger.debug("Fetching all local files.")
+            options.fetch = "*"
+            fetchJobStoreFiles(jobStore=jobStore, options=options)
+        else:
+            logger.critical("Can only fetch by name or glob from file-based job stores")
+            sys.exit(1)
 
     if options.listFilesInJobStore:
         # Log filenames and create a file containing these names in cwd
-        printContentsOfJobStore(jobStorePath=options.jobStore)
+
+        if isinstance(jobStore, FileJobStore):
+            printContentsOfJobStore(job_store=jobStore)
+        else:
+            logger.critical("Can only list files from file-based job stores")
+            sys.exit(1)
+
+    # TODO: We can't actually do *anything* for non-file job stores.
 
 
 if __name__ == "__main__":

toil/utils/toilDebugJob.py

@@ -14,7 +14,11 @@
 """Debug tool for running a toil job locally."""
 import logging
 
+import pprint
+import sys
+
 from toil.common import Config, Toil, parser_with_common_options
+from toil.jobStores.fileJobStore import FileJobStore
 from toil.statsAndLogging import set_logging_from_options
 from toil.utils.toilDebugFile import printContentsOfJobStore
 from toil.worker import workerScript
@@ -23,28 +27,51 @@ logger = logging.getLogger(__name__)
 
 
 def main() -> None:
-    parser = parser_with_common_options(jobstore_option=True)
-    parser.add_argument("jobID", nargs=1,
+    parser = parser_with_common_options(jobstore_option=True, prog="toil debug-job")
+    parser.add_argument("jobID", type=str, nargs='?', default=None,
                         help="The job store id of a job within the provided jobstore to run by itself.")
-    parser.add_argument("--printJobInfo", nargs=1,
-                        help="Return information about this job to the user including preceding jobs, "
-                             "inputs, outputs, and runtime from the last known run.")
+    parser.add_argument("--printJobInfo", type=str,
+                        help="Dump debugging info about this job ID")
 
     options = parser.parse_args()
     set_logging_from_options(options)
-    config = Config()
+
+    jobStore = Toil.resumeJobStore(options.jobStore)
+    # Get the config with the workflow ID from the job store
+    config = jobStore.config
+    # But override its options
     config.setOptions(options)
 
-    jobStore = Toil.resumeJobStore(config.jobStore)
+    did_something = False
 
     if options.printJobInfo:
-        printContentsOfJobStore(jobStorePath=config.jobStore, nameOfJob=options.printJobInfo)
+        if isinstance(jobStore, FileJobStore):
+            # List all its files if we can
+            printContentsOfJobStore(job_store=jobStore, job_id=options.printJobInfo)
+        # Print the job description itself
+        job_desc = jobStore.load_job(options.printJobInfo)
+        print(f"Job: {job_desc}")
+        pprint.pprint(job_desc.__dict__)
+
+        did_something = True
 
     # TODO: Option to print list of successor jobs
     # TODO: Option to run job within python debugger, allowing step through of arguments
     # idea would be to have option to import pdb and set breakpoint at the start of the user's code
 
-    jobID = options.jobID[0]
-    logger.debug(f"Running the following job locally: {jobID}")
-    workerScript(jobStore, config, jobID, jobID, redirectOutputToLogFile=False)
-    logger.debug(f"Finished running: {jobID}")
+    if options.jobID is not None:
+        # We actually want to run a job.
+
+        jobID = options.jobID
+        logger.debug(f"Running the following job locally: {jobID}")
+        workerScript(jobStore, config, jobID, jobID, redirectOutputToLogFile=False)
+        logger.debug(f"Finished running: {jobID}")
+        # Even if the job fails, the worker script succeeds unless something goes wrong with it internally.
+
+        did_something = True
+
+    if not did_something:
+        # Somebody forgot to tell us to do anything.
+        # Show the usage instructions.
+        parser.print_help()
+        sys.exit(1)

toil/utils/toilDestroyCluster.py

@@ -21,7 +21,7 @@ from toil.statsAndLogging import set_logging_from_options
 logger = logging.getLogger(__name__)
 
 def main() -> None:
-    parser = parser_with_common_options(provisioner_options=True, jobstore_option=False)
+    parser = parser_with_common_options(provisioner_options=True, jobstore_option=False, prog="toil destroy-cluster")
     options = parser.parse_args()
     set_logging_from_options(options)
 

toil/utils/toilKill.py

@@ -25,7 +25,7 @@ logger = logging.getLogger(__name__)
 
 
 def main() -> None:
-    parser = parser_with_common_options()
+    parser = parser_with_common_options(prog="toil kill")
     parser.add_argument('--force', action='store_true',
                         help="Send SIGKILL to the leader process if local.")
     options = parser.parse_args()

toil/utils/toilLaunchCluster.py

@@ -37,9 +37,10 @@ def create_tags_dict(tags: List[str]) -> Dict[str, str]:
 
 
 def main() -> None:
-    parser = parser_with_common_options(provisioner_options=True, jobstore_option=False)
+    parser = parser_with_common_options(provisioner_options=True, jobstore_option=False, prog="toil launch-cluster")
     parser.add_argument("-T", "--clusterType", dest="clusterType",
-                        choices=['mesos', 'kubernetes'], default='mesos',
+                        choices=['mesos', 'kubernetes'],
+                        default=None,  # TODO: change default to "kubernetes" when we are ready.
                         help="Cluster scheduler to use.")
     parser.add_argument("--leaderNodeType", dest="leaderNodeType", required=True,
                         help="Non-preemptible node type to use for the cluster leader.")
@@ -160,6 +161,16 @@ def main() -> None:
         raise RuntimeError(f'Please provide a value for --zone or set a default in the '
                            f'TOIL_{options.provisioner.upper()}_ZONE environment variable.')
 
+    if options.clusterType == "mesos":
+        logger.warning('You are using a Mesos cluster, which is no longer recommended as Toil is '
+                       'transitioning to Kubernetes-based clusters. Consider switching to '
+                       '--clusterType=kubernetes instead.')
+
+    if options.clusterType is None:
+        logger.warning('Argument --clusterType is not set... using "mesos". '
+                       'In future versions of Toil, the default cluster scheduler will be '
+                       'set to "kubernetes" if the cluster type is not specified.')
+        options.clusterType = "mesos"
 
     logger.info('Creating cluster %s...', options.clusterName)
 

toil/utils/toilMain.py

@@ -5,7 +5,7 @@ import textwrap
 import types
 from typing import Any, Dict
 
-import pkg_resources
+from importlib.metadata import version as metadata_version
 
 from toil.version import version
 
@@ -47,6 +47,7 @@ def get_or_die(module: types.ModuleType, name: str) -> Any:
 def loadModules() -> Dict[str, types.ModuleType]:
     # noinspection PyUnresolvedReferences
     from toil.utils import toilClean  # noqa
+    from toil.utils import toilConfig  # noqa
     from toil.utils import toilDebugFile  # noqa
     from toil.utils import toilDebugJob  # noqa
     from toil.utils import toilDestroyCluster  # noqa
@@ -77,6 +78,6 @@ def printHelp(modules: Dict[str, types.ModuleType]) -> None:
 
 def printVersion() -> None:
     try:
-        print(pkg_resources.get_distribution('toil').version)
+        print(metadata_version('toil'))
     except:
         print(f'Version gathered from toil.version: {version}')

toil/utils/toilRsyncCluster.py

@@ -23,7 +23,7 @@ logger = logging.getLogger(__name__)
 
 
 def main() -> None:
-    parser = parser_with_common_options(provisioner_options=True, jobstore_option=False)
+    parser = parser_with_common_options(provisioner_options=True, jobstore_option=False, prog="toil rsync-cluster")
     parser.add_argument("--insecure", dest='insecure', action='store_true', required=False,
                         help="Temporarily disable strict host key checking.")
     parser.add_argument("args", nargs=argparse.REMAINDER, help="Arguments to pass to"

toil/utils/toilSshCluster.py

@@ -25,7 +25,7 @@ logger = logging.getLogger(__name__)
 
 
 def main() -> None:
-    parser = parser_with_common_options(provisioner_options=True, jobstore_option=False)
+    parser = parser_with_common_options(provisioner_options=True, jobstore_option=False, prog="toil ssh-cluster")
     parser.add_argument("--insecure", action='store_true',
                         help="Temporarily disable strict host key checking.")
     parser.add_argument("--sshOption", dest='sshOptions', default=[], action='append',