toil 8.0.0__py3-none-any.whl → 8.1.0b1__py3-none-any.whl

toil/common.py

@@ -14,6 +14,7 @@
 import json
 import logging
 import os
+import platform
 import pickle
 import re
 import signal
@@ -53,6 +54,7 @@ import requests
 from configargparse import ArgParser, YAMLConfigFileParser
 from ruamel.yaml import YAML
 from ruamel.yaml.comments import CommentedMap
+from ruamel.yaml.scalarstring import DoubleQuotedScalarString
 
 from toil import logProcessContext, lookupEnvVar
 from toil.batchSystems.options import set_batchsystem_options
@@ -69,7 +71,10 @@ from toil.bus import (
 )
 from toil.fileStores import FileID
 from toil.lib.compatibility import deprecated
-from toil.lib.io import AtomicFileCreate, try_path
+from toil.lib.history import HistoryManager
+from toil.lib.history_submission import ask_user_about_publishing_metrics, create_history_submission, create_current_submission
+from toil.lib.io import AtomicFileCreate, try_path, get_toil_home
+from toil.lib.memoize import memoize
 from toil.lib.retry import retry
 from toil.lib.threading import ensure_filesystem_lockable
 from toil.options.common import JOBSTORE_HELP, add_base_toil_options
@@ -79,7 +84,7 @@ from toil.options.wdl import add_wdl_options
 from toil.provisioners import add_provisioner_options, cluster_factory
 from toil.realtimeLogger import RealtimeLogger
 from toil.statsAndLogging import add_logging_options, set_logging_from_options
-from toil.version import dockerRegistry, dockerTag, version
+from toil.version import dockerRegistry, dockerTag, version, baseVersion
 
 if TYPE_CHECKING:
     from toil.batchSystems.abstractBatchSystem import AbstractBatchSystem
@@ -92,11 +97,14 @@ if TYPE_CHECKING:
 UUID_LENGTH = 32
 logger = logging.getLogger(__name__)
 
-# TODO: should this use an XDG config directory or ~/.config to not clutter the
-# base home directory?
-TOIL_HOME_DIR: str = os.path.join(os.path.expanduser("~"), ".toil")
-DEFAULT_CONFIG_FILE: str = os.path.join(TOIL_HOME_DIR, "default.yaml")
+@memoize
+def get_default_config_path() -> str:
+    """
+    Get the default path where the Toil configuration file lives.
 
+    The file at the path will not necessarily exist.
+    """
+    return os.path.join(get_toil_home(), "default.yaml")
 
 class Config:
     """Class to represent configuration operations for a toil workflow run."""
@@ -214,6 +222,9 @@ class Config:
     write_messages: Optional[str]
     realTimeLogging: bool
 
+    # Data publishing
+    publish_workflow_metrics: Union[Literal["all"], Literal["current"], Literal["no"], None]
+
     # Misc
     environment: dict[str, str]
     disableChaining: bool
@@ -387,6 +398,9 @@ class Config:
         set_option("writeLogsGzip")
         set_option("writeLogsFromAllJobs")
         set_option("write_messages")
+        
+        # Data Publishing Options
+        set_option("publish_workflow_metrics")
 
         if self.write_messages is None:
             # The user hasn't specified a place for the message bus so we
@@ -463,44 +477,20 @@ class Config:
     def __hash__(self) -> int:
         return self.__dict__.__hash__()  # type: ignore
 
-
-def check_and_create_toil_home_dir() -> None:
-    """
-    Ensure that TOIL_HOME_DIR exists.
-
-    Raises an error if it does not exist and cannot be created. Safe to run
-    simultaneously in multiple processes.
-    """
-
-    dir_path = try_path(TOIL_HOME_DIR)
-    if dir_path is None:
-        raise RuntimeError(
-            f"Cannot create or access Toil configuration directory {TOIL_HOME_DIR}"
-        )
-
-
-def check_and_create_default_config_file() -> None:
+def ensure_config(filepath: str) -> None:
     """
-    If the default config file does not exist, create it in the Toil home directory. Create the Toil home directory
-    if needed
+    If the config file at the filepath does not exist, create it.
+    The parent directory should be created prior to calling this.
 
-    Raises an error if the default config file cannot be created.
+    Raises an error if the config file cannot be created.
     Safe to run simultaneously in multiple processes. If this process runs
-    this function, it will always see the default config file existing with
+    this function, it will always see the config file existing with
     parseable contents, even if other processes are racing to create it.
 
-    No process will see an empty or partially-written default config file.
-    """
-    check_and_create_toil_home_dir()
-    # The default config file did not appear to exist when we checked.
-    # It might exist now, though. Try creating it.
-    check_and_create_config_file(DEFAULT_CONFIG_FILE)
-
+    No process will see a new empty or partially-written config file. The
+    caller should still check to make sure there isn't a preexisting empty file
+    here.
 
-def check_and_create_config_file(filepath: str) -> None:
-    """
-    If the config file at the filepath does not exist, try creating it.
-    The parent directory should be created prior to calling this
     :param filepath: path to config file
     :return: None
     """
@@ -648,9 +638,39 @@ def generate_config(filepath: str) -> None:
                 yaml.dump(
                     data,
                     f,
+                    # Comment everything out, Unix config file style, to show defaults
                     transform=lambda s: re.sub(r"^(.)", r"#\1", s, flags=re.MULTILINE),
                 )
 
+def update_config(filepath: str, key: str, new_value: Union[str, bool, int, float]) -> None:
+    """
+    Set the given top-level key to the given value in the given YAML config
+    file.
+
+    Does not dramatically alter comments or formatting, and does not make a
+    partially-written file visible.
+
+    :param key: Setting to set. Must be the command-line option name, not the
+        destination variable name.
+    """
+    
+    yaml = YAML(typ="rt")
+    data = yaml.load(open(filepath))
+
+    logger.info("Change config field %s from %s to %s", key, repr(data.get(key, None)), repr(new_value))
+
+    if isinstance(new_value, str):
+        # Strings with some values (no, yes) will be interpreted as booleans on
+        # load if not quoted. But ruamel is not determining that this is needed
+        # on serialization for newly-added values. So if we set something to a
+        # string we always quote it.
+        data[key] = DoubleQuotedScalarString(new_value)
+    else:
+        data[key] = new_value
+
+    with AtomicFileCreate(filepath) as temp_path:
+        with open(temp_path, "w") as f:
+            yaml.dump(data, f)
 
 def parser_with_common_options(
     provisioner_options: bool = False,
@@ -708,11 +728,13 @@ def addOptions(
             f"Unanticipated class: {parser.__class__}.  Must be: argparse.ArgumentParser or ArgumentGroup."
         )
 
+    config_path = get_default_config_path()
+
     if isinstance(parser, ArgParser):
         # in case the user passes in their own configargparse instance instead of calling getDefaultArgumentParser()
         # this forces configargparser to process the config file in YAML rather than in it's own format
         parser._config_file_parser = YAMLConfigFileParser()  # type: ignore[misc]
-        parser._default_config_files = [DEFAULT_CONFIG_FILE]  # type: ignore[misc]
+        parser._default_config_files = [config_path]  # type: ignore[misc]
     else:
         # configargparse advertises itself as a drag and drop replacement, and running the normal argparse ArgumentParser
         # through this code still seems to work (with the exception of --config and environmental variables)
@@ -723,24 +745,24 @@ def addOptions(
             DeprecationWarning,
         )
 
-    check_and_create_default_config_file()
+    ensure_config(config_path)
     # Check on the config file to make sure it is sensible
-    config_status = os.stat(DEFAULT_CONFIG_FILE)
+    config_status = os.stat(config_path)
     if config_status.st_size == 0:
         # If we have an empty config file, someone has to manually delete
         # it before we will work again.
         raise RuntimeError(
-            f"Config file {DEFAULT_CONFIG_FILE} exists but is empty. Delete it! Stat says: {config_status}"
+            f"Config file {config_path} exists but is empty. Delete it! Stat says: {config_status}"
         )
     try:
-        with open(DEFAULT_CONFIG_FILE) as f:
+        with open(config_path) as f:
             yaml = YAML(typ="safe")
             s = yaml.load(f)
             logger.debug("Initialized default configuration: %s", json.dumps(s))
     except:
         # Something went wrong reading the default config, so dump its
         # contents to the log.
-        logger.info("Configuration file contents: %s", open(DEFAULT_CONFIG_FILE).read())
+        logger.info("Configuration file contents: %s", open(config_path).read())
         raise
 
     # Add base toil options
@@ -902,8 +924,9 @@ class Toil(ContextManager["Toil"]):
     _jobStore: "AbstractJobStore"
     _batchSystem: "AbstractBatchSystem"
     _provisioner: Optional["AbstractProvisioner"]
+    _start_time: float
 
-    def __init__(self, options: Namespace) -> None:
+    def __init__(self, options: Namespace, workflow_name: Optional[str] = None, trs_spec: Optional[str] = None) -> None:
         """
         Initialize a Toil object from the given options.
 
@@ -911,6 +934,12 @@ class Toil(ContextManager["Toil"]):
         done when the context is entered.
 
         :param options: command line options specified by the user
+        :param workflow_name: A human-readable name (probably a filename, URL,
+            or TRS specifier) for the workflow being run. Used for Toil history
+            storage.
+        :param trs_spec: A TRS id:version string for the workflow being run, if
+            any. Used for Toil history storage and publishing workflow
+            execution metrics to Dockstore.
         """
         super().__init__()
         self.options = options
@@ -918,6 +947,17 @@ class Toil(ContextManager["Toil"]):
         self._inContextManager: bool = False
         self._inRestart: bool = False
 
+        if workflow_name is None:
+            # Try to use the entrypoint file.
+            import __main__
+            if hasattr(__main__, '__file__'):
+                workflow_name = __main__.__file__
+        if workflow_name is None:
+            # If there's no file, say this is an interactive usage of Toil.
+            workflow_name = "<interactive>"
+        self._workflow_name: str = workflow_name
+        self._trs_spec = trs_spec
+
     def __enter__(self) -> "Toil":
         """
         Derive configuration from the command line options.
@@ -937,9 +977,16 @@ class Toil(ContextManager["Toil"]):
             # Set the caching option because it wasn't set originally, resuming jobstore rebuilds config from CLI options
             self.options.caching = config.caching
 
+        if self._trs_spec and config.publish_workflow_metrics is None:
+            # We could potentially publish this workflow run. Get a call from the user.
+            config.publish_workflow_metrics = ask_user_about_publishing_metrics()
+
         if not config.restart:
             config.prepare_start()
             jobStore.initialize(config)
+            assert config.workflowID is not None
+            # Record that there is a workflow beign run
+            HistoryManager.record_workflow_creation(config.workflowID, self.canonical_locator(config.jobStore))
         else:
             jobStore.resume()
             # Merge configuration from job store with command line options
@@ -949,6 +996,7 @@ class Toil(ContextManager["Toil"]):
             jobStore.write_config()
         self.config = config
         self._jobStore = jobStore
+        self._start_time = time.time()
         self._inContextManager = True
 
         # This will make sure `self.__exit__()` is called when we get a SIGTERM signal.
@@ -968,6 +1016,50 @@ class Toil(ContextManager["Toil"]):
         Depending on the configuration, delete the job store.
         """
         try:
+            if self.config.workflowID is not None:
+                # Record that this attempt to run the workflow succeeded or failed.
+                # TODO: Get ahold of the timing from statsAndLogging instead of redoing it here!
+                # To record the batch system, we need to avoid capturing typos/random text the user types instead of a real batch system.
+                batch_system_type="<Not Initialized>"
+                if hasattr(self, "_batchSystem"):
+                    batch_system_type = type(self._batchSystem).__module__ + "." + type(self._batchSystem).__qualname__
+                HistoryManager.record_workflow_attempt(
+                    self.config.workflowID,
+                    self.config.workflowAttemptNumber,
+                    exc_type is None,
+                    self._start_time,
+                    time.time() - self._start_time,
+                    batch_system=batch_system_type,
+                    caching=self.config.caching,
+                    # Use the git-hash-free Toil version which should not be unique
+                    toil_version=baseVersion,
+                    # This should always be major.minor.patch.
+                    python_version=platform.python_version(),
+                    platform_system=platform.system(),
+                    platform_machine=platform.machine()
+                )
+
+            if self.config.publish_workflow_metrics == "all":
+                # Publish metrics for all workflows, including previous ones.
+                submission = create_history_submission()
+                while not submission.empty():
+                    if not submission.submit():
+                        # Submitting this batch failed. An item might be broken
+                        # and we don't want to get stuck making no progress on
+                        # a batch of stuff that can't really be submitted.
+                        break
+                    # Keep making submissions until we've uploaded the whole
+                    # history or something goes wrong.
+                    submission = create_history_submission()
+
+            elif self.config.publish_workflow_metrics == "current" and self.config.workflowID is not None:
+                # Publish metrics for this run only. Might be empty if we had no TRS ID.
+                create_current_submission(self.config.workflowID, self.config.workflowAttemptNumber).submit()
+
+            # Make sure the history doesn't stay too big
+            HistoryManager.enforce_byte_size_limit()
+
+
             if (
                 exc_type is not None
                 and self.config.clean == "onError"
@@ -1012,6 +1104,9 @@ class Toil(ContextManager["Toil"]):
         """
         self._assertContextManagerUsed()
 
+        assert self.config.workflowID is not None
+        HistoryManager.record_workflow_metadata(self.config.workflowID, self._workflow_name, self._trs_spec)
+
         from toil.job import Job
 
         # Check that the rootJob is an instance of the Job class
@@ -1110,6 +1205,8 @@ class Toil(ContextManager["Toil"]):
             )
             self._provisioner.setAutoscaledNodeTypes(self.config.nodeTypes)
 
+    JOB_STORE_TYPES = ["file", "aws", "google"]
+
     @classmethod
     def getJobStore(cls, locator: str) -> "AbstractJobStore":
         """
@@ -1137,6 +1234,14 @@ class Toil(ContextManager["Toil"]):
 
     @staticmethod
     def parseLocator(locator: str) -> tuple[str, str]:
+        """
+        Parse a job store locator to a type string and the data needed for that
+        implementation to connect to it.
+
+        Does not validate the set of possible job store types.
+
+        :raises RuntimeError: if the locator is not in the approproate syntax.
+        """
         if locator[0] in "/." or ":" not in locator:
             return "file", locator
         else:
@@ -1153,6 +1258,17 @@ class Toil(ContextManager["Toil"]):
             raise ValueError(f"Can't have a ':' in the name: '{name}'.")
         return f"{name}:{rest}"
 
+    @classmethod
+    def canonical_locator(cls, locator: str) -> str:
+        """
+        Turn a job store locator into one that will work from any directory and
+        always includes the explicit type of job store.
+        """
+        job_store_type, rest = cls.parseLocator(locator)
+        if job_store_type == "file":
+            rest = os.path.abspath(rest)
+        return cls.buildLocator(job_store_type, rest)
+
     @classmethod
     def resumeJobStore(cls, locator: str) -> "AbstractJobStore":
         jobStore = cls.getJobStore(locator)

toil/cwl/cwltoil.py

@@ -111,7 +111,7 @@ from toil.batchSystems.abstractBatchSystem import InsufficientSystemResources
 from toil.batchSystems.registry import DEFAULT_BATCH_SYSTEM
 from toil.common import Config, Toil, addOptions
 from toil.cwl import check_cwltool_version
-from toil.lib.integration import resolve_workflow
+from toil.lib.trs import resolve_workflow
 from toil.lib.misc import call_command
 from toil.provisioners.clusterScaler import JobTooBigError
 
@@ -1214,7 +1214,7 @@ def toil_make_tool(
     return cwltool.workflow.default_make_tool(toolpath_object, loadingContext)
 
 
-# When a file we want to have is missing, we can give it this sentinal location
+# When a file we want to have is missing, we can give it this sentinel location
 # URI instead of raising an error right away, in case it is optional.
 MISSING_FILE = "missing://"
 
@@ -1812,6 +1812,9 @@ def convert_file_uri_to_toil_uri(
     # with unsupportedRequirement when retrieving later with getFile
     elif file_uri.startswith("_:"):
         return file_uri
+    elif file_uri.startswith(MISSING_FILE):
+        # We cannot import a missing file
+        raise FileNotFoundError(f"Could not find {file_uri[len(MISSING_FILE):]}")
     else:
         file_uri = existing.get(file_uri, file_uri)
         if file_uri not in index:
@@ -1876,7 +1879,7 @@ def extract_file_uri_once(
     ):
         if mark_broken:
             logger.debug("File %s is missing", file_metadata)
-            file_metadata["location"] = location = MISSING_FILE
+            file_metadata["location"] = location = MISSING_FILE + location
         else:
             raise cwl_utils.errors.WorkflowException(
                 "File is missing: %s" % file_metadata
@@ -3599,6 +3602,7 @@ class CWLInstallImportsJob(Job):
         """
         Given a mapping of filenames to Toil file IDs, replace the filename with the file IDs throughout the CWL object.
         """
+
         def fill_in_file(filename: str) -> FileID:
             """
             Return the file name's associated Toil file ID
@@ -3954,10 +3958,10 @@ def filtered_secondary_files(
                 sf,
             )
     # remove secondary files that are not present in the filestore or pointing
-    # to existant things on disk
+    # to existent things on disk
     for sf in intermediate_secondary_files:
         sf_loc = cast(str, sf.get("location", ""))
-        if sf_loc != MISSING_FILE or sf.get("class", "") == "Directory":
+        if not sf_loc.startswith(MISSING_FILE) or sf.get("class", "") == "Directory":
             # Pass imported files, and all Directories
             final_secondary_files.append(sf)
         else:
@@ -4166,15 +4170,15 @@ def get_options(args: list[str]) -> Namespace:
         description=textwrap.dedent(
             """
             positional arguments:
-              
+
               WORKFLOW              CWL file to run.
 
               INFILE                YAML or JSON file of workflow inputs.
-              
+
               WF_OPTIONS            Additional inputs to the workflow as command-line
                                     flags. If CWL workflow takes an input, the name of the
                                     input can be used as an option. For example:
-                                    
+
                                       %(prog)s workflow.cwl --file1 file
 
                                     If an input has the same name as a Toil option, pass
@@ -4261,8 +4265,18 @@ def main(args: Optional[list[str]] = None, stdout: TextIO = sys.stdout) -> int:
     runtime_context.move_outputs = "leave"
     runtime_context.rm_tmpdir = False
     runtime_context.streaming_allowed = not options.disable_streaming
+    if options.cachedir is not None:
+        runtime_context.cachedir = os.path.abspath(options.cachedir)
+        # Automatically bypass the file store to be compatible with cwltool caching
+        # Otherwise, the CWL caching code makes links to temporary local copies
+        # of filestore files and caches those.
+        logger.debug("CWL task caching is turned on. Bypassing file store.")
+        options.bypass_file_store = True
     if options.mpi_config_file is not None:
         runtime_context.mpi_config = MpiConfig.load(options.mpi_config_file)
+    if cwltool.main.check_working_directories(runtime_context) is not None:
+        logger.error("Failed to create directory. If using tmpdir_prefix, tmpdir_outdir_prefix, or cachedir, consider changing directory locations.")
+        return 1
     setattr(runtime_context, "bypass_file_store", options.bypass_file_store)
     if options.bypass_file_store and options.destBucket:
         # We use the file store to write to buckets, so we can't do this (yet?)
@@ -4293,6 +4307,10 @@ def main(args: Optional[list[str]] = None, stdout: TextIO = sys.stdout) -> int:
 
     try:
 
+        # We might have workflow metadata to pass to Toil
+        workflow_name=None
+        trs_spec = None
+
         if not options.restart:
             # Make a version of the config based on the initial options, for
             # setting up CWL option stuff
@@ -4302,7 +4320,9 @@ def main(args: Optional[list[str]] = None, stdout: TextIO = sys.stdout) -> int:
             # Before showing the options to any cwltool stuff that wants to
             # load the workflow, transform options.cwltool, where our
             # argument for what to run is, to handle Dockstore workflows.
-            options.cwltool = resolve_workflow(options.cwltool)
+            options.cwltool, trs_spec = resolve_workflow(options.cwltool)
+            # Figure out what to call the workflow
+            workflow_name = trs_spec or options.cwltool
 
             # TODO: why are we doing this? Does this get applied to all
             # tools as a default or something?
@@ -4474,7 +4494,7 @@ def main(args: Optional[list[str]] = None, stdout: TextIO = sys.stdout) -> int:
             logger.debug("Root tool: %s", tool)
             tool = remove_pickle_problems(tool)
 
-        with Toil(options) as toil:
+        with Toil(options, workflow_name=workflow_name, trs_spec=trs_spec) as toil:
             if options.restart:
                 outobj = toil.restart()
             else:
@@ -4575,6 +4595,7 @@ def main(args: Optional[list[str]] = None, stdout: TextIO = sys.stdout) -> int:
         InvalidImportExportUrlException,
         UnimplementedURLException,
         JobTooBigError,
+        FileNotFoundError
     ) as err:
         logging.error(err)
         return 1

toil/job.py

@@ -3140,46 +3140,55 @@ class Job:
             startTime = time.time()
             startClock = ResourceMonitor.get_total_cpu_time()
         baseDir = os.getcwd()
+        
+        succeeded = False
+        try:
+            yield
+
+            if "download_only" in self._debug_flags:
+                # We should stop right away
+                logger.debug("Job did not stop itself after downloading files; stopping.")
+                raise DebugStoppingPointReached()
+
+            # If the job is not a checkpoint job, add the promise files to delete
+            # to the list of jobStoreFileIDs to delete
+            # TODO: why is Promise holding a global list here???
+            if not self.checkpoint:
+                for jobStoreFileID in Promise.filesToDelete:
+                    # Make sure to wrap the job store ID in a FileID object so the file store will accept it
+                    # TODO: talk directly to the job store here instead.
+                    fileStore.deleteGlobalFile(FileID(jobStoreFileID, 0))
+            else:
+                # Else copy them to the job description to delete later
+                self.description.checkpointFilesToDelete = list(Promise.filesToDelete)
+            Promise.filesToDelete.clear()
+            # Now indicate the asynchronous update of the job can happen
+            fileStore.startCommit(jobState=True)
 
-        yield
-
-        if "download_only" in self._debug_flags:
-            # We should stop right away
-            logger.debug("Job did not stop itself after downloading files; stopping.")
-            raise DebugStoppingPointReached()
-
-        # If the job is not a checkpoint job, add the promise files to delete
-        # to the list of jobStoreFileIDs to delete
-        # TODO: why is Promise holding a global list here???
-        if not self.checkpoint:
-            for jobStoreFileID in Promise.filesToDelete:
-                # Make sure to wrap the job store ID in a FileID object so the file store will accept it
-                # TODO: talk directly to the job store here instead.
-                fileStore.deleteGlobalFile(FileID(jobStoreFileID, 0))
-        else:
-            # Else copy them to the job description to delete later
-            self.description.checkpointFilesToDelete = list(Promise.filesToDelete)
-        Promise.filesToDelete.clear()
-        # Now indicate the asynchronous update of the job can happen
-        fileStore.startCommit(jobState=True)
-        # Change dir back to cwd dir, if changed by job (this is a safety issue)
-        if os.getcwd() != baseDir:
-            os.chdir(baseDir)
-        # Finish up the stats
-        if stats is not None:
-            totalCpuTime, totalMemoryUsage = (
-                ResourceMonitor.get_total_cpu_time_and_memory_usage()
-            )
-            stats.jobs.append(
-                Expando(
-                    time=str(time.time() - startTime),
-                    clock=str(totalCpuTime - startClock),
-                    class_name=self._jobName(),
-                    memory=str(totalMemoryUsage),
-                    requested_cores=str(self.cores),
-                    disk=str(fileStore.get_disk_usage()),
+            succeeded = True
+        finally:
+            # Change dir back to cwd dir, if changed by job (this is a safety issue)
+            if os.getcwd() != baseDir:
+                os.chdir(baseDir)
+            # Finish up the stats
+            if stats is not None:
+                totalCpuTime, total_memory_kib = (
+                    ResourceMonitor.get_total_cpu_time_and_memory_usage()
+                )
+                stats.jobs.append(
+                    # TODO: We represent everything as strings in the stats
+                    # even though the JSON transport can take bools and floats.
+                    Expando(
+                        start=str(startTime),
+                        time=str(time.time() - startTime),
+                        clock=str(totalCpuTime - startClock),
+                        class_name=self._jobName(),
+                        memory=str(total_memory_kib),
+                        requested_cores=str(self.cores), # TODO: Isn't this really consumed cores?
+                        disk=str(fileStore.get_disk_usage()),
+                        succeeded=str(succeeded),
+                    )
                 )
-            )
 
     def _runner(
         self,