toil 6.1.0a1__py3-none-any.whl → 7.0.0__py3-none-any.whl

toil/common.py

@@ -23,6 +23,7 @@ import tempfile
 import time
 import uuid
 import warnings
+from io import StringIO
 
 from ruamel.yaml import YAML
 from ruamel.yaml.comments import CommentedMap
@@ -63,8 +64,7 @@ else:
     from typing_extensions import Literal
 
 from toil import logProcessContext, lookupEnvVar
-from toil.batchSystems.options import (add_all_batchsystem_options,
-                                       set_batchsystem_options)
+from toil.batchSystems.options import set_batchsystem_options
 from toil.bus import (ClusterDesiredSizeMessage,
                       ClusterSizeMessage,
                       JobCompletedMessage,
@@ -74,17 +74,15 @@ from toil.bus import (ClusterDesiredSizeMessage,
                       MessageBus,
                       QueueSizeMessage)
 from toil.fileStores import FileID
-from toil.lib.aws import zone_to_region, build_tag_dict_from_env
 from toil.lib.compatibility import deprecated
 from toil.lib.io import try_path, AtomicFileCreate
 from toil.lib.retry import retry
 from toil.provisioners import (add_provisioner_options,
-                               cluster_factory,
-                               parse_node_types)
+                               cluster_factory)
 from toil.realtimeLogger import RealtimeLogger
 from toil.statsAndLogging import (add_logging_options,
                                   set_logging_from_options)
-from toil.version import dockerRegistry, dockerTag, version, baseVersion
+from toil.version import dockerRegistry, dockerTag, version
 
 if TYPE_CHECKING:
     from toil.batchSystems.abstractBatchSystem import AbstractBatchSystem
@@ -124,6 +122,7 @@ class Config:
     kubernetes_owner: Optional[str]
     kubernetes_service_account: Optional[str]
     kubernetes_pod_timeout: float
+    kubernetes_privileged: bool
     tes_endpoint: str
     tes_user: str
     tes_password: str
@@ -137,6 +136,7 @@ class Config:
     """The backing scheduler will be instructed, if possible, to save logs
     to this directory, where the leader can read them."""
     statePollingWait: int
+    state_polling_timeout: int
     disableAutoDeployment: bool
 
     # Core options
@@ -148,6 +148,7 @@ class Config:
     workflowAttemptNumber: int
     jobStore: str
     logLevel: str
+    colored_logs: bool
     workDir: Optional[str]
     coordination_dir: Optional[str]
     noStdOutErr: bool
@@ -208,6 +209,7 @@ class Config:
     doubleMem: bool
     maxJobDuration: int
     rescueJobsFrequency: int
+    job_store_timeout: float
 
     # Log management
     maxLogFileSize: int
@@ -285,8 +287,6 @@ class Config:
 
             1. options object under option_name
             2. options object under old_names
-            3. environment variables in env
-            4. provided default value
 
             Selected option value is run through parsing_funtion if it is set.
             Then the parsed value is run through check_function to check it for
@@ -375,6 +375,7 @@ class Config:
         set_option("doubleMem")
         set_option("maxJobDuration")
         set_option("rescueJobsFrequency")
+        set_option("job_store_timeout")
 
         # Log management
         set_option("maxLogFileSize")
@@ -401,10 +402,17 @@ class Config:
         set_option("badWorker")
         set_option("badWorkerFailInterval")
         set_option("logLevel")
+        set_option("colored_logs")
 
-        self.check_configuration_consistency()
+        # Apply overrides as highest priority
+        # Override workDir with value of TOIL_WORKDIR_OVERRIDE if it exists
+        if os.getenv('TOIL_WORKDIR_OVERRIDE') is not None:
+            self.workDir = os.getenv('TOIL_WORKDIR_OVERRIDE')
+        # Override workDir with value of TOIL_WORKDIR_OVERRIDE if it exists
+        if os.getenv('TOIL_COORDINATION_DIR_OVERRIDE') is not None:
+            self.workDir = os.getenv('TOIL_COORDINATION_DIR_OVERRIDE')
 
-        logger.debug("Loaded configuration: %s", vars(options))
+        self.check_configuration_consistency()
 
     def check_configuration_consistency(self) -> None:
         """Old checks that cannot be fit into an action class for argparse"""
@@ -580,19 +588,23 @@ def generate_config(filepath: str) -> None:
     with AtomicFileCreate(filepath) as temp_path:
         with open(temp_path, "w") as f:
             f.write("config_version: 1.0\n")
-            yaml = YAML(typ=['rt', 'string'])
+            yaml = YAML(typ='rt')
             for data in all_data:
                 if "config_version" in data:
                     del data["config_version"]
-                for line in yaml.dump_to_string(data).split("\n"):  # type: ignore[attr-defined]
-                    if line:
-                        f.write("#")
-                    f.write(line)
-                    f.write("\n")
+                with StringIO() as data_string:
+                    yaml.dump(data, data_string)
+                    for line in data_string.readline():
+                        if line:
+                            f.write("#")
+                        f.write(f"{line}\n")
 
 
 def parser_with_common_options(
-        provisioner_options: bool = False, jobstore_option: bool = True, prog: Optional[str] = None
+    provisioner_options: bool = False,
+    jobstore_option: bool = True,
+    prog: Optional[str] = None,
+    default_log_level: Optional[int] = None
 ) -> ArgParser:
     parser = ArgParser(prog=prog or "Toil", formatter_class=ArgumentDefaultsHelpFormatter)
 
@@ -603,7 +615,7 @@ def parser_with_common_options(
         parser.add_argument('jobStore', type=str, help=JOBSTORE_HELP)
 
     # always add these
-    add_logging_options(parser)
+    add_logging_options(parser, default_log_level)
     parser.add_argument("--version", action='version', version=version)
     parser.add_argument("--tempDirRoot", dest="tempDirRoot", type=str, default=tempfile.gettempdir(),
                         help="Path to where temporary directory containing all temp files are created, "
@@ -701,7 +713,7 @@ def addOptions(parser: ArgumentParser, jobstore_as_flag: bool = False, cwl: bool
                             help="WDL document URI")
         parser.add_argument("inputs_uri", type=str, nargs='?',
                             help="WDL input JSON URI")
-        parser.add_argument("--input", "-i", dest="inputs_uri", type=str,
+        parser.add_argument("--input", "--inputs", "-i", dest="inputs_uri", type=str,
                             help="WDL input JSON URI")
         check_arguments(typ="wdl")
 
@@ -709,7 +721,7 @@ def addOptions(parser: ArgumentParser, jobstore_as_flag: bool = False, cwl: bool
 @lru_cache(maxsize=None)
 def getNodeID() -> str:
     """
-    Return unique ID of the current node (host). The resulting string will be convertable to a uuid.UUID.
+    Return unique ID of the current node (host). The resulting string will be convertible to a uuid.UUID.
 
     Tries several methods until success. The returned ID should be identical across calls from different processes on
     the same node at least until the next OS reboot.
@@ -757,7 +769,7 @@ def getNodeID() -> str:
                        "experience cryptic job failures")
     if len(nodeID.replace('-', '')) < UUID_LENGTH:
         # Some platforms (Mac) give us not enough actual hex characters.
-        # Repeat them so the result is convertable to a uuid.UUID
+        # Repeat them so the result is convertible to a uuid.UUID
         nodeID = nodeID.replace('-', '')
         num_repeats = UUID_LENGTH // len(nodeID) + 1
         nodeID = nodeID * num_repeats
@@ -801,6 +813,7 @@ class Toil(ContextManager["Toil"]):
         set_logging_from_options(self.options)
         config = Config()
         config.setOptions(self.options)
+        logger.debug("Loaded configuration: %s", vars(self.options))
         if config.jobStore is None:
             raise RuntimeError("No jobstore provided!")
         jobStore = self.getJobStore(config.jobStore)
@@ -875,6 +888,16 @@ class Toil(ContextManager["Toil"]):
         """
         self._assertContextManagerUsed()
 
+        from toil.job import Job
+
+        # Check that the rootJob is an instance of the Job class
+        if not isinstance(rootJob, Job):
+            raise RuntimeError("The type of the root job is not a job.")
+
+        # Check that the rootJob has been initialized
+        rootJob.check_initialized()
+
+
         # Write shared files to the job store
         self._jobStore.write_leader_pid()
         self._jobStore.write_leader_node_id()
@@ -1255,18 +1278,14 @@ class Toil(ContextManager["Toil"]):
                --workDir flag
         :param config_coordination_dir: Value passed to the program using the
                --coordinationDir flag
+        :param workflow_id: Used if a tmpdir_prefix exists to create full
+               directory paths unique per workflow
 
         :return: Path to the Toil coordination directory. Ought to be on a
                  POSIX filesystem that allows directories containing open files to be
                  deleted.
         """
 
-        if 'XDG_RUNTIME_DIR' in os.environ and not os.path.exists(os.environ['XDG_RUNTIME_DIR']):
-            # Slurm has been observed providing this variable but not keeping
-            # the directory live as long as we run for.
-            logger.warning('XDG_RUNTIME_DIR is set to nonexistent directory %s; your environment may be out of spec!',
-                           os.environ['XDG_RUNTIME_DIR'])
-
         # Go get a coordination directory, using a lot of short-circuiting of
         # or and the fact that and returns its second argument when it
         # succeeds.
@@ -1289,6 +1308,9 @@ class Toil(ContextManager["Toil"]):
                     os.path.join(os.environ['XDG_RUNTIME_DIR'], 'toil'))) or
                 # Try under /run/lock. It might be a temp dir style sticky directory.
                 try_path('/run/lock') or
+                # Try all possible temp directories, falling back to the current working
+                # directory
+                tempfile.gettempdir() or
                 # Finally, fall back on the work dir and hope it's a legit filesystem.
                 cls.getToilWorkDir(config_work_dir)
         )
@@ -1299,7 +1321,7 @@ class Toil(ContextManager["Toil"]):
         return coordination_dir
 
     @staticmethod
-    def _get_workflow_path_component(workflow_id: str) -> str:
+    def get_workflow_path_component(workflow_id: str) -> str:
         """
         Get a safe filesystem path component for a workflow.
 
@@ -1308,7 +1330,7 @@ class Toil(ContextManager["Toil"]):
 
         :param workflow_id: The ID of the current Toil workflow.
         """
-        return str(uuid.uuid5(uuid.UUID(getNodeID()), workflow_id)).replace('-', '')
+        return "toilwf-" + str(uuid.uuid5(uuid.UUID(getNodeID()), workflow_id)).replace('-', '')
 
     @classmethod
     def getLocalWorkflowDir(
@@ -1325,7 +1347,7 @@ class Toil(ContextManager["Toil"]):
 
         # Create a directory unique to each host in case workDir is on a shared FS.
         # This prevents workers on different nodes from erasing each other's directories.
-        workflowDir: str = os.path.join(base, cls._get_workflow_path_component(workflowID))
+        workflowDir: str = os.path.join(base, cls.get_workflow_path_component(workflowID))
         try:
             # Directory creation is atomic
             os.mkdir(workflowDir)
@@ -1367,7 +1389,8 @@ class Toil(ContextManager["Toil"]):
         base = cls.get_toil_coordination_dir(config_work_dir, config_coordination_dir)
 
         # Make a per-workflow and node subdirectory
-        subdir = os.path.join(base, cls._get_workflow_path_component(workflow_id))
+        subdir = os.path.join(base, cls.get_workflow_path_component(workflow_id))
+
         # Make it exist
         os.makedirs(subdir, exist_ok=True)
         # TODO: May interfere with workflow directory creation logging if it's the same directory.
@@ -1425,6 +1448,8 @@ class ToilMetrics:
             clusterName = str(provisioner.clusterName)
             if provisioner._zone is not None:
                 if provisioner.cloud == 'aws':
+                    # lazy import to avoid AWS dependency if the aws extra is not installed
+                    from toil.lib.aws import zone_to_region
                     # Remove AZ name
                     region = zone_to_region(provisioner._zone)
                 else: