toil 6.1.0a1__py3-none-any.whl → 7.0.0__py3-none-any.whl

toil/jobStores/abstractJobStore.py

@@ -70,6 +70,14 @@ except ImportError:
     class ProxyConnectionError(BaseException):  # type: ignore
         """Dummy class."""
 
+class LocatorException(Exception):
+    """
+    Base exception class for all locator exceptions.
+    For example, job store/aws bucket exceptions where they already exist
+    """
+    def __init__(self, error_msg: str, locator: str, prefix: Optional[str]=None):
+        full_locator = locator if prefix is None else f"{prefix}:{locator}"
+        super().__init__(error_msg % full_locator)
 
 class InvalidImportExportUrlException(Exception):
     def __init__(self, url: ParseResult) -> None:
@@ -136,24 +144,24 @@ class NoSuchFileException(Exception):
         super().__init__(message)
 
 
-class NoSuchJobStoreException(Exception):
+class NoSuchJobStoreException(LocatorException):
     """Indicates that the specified job store does not exist."""
-    def __init__(self, locator: str):
+    def __init__(self, locator: str, prefix: str):
         """
         :param str locator: The location of the job store
         """
-        super().__init__("The job store '%s' does not exist, so there is nothing to restart." % locator)
+        super().__init__("The job store '%s' does not exist, so there is nothing to restart.", locator, prefix)
 
 
-class JobStoreExistsException(Exception):
+class JobStoreExistsException(LocatorException):
     """Indicates that the specified job store already exists."""
-    def __init__(self, locator: str):
+    def __init__(self, locator: str, prefix: str):
         """
         :param str locator: The location of the job store
         """
         super().__init__(
             "The job store '%s' already exists. Use --restart to resume the workflow, or remove "
-            "the job store with 'toil clean' to start the workflow from scratch." % locator)
+            "the job store with 'toil clean' to start the workflow from scratch.", locator, prefix)
 
 
 class AbstractJobStore(ABC):
@@ -835,16 +843,17 @@ class AbstractJobStore(ABC):
             root_job_description = self.load_root_job()
             reachable_from_root: Set[str] = set()
 
-            # Add first root job outside of the loop below.
-            reachable_from_root.add(str(root_job_description.jobStoreID))
-            # add all of root's linked service jobs as well
-            for service_jobstore_id in root_job_description.services:
-                if haveJob(service_jobstore_id):
-                    reachable_from_root.add(service_jobstore_id)
-            for merged_jobstore_id in root_job_description.merged_jobs:
+
+            for merged_in in root_job_description.get_chain():
+                # Add the job itself and any other jobs that chained with it.
                 # Keep merged-in jobs around themselves, but don't bother
                 # exploring them, since we took their successors.
-                reachable_from_root.add(merged_jobstore_id)
+                reachable_from_root.add(merged_in.job_store_id)
+            # add all of root's linked service jobs as well
+            for service_job_store_id in root_job_description.services:
+                if haveJob(service_job_store_id):
+                    reachable_from_root.add(service_job_store_id)
+
 
             # Unprocessed means it might have successor jobs we need to add.
             unprocessed_job_descriptions = [root_job_description]
@@ -852,24 +861,21 @@ class AbstractJobStore(ABC):
             while unprocessed_job_descriptions:
                 new_job_descriptions_to_process = []  # Reset.
                 for job_description in unprocessed_job_descriptions:
-                    for successor_jobstore_id in job_description.allSuccessors():
-                        if successor_jobstore_id not in reachable_from_root and haveJob(successor_jobstore_id):
-                            successor_job_description = getJobDescription(successor_jobstore_id)
-
-                            # Add each successor job.
-                            reachable_from_root.add(
-                                str(successor_job_description.jobStoreID)
-                            )
+                    for merged_in in job_description.get_chain():
+                        # Add the job and anything chained with it.
+                        # Keep merged-in jobs around themselves, but don't bother
+                        # exploring them, since we took their successors.
+                        reachable_from_root.add(merged_in.job_store_id)
+                    for successor_job_store_id in job_description.allSuccessors():
+                        if successor_job_store_id not in reachable_from_root and haveJob(successor_job_store_id):
+                            successor_job_description = getJobDescription(successor_job_store_id)
+
                             # Add all of the successor's linked service jobs as well.
-                            for service_jobstore_id in successor_job_description.services:
-                                if haveJob(service_jobstore_id):
-                                    reachable_from_root.add(service_jobstore_id)
+                            for service_job_store_id in successor_job_description.services:
+                                if haveJob(service_job_store_id):
+                                    reachable_from_root.add(service_job_store_id)
 
                             new_job_descriptions_to_process.append(successor_job_description)
-                    for merged_jobstore_id in job_description.merged_jobs:
-                        # Keep merged-in jobs around themselves, but don't bother
-                        # exploring them, since we took their successors.
-                        reachable_from_root.add(merged_jobstore_id)
                 unprocessed_job_descriptions = new_job_descriptions_to_process
 
             logger.debug(f"{len(reachable_from_root)} jobs reachable from root.")
@@ -926,11 +932,11 @@ class AbstractJobStore(ABC):
                 jobDescription.filesToDelete = []
                 changed[0] = True
 
-            # For a job whose command is already executed, remove jobs from the
+            # For a job whose body has already executed, remove jobs from the
             # stack that are already deleted. This cleans up the case that the
             # jobDescription had successors to run, but had not been updated to
             # reflect this.
-            if jobDescription.command is None:
+            if not jobDescription.has_body():
 
                 def stackSizeFn() -> int:
                     return len(list(jobDescription.allSuccessors()))