toil 5.12.0__py3-none-any.whl → 6.1.0a1__py3-none-any.whl

toil/jobStores/fileJobStore.py

@@ -20,11 +20,10 @@ import re
 import shutil
 import stat
 import sys
-import tempfile
 import time
 import uuid
 from contextlib import contextmanager
-from typing import IO, Iterator, List, Optional, Union, overload
+from typing import IO, Iterable, Iterator, List, Optional, Union, overload
 from urllib.parse import ParseResult, quote, unquote
 
 if sys.version_info >= (3, 8):
@@ -42,6 +41,7 @@ from toil.jobStores.abstractJobStore import (AbstractJobStore,
 from toil.lib.io import (AtomicFileCreate,
                          atomic_copy,
                          atomic_copyobj,
+                         mkdtemp,
                          robust_rmtree)
 
 logger = logging.getLogger(__name__)
@@ -121,8 +121,8 @@ class FileJobStore(AbstractJobStore):
         os.makedirs(self.filesDir, exist_ok=True)
         os.makedirs(self.jobFilesDir, exist_ok=True)
         os.makedirs(self.sharedFilesDir, exist_ok=True)
-        self.linkImports = config.linkImports
-        self.moveExports = config.moveExports
+        self.linkImports = config.symlinkImports
+        self.moveExports = config.moveOutputs
         super().initialize(config)
 
     def resume(self):
@@ -147,8 +147,8 @@ class FileJobStore(AbstractJobStore):
 
         # Make a unique temp directory under a directory for this job name,
         # possibly sprayed across multiple levels of subdirectories.
-        absJobDir = tempfile.mkdtemp(prefix=self.JOB_DIR_PREFIX,
-                                     dir=self._get_arbitrary_jobs_dir_for_name(usefulFilename))
+        absJobDir = mkdtemp(prefix=self.JOB_DIR_PREFIX,
+                            dir=self._get_arbitrary_jobs_dir_for_name(usefulFilename))
 
         job_description.jobStoreID = self._get_job_id_from_dir(absJobDir)
 
@@ -252,18 +252,19 @@ class FileJobStore(AbstractJobStore):
 
         job.pre_update_hook()
 
+        dest_filename = self._get_job_file_name(job.jobStoreID)
+
         # The job is serialised to a file suffixed by ".new"
         # We insist on creating the file; an existing .new file indicates
         # multiple simultaneous attempts to update the job, which will lose
         # updates.
         # The file is then moved to its correct path.
-        # Atomicity guarantees use the fact the underlying file systems "move"
+        # Atomicity guarantees use the fact the underlying file system's "move"
         # function is atomic.
-        with open(self._get_job_file_name(job.jobStoreID) + ".new", 'xb') as f:
+        with open(dest_filename + ".new", 'xb') as f:
             pickle.dump(job, f)
         # This should be atomic for the file system
-        os.rename(self._get_job_file_name(job.jobStoreID) + ".new", self._get_job_file_name(job.jobStoreID))
-
+        os.rename(dest_filename + ".new", dest_filename)
     def delete_job(self, job_id):
         # The jobStoreID is the relative path to the directory containing the job,
         # removing this directory deletes the job.
@@ -316,13 +317,14 @@ class FileJobStore(AbstractJobStore):
         # symlink argument says whether the caller can take symlinks or not
         # ex: if false, it implies the workflow cannot work with symlinks and thus will hardlink imports
         # default is true since symlinking everything is ideal
+        uri_path = unquote(uri.path)
         if issubclass(otherCls, FileJobStore):
-            if os.path.isdir(uri.path):
+            if os.path.isdir(uri_path):
                 # Don't allow directories (unless someone is racing us)
                 raise IsADirectoryError(f"URI {uri} points to a directory but a file was expected")
             if shared_file_name is None:
-                executable = os.stat(uri.path).st_mode & stat.S_IXUSR != 0
-                absPath = self._get_unique_file_path(uri.path)  # use this to get a valid path to write to in job store
+                executable = os.stat(uri_path).st_mode & stat.S_IXUSR != 0
+                absPath = self._get_unique_file_path(uri_path)  # use this to get a valid path to write to in job store
                 with self.optional_hard_copy(hardlink):
                     self._copy_or_link(uri, absPath, symlink=symlink)
                 # TODO: os.stat(absPath).st_size consistently gives values lower than
@@ -341,6 +343,9 @@ class FileJobStore(AbstractJobStore):
         if issubclass(otherCls, FileJobStore):
             srcPath = self._get_file_path_from_id(file_id)
             destPath = self._extract_path_from_url(uri)
+            # Make sure we don't need to worry about directories when exporting
+            # to local files, just like for cloud storage.
+            os.makedirs(os.path.dirname(destPath), exist_ok=True)
             executable = getattr(file_id, 'executable', False)
             if self.moveExports:
                 self._move_and_linkback(srcPath, destPath, executable=executable)
@@ -357,7 +362,11 @@ class FileJobStore(AbstractJobStore):
             os.chmod(destPath, os.stat(destPath).st_mode | stat.S_IXUSR)
 
     @classmethod
-    def get_size(cls, url):
+    def _url_exists(cls, url: ParseResult) -> bool:
+        return os.path.exists(cls._extract_path_from_url(url))
+
+    @classmethod
+    def _get_size(cls, url):
         return os.stat(cls._extract_path_from_url(url)).st_size
 
     @classmethod
@@ -371,12 +380,18 @@ class FileJobStore(AbstractJobStore):
         """
 
         # we use a ~10Mb buffer to improve speed
-        with open(cls._extract_path_from_url(url), 'rb') as readable:
+        with cls._open_url(url) as readable:
             shutil.copyfileobj(readable, writable, length=cls.BUFFER_SIZE)
             # Return the number of bytes we read when we reached EOF.
             executable = os.stat(readable.name).st_mode & stat.S_IXUSR
             return readable.tell(), executable
 
+    @classmethod
+    def _open_url(cls, url: ParseResult) -> IO[bytes]:
+        """
+        Open a file URL as a binary stream.
+        """
+        return open(cls._extract_path_from_url(url), 'rb')
 
     @classmethod
     def _write_to_url(cls, readable, url, executable=False):
@@ -484,7 +499,7 @@ class FileJobStore(AbstractJobStore):
 
         atomic_copy(local_path, jobStoreFilePath)
 
-    def read_file(self, file_id, local_path, symlink=False):
+    def read_file(self, file_id: str, local_path: str, symlink: bool = False) -> None:
         self._check_job_store_file_id(file_id)
         jobStoreFilePath = self._get_file_path_from_id(file_id)
         localDirPath = os.path.dirname(local_path)
@@ -701,6 +716,69 @@ class FileJobStore(AbstractJobStore):
             else:
                 raise
 
+    def list_all_file_names(self, for_job: Optional[str] = None) -> Iterable[str]:
+        """
+        Get all the file names (not file IDs) of files stored in the job store.
+
+        Used for debugging.
+
+        :param for_job: If set, restrict the list to files for a particular job.
+        """
+
+        # TODO: Promote to AbstractJobStore.
+        # TODO: Include stats-and-logging files?
+
+        if for_job is not None:
+            # Run on one job
+            jobs = [for_job]
+        else:
+            # Run on all the jobs
+            jobs = []
+            # But not all the jobs that exist, we want all the jobs that have
+            # files. So look at the file directories which mirror the job
+            # directories' structure.
+            for job_kind_dir in self._list_dynamic_spray_dir(self.jobFilesDir):
+                # First we sprayed all the job kinds over a tree
+                for job_instance_dir in self._list_dynamic_spray_dir(job_kind_dir):
+                    # Then we sprayed the job instances over a tree
+                    # And based on those we get the job name
+                    job_id = self._get_job_id_from_files_dir(job_instance_dir)
+                    jobs.append(job_id)
+
+            for name in os.listdir(self.sharedFilesDir):
+                # Announce all the shared files
+                yield name
+
+            for file_dir_path in self._list_dynamic_spray_dir(self.filesDir):
+                # Run on all the no-job files
+                for dir_file in os.listdir(file_dir_path):
+                    # There ought to be just one file in here.
+                    yield dir_file
+
+        for job_store_id in jobs:
+            # Files from _get_job_files_dir
+            job_files_dir = os.path.join(self.jobFilesDir, job_store_id)
+            if os.path.exists(job_files_dir):
+                for file_dir in os.listdir(job_files_dir):
+                    # Each file is in its own directory
+                    if file_dir == "cleanup":
+                        # Except the cleanup directory which we do later.
+                        continue
+                    file_dir_path = os.path.join(job_files_dir, file_dir)
+                    for dir_file in os.listdir(file_dir_path):
+                        # There ought to be just one file in here.
+                        yield dir_file
+
+                # Files from _get_job_files_cleanup_dir
+                job_cleanup_files_dir = os.path.join(job_files_dir, "cleanup")
+                if os.path.exists(job_cleanup_files_dir):
+                    for file_dir in os.listdir(job_cleanup_files_dir):
+                        # Each file is in its own directory
+                        file_dir_path = os.path.join(job_cleanup_files_dir, file_dir)
+                        for dir_file in os.listdir(file_dir_path):
+                            # There ought to be just one file in here.
+                            yield dir_file
+
     def write_logs(self, msg):
         # Temporary files are placed in the stats directory tree
         tempStatsFileName = "stats" + str(uuid.uuid4().hex) + ".new"
@@ -748,6 +826,13 @@ class FileJobStore(AbstractJobStore):
         """
         return absPath[len(self.jobsDir)+1:]
 
+    def _get_job_id_from_files_dir(self, absPath: str) -> str:
+        """
+        :param str absPath: The absolute path to a job directory under self.jobFilesDir which holds a job's files.
+        :rtype : string, string is the job ID
+        """
+        return absPath[len(self.jobFilesDir)+1:]
+
     def _get_job_file_name(self, jobStoreID):
         """
         Return the path to the file containing the serialised JobDescription instance for the given
@@ -815,7 +900,7 @@ class FileJobStore(AbstractJobStore):
         """
 
         # We just make the file IDs paths under the job store overall.
-        absPath = os.path.join(self.jobStoreDir, jobStoreFileID)
+        absPath = os.path.join(self.jobStoreDir, unquote(jobStoreFileID))
 
         # Don't validate here, we are called by the validation logic
 
@@ -828,14 +913,14 @@ class FileJobStore(AbstractJobStore):
         :rtype : string, string is the file ID.
         """
 
-        return absPath[len(self.jobStoreDir)+1:]
+        return quote(absPath[len(self.jobStoreDir)+1:])
 
     def _check_job_store_file_id(self, jobStoreFileID):
         """
         :raise NoSuchFileException: if the file with ID jobStoreFileID does
                                     not exist or is not a file
         """
-        if not self.file_exists(jobStoreFileID):
+        if not self.file_exists(unquote(jobStoreFileID)):
             raise NoSuchFileException(jobStoreFileID)
 
     def _get_arbitrary_jobs_dir_for_name(self, jobNameSlug):
@@ -966,6 +1051,19 @@ class FileJobStore(AbstractJobStore):
             # Recurse
             yield from self._walk_dynamic_spray_dir(childPath)
 
+    def _list_dynamic_spray_dir(self, root):
+        """
+        For a directory tree filled in by _getDynamicSprayDir, yields each
+        highest-level file or or directory *not* created by _getDynamicSprayDir
+        (i.e. the actual contents).
+        """
+
+        for spray_dir in self._walk_dynamic_spray_dir(root):
+            for child in os.listdir(spray_dir):
+                if child not in self.validDirsSet:
+                    # This is a real content item we are storing
+                    yield os.path.join(spray_dir, child)
+
     def _job_directories(self):
         """
         :rtype : an iterator to the temporary directories containing job

toil/jobStores/googleJobStore.py

@@ -20,23 +20,20 @@ import uuid
 from contextlib import contextmanager
 from functools import wraps
 from io import BytesIO
-from typing import List, Optional
+from typing import IO, List, Optional
 from urllib.parse import ParseResult
 
 from google.api_core.exceptions import (GoogleAPICallError,
                                         InternalServerError,
                                         ServiceUnavailable)
-from google.cloud import exceptions, storage
 from google.auth.exceptions import DefaultCredentialsError
+from google.cloud import exceptions, storage
 
 from toil.jobStores.abstractJobStore import (AbstractJobStore,
                                              JobStoreExistsException,
                                              NoSuchFileException,
                                              NoSuchJobException,
                                              NoSuchJobStoreException)
-
-from toil.fileStores import FileID
-
 from toil.jobStores.utils import ReadablePipe, WritablePipe
 from toil.lib.compatibility import compat_bytes
 from toil.lib.io import AtomicFileCreate
@@ -146,7 +143,6 @@ class GoogleJobStore(AbstractJobStore):
                 # Probably we don't have permission to use the file.
                 log.warning("File '%s' exists but didn't work to authenticate!",
                             cls.nodeServiceAccountJson)
-                pass
 
         # Either a filename is specified, or our fallback file isn't there.
         try:
@@ -394,7 +390,15 @@ class GoogleJobStore(AbstractJobStore):
         return blob
 
     @classmethod
-    def get_size(cls, url):
+    def _url_exists(cls, url: ParseResult) -> bool:
+        try:
+            cls._get_blob_from_url(url, exists=True)
+            return True
+        except NoSuchFileException:
+            return False
+
+    @classmethod
+    def _get_size(cls, url):
         return cls._get_blob_from_url(url, exists=True).size
 
     @classmethod
@@ -403,6 +407,11 @@ class GoogleJobStore(AbstractJobStore):
         blob.download_to_file(writable)
         return blob.size, False
 
+    @classmethod
+    def _open_url(cls, url: ParseResult) -> IO[bytes]:
+        blob = cls._get_blob_from_url(url, exists=True)
+        return blob.open("rb")
+
     @classmethod
     def _supports_url(cls, url, export=False):
         return url.scheme.lower() == 'gs'

toil/jobStores/utils.py

@@ -79,7 +79,7 @@ class WritablePipe(ABC):
         binary and text mode output.
 
         :param file readable: the file object representing the readable end of the pipe. Do not
-        explicitly invoke the close() method of the object, that will be done automatically.
+            explicitly invoke the close() method of the object, that will be done automatically.
         """
         raise NotImplementedError()
 
@@ -211,7 +211,7 @@ class ReadablePipe(ABC):
         binary and text mode input.
 
         :param file writable: the file object representing the writable end of the pipe. Do not
-        explicitly invoke the close() method of the object, that will be done automatically.
+            explicitly invoke the close() method of the object, that will be done automatically.
         """
         raise NotImplementedError()
 
@@ -316,7 +316,7 @@ class ReadableTransformingPipe(ReadablePipe):
         :param file readable: the input stream file object to transform.
 
         :param file writable: the file object representing the writable end of the pipe. Do not
-        explicitly invoke the close() method of the object, that will be done automatically.
+            explicitly invoke the close() method of the object, that will be done automatically.
         """
         raise NotImplementedError()
 
@@ -327,7 +327,6 @@ class JobStoreUnavailableException(RuntimeError):
     """
     Raised when a particular type of job store is requested but can't be used.
     """
-    pass
 
 def generate_locator(
     job_store_type: str,
@@ -340,9 +339,9 @@ def generate_locator(
 
     :param job_store_type: Registry name of the job store to use.
     :param local_suggestion: Path to a nonexistent local directory suitable for
-    use as a file job store.
+        use as a file job store.
     :param decoration: Extra string to add to the job store locator, if
-    convenient.
+        convenient.
 
     :return str: Job store locator for a usable job store.
     """

toil/leader.py

@@ -29,21 +29,21 @@ from toil import resolveEntryPoint
 from toil.batchSystems import DeadlockException
 from toil.batchSystems.abstractBatchSystem import (AbstractBatchSystem,
                                                    BatchJobExitReason)
-from toil.bus import (JobAnnotationMessage,
-                      JobCompletedMessage,
+from toil.bus import (JobCompletedMessage,
                       JobFailedMessage,
                       JobIssuedMessage,
                       JobMissingMessage,
                       JobUpdatedMessage,
-                      QueueSizeMessage)
-from toil.common import Config, Toil, ToilMetrics
+                      QueueSizeMessage,
+                      gen_message_bus_path)
+from toil.common import Config, ToilMetrics
 from toil.cwl.utils import CWL_UNSUPPORTED_REQUIREMENT_EXIT_CODE
+from toil.exceptions import FailedJobsException
 from toil.job import (CheckpointJobDescription,
                       JobDescription,
                       ServiceJobDescription,
                       TemporaryID)
 from toil.jobStores.abstractJobStore import (AbstractJobStore,
-                                             NoSuchFileException,
                                              NoSuchJobException)
 from toil.lib.throttle import LocalThrottle
 from toil.provisioners.abstractProvisioner import AbstractProvisioner
@@ -51,7 +51,6 @@ from toil.provisioners.clusterScaler import ScalerThread
 from toil.serviceManager import ServiceManager
 from toil.statsAndLogging import StatsAndLogging
 from toil.toilState import ToilState
-from toil.exceptions import FailedJobsException
 
 logger = logging.getLogger(__name__)
 
@@ -115,10 +114,14 @@ class Leader:
         # state change information about jobs.
         self.toilState = ToilState(self.jobStore)
 
-        if self.config.write_messages is not None:
-            # Message bus messages need to go to the given file.
-            # Keep a reference to the return value so the listener stays alive.
-            self._message_subscription = self.toilState.bus.connect_output_file(self.config.write_messages)
+        if self.config.write_messages is None:
+            # The user hasn't specified a place for the message bus so we
+            # should make one.
+            self.config.write_messages = gen_message_bus_path()
+
+        # Message bus messages need to go to the given file.
+        # Keep a reference to the return value so the listener stays alive.
+        self._message_subscription = self.toilState.bus.connect_output_file(self.config.write_messages)
 
         # Connect to the message bus, so we will get all the messages of these
         # types in an inbox.
@@ -138,7 +141,8 @@ class Leader:
 
         # Batch system
         self.batchSystem = batchSystem
-        assert len(self.batchSystem.getIssuedBatchJobIDs()) == 0  # Batch system must start with no active jobs!
+        if len(self.batchSystem.getIssuedBatchJobIDs()) != 0:
+            raise RuntimeError("The initialized batchsystem did not start with 0 active jobs.")
         logger.debug("Checked batch system has no running jobs and no updated jobs")
 
         # Map of batch system IDs to job store IDs
@@ -370,7 +374,8 @@ class Leader:
 
         # If the successor job's predecessors have all not all completed then
         # ignore the successor as is not yet ready to run
-        assert len(successor.predecessorsFinished) <= successor.predecessorNumber
+        if len(successor.predecessorsFinished) > successor.predecessorNumber:
+            raise RuntimeError("There are more finished predecessors than possible.")
         if len(successor.predecessorsFinished) == successor.predecessorNumber:
             # All the successor's predecessors are done now.
             # Remove the successor job from the set of waiting multi-predecessor jobs.
@@ -391,8 +396,10 @@ class Leader:
         #Build map from successor to predecessors.
         if successor_id not in self.toilState.successor_to_predecessors:
             self.toilState.successor_to_predecessors[successor_id] = set()
-        assert isinstance(successor_id, str)
-        assert isinstance(predecessor_id, str)
+        if not isinstance(successor_id, str):
+            raise RuntimeError("The given successor ID is invalid.")
+        if not isinstance(predecessor_id, str):
+            raise RuntimeError("The given predecessor ID is invalid.")
         self.toilState.successor_to_predecessors[successor_id].add(predecessor_id)
 
         # Grab the successor
@@ -423,7 +430,8 @@ class Leader:
                      predecessor_id, len(next_successors))
         #Record the number of successors that must be completed before
         #the job can be considered again
-        assert self.toilState.count_pending_successors(predecessor_id) == 0, 'Attempted to schedule successors of the same job twice!'
+        if self.toilState.count_pending_successors(predecessor_id) != 0:
+            raise RuntimeError('Attempted to schedule successors of the same job twice!')
         self.toilState.successors_pending(predecessor_id, len(next_successors))
 
         # For each successor schedule if all predecessors have been completed
@@ -534,11 +542,13 @@ class Leader:
             # the job has services to run, which have not been started, start them
             # Build a map from the service jobs to the job and a map
             # of the services created for the job
-            assert readyJob.jobStoreID not in self.toilState.servicesIssued
+            if readyJob.jobStoreID in self.toilState.servicesIssued:
+                raise RuntimeError(f"The ready job: {readyJob.jobStoreID} was already issued.")
             self.toilState.servicesIssued[readyJob.jobStoreID] = set()
             for serviceJobList in readyJob.serviceHostIDsInBatches():
                 for serviceID in serviceJobList:
-                    assert serviceID not in self.toilState.service_to_client
+                    if serviceID in self.toilState.service_to_client:
+                        raise RuntimeError(f"The ready service ID: {serviceID} was already added.")
                     self.toilState.reset_job(serviceID)
                     serviceHost = self.toilState.get_job(serviceID)
                     self.toilState.service_to_client[serviceID] = readyJob.jobStoreID
@@ -675,7 +685,8 @@ class Leader:
             client = self.toilState.get_job(client_id)
 
             # Make sure services still want to run
-            assert next(client.serviceHostIDsInBatches(), None) is not None
+            if next(client.serviceHostIDsInBatches(), None) is None:
+                raise RuntimeError("No more services want to run.")
 
             # Mark the service job updated so we don't stop here.
             self._messages.publish(JobUpdatedMessage(client_id, 1))
@@ -784,13 +795,16 @@ class Leader:
         logger.debug("Finished the main loop: no jobs left to run.")
 
         # Consistency check the toil state
-        assert self._messages.empty(), f"Pending messages at shutdown: {self._messages}"
-        assert self.toilState.successorCounts == {}, f"Jobs waiting on successors at shutdown: {self.toilState.successorCounts}"
-        assert self.toilState.successor_to_predecessors == {}, f"Successors pending for their predecessors at shutdown: {self.toilState.successor_to_predecessors}"
-        assert self.toilState.service_to_client == {}, f"Services pending for their clients at shutdown: {self.toilState.service_to_client}"
-        assert self.toilState.servicesIssued == {}, f"Services running at shutdown: {self.toilState.servicesIssued}"
-        # assert self.toilState.jobsToBeScheduledWithMultiplePredecessors # These are not properly emptied yet
-        # assert self.toilState.hasFailedSuccessors == set() # These are not properly emptied yet
+        if not self._messages.empty():
+            raise RuntimeError(f"Pending messages at shutdown: {self._messages}")
+        if self.toilState.successorCounts != {}:
+            raise RuntimeError(f"Jobs waiting on successors at shutdown: {self.toilState.successorCounts}")
+        if self.toilState.successor_to_predecessors != {}:
+            raise RuntimeError(f"Successors pending for their predecessors at shutdown: {self.toilState.successor_to_predecessors}")
+        if self.toilState.service_to_client != {}:
+            raise RuntimeError(f"Services pending for their clients at shutdown: {self.toilState.service_to_client}")
+        if self.toilState.servicesIssued != {}:
+            raise RuntimeError(f"Services running at shutdown: {self.toilState.servicesIssued}")
 
     def checkForDeadlocks(self):
         """Check if the system is deadlocked running service jobs."""
@@ -865,8 +879,8 @@ class Leader:
     def issueJob(self, jobNode: JobDescription) -> None:
         """Add a job to the queue of jobs currently trying to run."""
         # Never issue the same job multiple times simultaneously
-        assert jobNode.jobStoreID not in self.toilState.jobs_issued, \
-            f"Attempted to issue {jobNode} multiple times simultaneously!"
+        if jobNode.jobStoreID in self.toilState.jobs_issued:
+            raise RuntimeError(f"Attempted to issue {jobNode} multiple times simultaneously!")
 
         workerCommand = [resolveEntryPoint('_toil_worker'),
                          jobNode.jobName,
@@ -926,7 +940,8 @@ class Leader:
         """
         # Grab the service job description
         service = self.toilState.get_job(service_id)
-        assert isinstance(service, ServiceJobDescription)
+        if not isinstance(service, ServiceJobDescription):
+            raise RuntimeError("The grabbed service job description is not the right type.")
 
         if service.preemptible:
             self.preemptibleServiceJobsToBeIssued.append(service_id)
@@ -956,7 +971,8 @@ class Leader:
         elif preemptible:
             return self.preemptibleJobsIssued
         else:
-            assert len(self.issued_jobs_by_batch_system_id) >= self.preemptibleJobsIssued
+            if len(self.issued_jobs_by_batch_system_id) < self.preemptibleJobsIssued:
+                raise RuntimeError("Number of jobs issued cannot be negative.")
             return len(self.issued_jobs_by_batch_system_id) - self.preemptibleJobsIssued
 
     def _getStatusHint(self) -> str:
@@ -990,16 +1006,19 @@ class Leader:
 
         :return: Job description as it was issued.
         """
-        assert jobBatchSystemID in self.issued_jobs_by_batch_system_id
+        if jobBatchSystemID not in self.issued_jobs_by_batch_system_id:
+            raise RuntimeError("Job was already removed or was never issued.")
         issuedDesc = self.toilState.get_job(self.issued_jobs_by_batch_system_id[jobBatchSystemID])
         if issuedDesc.preemptible:
             # len(issued_jobs_by_batch_system_id) should always be greater than or equal to preemptibleJobsIssued,
             # so decrement this value before removing the job from the issuedJob map
-            assert self.preemptibleJobsIssued > 0
+            if self.preemptibleJobsIssued <= 0:
+                raise RuntimeError("The number of preemptive issued jobs cannot be negative.")
             self.preemptibleJobsIssued -= 1
         # It's not issued anymore.
         del self.issued_jobs_by_batch_system_id[jobBatchSystemID]
-        assert issuedDesc.jobStoreID in self.toilState.jobs_issued, f"Job {issuedDesc} came back without being issued"
+        if issuedDesc.jobStoreID not in self.toilState.jobs_issued:
+            raise RuntimeError(f"Job {issuedDesc} came back without being issued")
         self.toilState.jobs_issued.remove(issuedDesc.jobStoreID)
         # If service job
         if issuedDesc.jobStoreID in self.toilState.service_to_client:
@@ -1090,8 +1109,9 @@ class Leader:
         for jobBatchSystemID in missingJobIDsSet.difference(jobBatchSystemIDsSet):
             self.reissueMissingJobs_missingHash.pop(jobBatchSystemID)
             logger.warning("Batch system id: %s is no longer missing", str(jobBatchSystemID))
-        assert issuedJobs.issubset(jobBatchSystemIDsSet) #Assert checks we have
-        #no unexpected jobs running
+        # checks we have no unexpected jobs running
+        if not issuedJobs.issubset(jobBatchSystemIDsSet):
+            raise RuntimeError("An unexpected job is still running.")
         jobsToKill = []
         for jobBatchSystemID in set(jobBatchSystemIDsSet.difference(issuedJobs)):
             jobStoreID = self.issued_jobs_by_batch_system_id[jobBatchSystemID]
@@ -1295,12 +1315,15 @@ class Leader:
             # Is a service job
             logger.debug("Service job is being processed as a totally failed job: %s", job_desc)
 
-            assert isinstance(job_desc, ServiceJobDescription)
+
+            if not isinstance(job_desc, ServiceJobDescription):
+                raise RuntimeError("The service job description type is incorrect.")
 
             # Grab the client, which is the predecessor.
             client_id = self.toilState.service_to_client[job_id]
 
-            assert client_id in self.toilState.servicesIssued
+            if client_id not in self.toilState.servicesIssued:
+                raise RuntimeError("The client was never issued.")
 
             # Leave the service job as a service of its predecessor, because it
             # didn't work.
@@ -1331,8 +1354,10 @@ class Leader:
             self.jobStore.delete_file(job_desc.startJobStoreID)
         else:
             # Is a non-service job
-            assert job_id not in self.toilState.servicesIssued
-            assert not isinstance(job_desc, ServiceJobDescription)
+            if job_id in self.toilState.servicesIssued:
+                raise RuntimeError("The non-service job should not have been issued.")
+            if isinstance(job_desc, ServiceJobDescription):
+                raise RuntimeError("The job description type is incorrect.")
 
             # Traverse failed job's successor graph and get the jobStoreID of new successors.
             # Any successor already in toilState.failedSuccessors will not be traversed
@@ -1401,11 +1426,13 @@ class Leader:
                              len(self.toilState.servicesIssued[client_id]))
         elif jobStoreID not in self.toilState.successor_to_predecessors:
             #We have reach the root job
-            assert self._messages.count(JobUpdatedMessage) == 0, "Root job is done but other jobs are still updated"
-            assert len(self.toilState.successor_to_predecessors) == 0, \
-                ("Job {} is finished and had no predecessor, but we have other outstanding jobs "
+            if self._messages.count(JobUpdatedMessage) != 0:
+                raise RuntimeError("Root job is done but other jobs are still updated")
+            if len(self.toilState.successor_to_predecessors) != 0:
+                raise RuntimeError("Job {} is finished and had no predecessor, but we have other outstanding jobs "
                  "with predecessors: {}".format(jobStoreID, self.toilState.successor_to_predecessors.keys()))
-            assert len(self.toilState.successorCounts) == 0, f"Root job is done but jobs waiting on successors: {self.toilState.successorCounts}"
+            if len(self.toilState.successorCounts) != 0:
+                raise RuntimeError("Root job is done but jobs waiting on successors: {self.toilState.successorCounts}")
             logger.debug("Reached root job %s so no predecessors to clean up" % jobStoreID)
 
         else:
@@ -1414,7 +1441,8 @@ class Leader:
 
             # For each predecessor
             for predecessor_id in self.toilState.successor_to_predecessors.pop(jobStoreID):
-                assert isinstance(predecessor_id, str), f"Predecessor ID should be str but is {type(predecessor_id)}"
+                if not isinstance(predecessor_id, str):
+                    raise RuntimeError("Predecessor ID should be str but is {type(predecessor_id)}")
                 predecessor = self.toilState.get_job(predecessor_id)
 
                 # Tell the predecessor that this job is done (keep only other successor jobs)