toil 9.0.0__py3-none-any.whl → 9.1.1__py3-none-any.whl

toil/fileStores/abstractFileStore.py

@@ -671,13 +671,16 @@ class AbstractFileStore(ABC):
         Send a logging message to the leader. The message will also be \
         logged by the worker at the same level.
 
+        Does not depend on the commit system, so this is safe to use during an
+        ansynchronous commit, or without a commit afterward.
+
         :param text: The string to log.
         :param level: The logging level.
         """
-        logger.log(level=level, msg=("LOG-TO-MASTER: " + text))
+        logger.log(level=level, msg=("LOG-TO-LEADER: " + text))
         self.logging_messages.append(dict(text=text, level=level))
 
-    @deprecated(new_function_name="export_file")
+    @deprecated(new_function_name="log_to_leader")
     def logToMaster(self, text: str, level: int = logging.INFO) -> None:
         self.log_to_leader(text, level)
 

toil/fileStores/cachingFileStore.py

@@ -1207,7 +1207,7 @@ class CachingFileStore(AbstractFileStore):
             # its temp dir and database entry.
             self._deallocateSpaceForJob()
 
-    def writeGlobalFile(self, localFileName, cleanup=False, executable=False):
+    def writeGlobalFile(self, localFileName, cleanup=False):
         """
         Creates a file in the jobstore and returns a FileID reference.
         """

toil/job.py

@@ -236,16 +236,16 @@ def parse_accelerator(
     {'count': 1, 'kind': 'gpu'}
 
     >>> parse_accelerator("nvidia-tesla-k80")
-    {'count': 1, 'kind': 'gpu', 'brand': 'nvidia', 'model': 'nvidia-tesla-k80'}
+    {'count': 1, 'kind': 'gpu', 'model': 'nvidia-tesla-k80', 'brand': 'nvidia'}
 
     >>> parse_accelerator("nvidia-tesla-k80:2")
-    {'count': 2, 'kind': 'gpu', 'brand': 'nvidia', 'model': 'nvidia-tesla-k80'}
+    {'count': 2, 'kind': 'gpu', 'model': 'nvidia-tesla-k80', 'brand': 'nvidia'}
 
     >>> parse_accelerator("gpu")
     {'count': 1, 'kind': 'gpu'}
 
     >>> parse_accelerator("cuda:1")
-    {'count': 1, 'kind': 'gpu', 'brand': 'nvidia', 'api': 'cuda'}
+    {'count': 1, 'kind': 'gpu', 'api': 'cuda', 'brand': 'nvidia'}
 
     >>> parse_accelerator({"kind": "gpu"})
     {'count': 1, 'kind': 'gpu'}
@@ -581,8 +581,8 @@ class Requirer:
         >>> Requirer._parseResource('cores', 1), Requirer._parseResource('disk', 1), \
         Requirer._parseResource('memory', 1)
         (1, 1, 1)
-        >>> Requirer._parseResource('cores', '1G'), Requirer._parseResource('disk', '1G'), \
-        Requirer._parseResource('memory', '1G')
+        >>> Requirer._parseResource('cores', '1Gi'), Requirer._parseResource('disk', '1Gi'), \
+        Requirer._parseResource('memory', '1Gi')
         (1073741824, 1073741824, 1073741824)
         >>> Requirer._parseResource('cores', 1.1)
         1.1
@@ -813,7 +813,6 @@ class JobDescription(Requirer):
     Subclassed into variants for checkpoint jobs and service jobs that have
     their specific parameters.
     """
-
     def __init__(
         self,
         requirements: Mapping[str, Union[int, str, float, bool, list]],
@@ -3146,9 +3145,8 @@ class Job:
 
         Will modify the job's description with changes that need to be committed back to the JobStore.
         """
-        if stats is not None:
-            startTime = time.time()
-            startClock = ResourceMonitor.get_total_cpu_time()
+        startTime = time.time()
+        startClock = ResourceMonitor.get_total_cpu_time()
         baseDir = os.getcwd()
 
         succeeded = False
@@ -3180,18 +3178,36 @@ class Job:
             # Change dir back to cwd dir, if changed by job (this is a safety issue)
             if os.getcwd() != baseDir:
                 os.chdir(baseDir)
+            
+            totalCpuTime, total_memory_kib = (
+                ResourceMonitor.get_total_cpu_time_and_memory_usage()
+            )
+            job_time = time.time() - startTime
+            job_cpu_time = totalCpuTime - startClock
+            allocated_cpu_time = job_time * self.cores
+
+            if job_cpu_time > allocated_cpu_time and allocated_cpu_time > 0:
+                # Too much CPU was used by this job! Maybe we're using a batch
+                # system that doesn't/can't sandbox us and we started too many
+                # threads. Complain to the user!
+                excess_factor = job_cpu_time / allocated_cpu_time
+                fileStore.log_to_leader(
+                    f"Job {self.description} used {excess_factor:.2f}x more "
+                    f"CPU than the requested {self.cores} cores. Consider "
+                    f"increasing the job's required CPU cores or limiting the "
+                    f"number of processes/threads launched.",
+                    level=logging.WARNING
+                )
+
             # Finish up the stats
             if stats is not None:
-                totalCpuTime, total_memory_kib = (
-                    ResourceMonitor.get_total_cpu_time_and_memory_usage()
-                )
                 stats.jobs.append(
                     # TODO: We represent everything as strings in the stats
                     # even though the JSON transport can take bools and floats.
                     Expando(
                         start=str(startTime),
-                        time=str(time.time() - startTime),
-                        clock=str(totalCpuTime - startClock),
+                        time=str(job_time),
+                        clock=str(job_cpu_time),
                         class_name=self._jobName(),
                         memory=str(total_memory_kib),
                         requested_cores=str(self.cores), # TODO: Isn't this really consumed cores?

toil/jobStores/abstractJobStore.py

@@ -90,7 +90,7 @@ class InvalidImportExportUrlException(Exception):
 class NoSuchJobException(Exception):
     """Indicates that the specified job does not exist."""
 
-    def __init__(self, jobStoreID: FileID):
+    def __init__(self, jobStoreID: Union[FileID, str]):
         """
         :param str jobStoreID: the jobStoreID that was mistakenly assumed to exist
         """
@@ -100,7 +100,7 @@ class NoSuchJobException(Exception):
 class ConcurrentFileModificationException(Exception):
     """Indicates that the file was attempted to be modified by multiple processes at once."""
 
-    def __init__(self, jobStoreFileID: FileID):
+    def __init__(self, jobStoreFileID: Union[FileID, str]):
         """
         :param jobStoreFileID: the ID of the file that was modified by multiple workers
                or processes concurrently
@@ -112,7 +112,7 @@ class NoSuchFileException(Exception):
     """Indicates that the specified file does not exist."""
 
     def __init__(
-        self, jobStoreFileID: FileID, customName: Optional[str] = None, *extra: Any
+        self, jobStoreFileID: Union[FileID, str], customName: Optional[str] = None, *extra: Any
     ):
         """
         :param jobStoreFileID: the ID of the file that was mistakenly assumed to exist
@@ -140,11 +140,12 @@ class NoSuchJobStoreException(LocatorException):
     def __init__(self, locator: str, prefix: str):
         """
         :param str locator: The location of the job store
+        :param str prefix: The type of job store
         """
         super().__init__(
             "The job store '%s' does not exist, so there is nothing to restart.",
             locator,
-            prefix,
+            prefix
         )
 
 
@@ -159,7 +160,7 @@ class JobStoreExistsException(LocatorException):
             "The job store '%s' already exists. Use --restart to resume the workflow, or remove "
             "the job store with 'toil clean' to start the workflow from scratch.",
             locator,
-            prefix,
+            prefix
         )
 
 
@@ -240,7 +241,12 @@ class AbstractJobStore(ABC):
 
     @property
     def config(self) -> Config:
-        """Return the Toil configuration associated with this job store."""
+        """
+        Return the Toil configuration associated with this job store.
+
+        :raises AttributeError: if the config has not yet been assigned (i.e.
+            during :meth:`resume`).
+        """
         return self.__config
 
     @property
@@ -561,7 +567,6 @@ class AbstractJobStore(ABC):
                 executable = jobStoreFileID.executable
             otherCls._write_to_url(readable, url, executable)
 
-
     @abstractmethod
     def destroy(self) -> None:
         """
@@ -930,15 +935,6 @@ class AbstractJobStore(ABC):
         """
         raise NotImplementedError()
 
-    @contextmanager
-    def batch(self) -> Iterator[None]:
-        """
-        If supported by the batch system, calls to create() with this context
-        manager active will be performed in a batch after the context manager
-        is released.
-        """
-        yield
-
     @deprecated(new_function_name="create_job")
     def create(self, jobDescription: JobDescription) -> JobDescription:
         return self.create_job(jobDescription)
@@ -1036,6 +1032,15 @@ class AbstractJobStore(ABC):
     def update(self, jobDescription: JobDescription) -> None:
         return self.update_job(jobDescription)
 
+    @contextmanager
+    def batch(self) -> Iterator[None]:
+        """
+        If supported by the batch system, calls to create() with this context
+        manager active will be performed in a batch after the context manager
+        is released.
+        """
+        yield
+
     @abstractmethod
     def update_job(self, job_description: JobDescription) -> None:
         """
@@ -1206,14 +1211,14 @@ class AbstractJobStore(ABC):
         Creates an empty file in the job store and returns its ID.
         Call to fileExists(getEmptyFileStoreID(jobStoreID)) will return True.
 
-        :param str job_id: the id of a job, or None. If specified, the may be associated
+        :param job_id: the id of a job, or None. If specified, the may be associated
                with that job in a job-store-specific way. This may influence the returned ID.
 
-        :param bool cleanup: Whether to attempt to delete the file when the job
+        :param cleanup: Whether to attempt to delete the file when the job
                whose jobStoreID was given as jobStoreID is deleted with
                jobStore.delete(job). If jobStoreID was not given, does nothing.
 
-        :param str basename: If supported by the implementation, use the given
+        :param basename: If supported by the implementation, use the given
                file basename so that when searching the job store with a query
                matching that basename, the file will be detected.