toil 8.2.0__py3-none-any.whl → 9.1.0__py3-none-any.whl

toil/batchSystems/abstractBatchSystem.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from dataclasses import dataclass
 import enum
 import logging
 import os
@@ -72,10 +73,13 @@ class BatchJobExitReason(enum.IntEnum):
         except ValueError:
             return str(value)
 
-
-class UpdatedBatchJobInfo(NamedTuple):
+@dataclass
+class UpdatedBatchJobInfo:
     jobID: int
-    exitStatus: int
+    """
+    The Toil batch system ID of the job.
+    """
+    exitStatus: int = EXIT_STATUS_UNAVAILABLE_VALUE
     """
     The exit status (integer value) of the job. 0 implies successful.
 
@@ -83,8 +87,12 @@ class UpdatedBatchJobInfo(NamedTuple):
     (e.g. job is lost, or otherwise died but actual exit code was not reported).
     """
 
-    exitReason: Optional[BatchJobExitReason]
-    wallTime: Union[float, int, None]
+    exitReason: Optional[BatchJobExitReason] = None
+    wallTime: Union[float, int, None] = None
+    backing_id: Optional[str] = None
+    """
+    The identifier for the job in the backing scheduler, if available.
+    """
 
 
 # Information required for worker cleanup on shutdown of the batch system.

toil/batchSystems/abstractGridEngineBatchSystem.py

@@ -159,14 +159,21 @@ class AbstractGridEngineBatchSystem(BatchSystemCleanupSupport):
                 logger.debug("Running %r", subLine)
                 batchJobID = self.boss.with_retries(self.submitJob, subLine)
                 if self.boss._outbox is not None:
-                    # JobID corresponds to the toil version of the jobID, dif from jobstore idea of the id, batchjobid is what we get from slurm
+                    # JobID corresponds to the toil version of the jobID,
+                    # different from the jobstore's idea of the id. batchjobid
+                    # is what we get from e.g. slurm
                     self.boss._outbox.publish(
                         ExternalBatchIdMessage(
                             jobID, batchJobID, self.boss.__class__.__name__
                         )
                     )
 
-                logger.debug("Submitted job %s", str(batchJobID))
+                logger.info(
+                    "Job %s with batch system ID %s queued as job %s",
+                    jobName,
+                    jobID,
+                    str(batchJobID)
+                )
 
                 # Store dict for mapping Toil job ID to batch job ID
                 # TODO: Note that this currently stores a tuple of (batch system
@@ -251,8 +258,8 @@ class AbstractGridEngineBatchSystem(BatchSystemCleanupSupport):
                     self.coalesce_job_exit_codes, batch_job_id_list
                 )
                 # We got the statuses as a batch
-                for running_job_id, status in zip(running_job_list, statuses):
-                    activity = self._handle_job_status(running_job_id, status, activity)
+                for running_job_id, status, backing_id in zip(running_job_list, statuses, batch_job_id_list):
+                    activity = self._handle_job_status(running_job_id, status, activity, backing_id)
 
             self._checkOnJobsCache = activity
             self._checkOnJobsTimestamp = datetime.now()
@@ -263,6 +270,7 @@ class AbstractGridEngineBatchSystem(BatchSystemCleanupSupport):
             job_id: int,
             status: Union[int, tuple[int, Optional[BatchJobExitReason]], None],
             activity: bool,
+            backing_id: str,
         ) -> bool:
             """
             Helper method for checkOnJobs to handle job statuses
@@ -275,7 +283,11 @@ class AbstractGridEngineBatchSystem(BatchSystemCleanupSupport):
                     code, reason = status
                 self.updatedJobsQueue.put(
                     UpdatedBatchJobInfo(
-                        jobID=job_id, exitStatus=code, exitReason=reason, wallTime=None
+                        jobID=job_id,
+                        exitStatus=code,
+                        exitReason=reason,
+                        wallTime=None,
+                        backing_id=backing_id,
                     )
                 )
                 self.forgetJob(job_id)

toil/batchSystems/kubernetes.py

@@ -37,6 +37,7 @@ from threading import Condition, Event, RLock, Thread
 from typing import Any, Callable, Literal, Optional, TypeVar, Union, cast, overload
 
 from toil.lib.conversions import opt_strtobool
+from toil.lib.throttle import LocalThrottle
 
 if sys.version_info < (3, 10):
     from typing_extensions import ParamSpec
@@ -281,6 +282,10 @@ class KubernetesBatchSystem(BatchSystemCleanupSupport):
         # in the queue or any resource becomes available.
         self._work_available: Condition = Condition(lock=self._mutex)
 
+        # To make sure we don't spam the log when the metrics server is down,
+        # we use a throttle
+        self._metrics_throttle: LocalThrottle = LocalThrottle(600)
+
         self.schedulingThread: Thread = Thread(target=self._scheduler, daemon=True)
         self.schedulingThread.start()
 
@@ -1363,7 +1368,8 @@ class KubernetesBatchSystem(BatchSystemCleanupSupport):
                 # This is the sort of error we would expect from an overloaded
                 # Kubernetes or a dead metrics service.
                 # We can't tell that the pod is stuck, so say that it isn't.
-                logger.warning("Could not query metrics service: %s", e)
+                if self._metrics_throttle.throttle(False):
+                    logger.warning("Kubernetes metrics service is not available: %s", e)
                 return False
             else:
                 raise
@@ -1602,6 +1608,7 @@ class KubernetesBatchSystem(BatchSystemCleanupSupport):
                         exitStatus=exitCode,
                         wallTime=runtime,
                         exitReason=exitReason,
+                        backing_id=jobObject.metadata.name,
                     )
 
                     if (exitReason == BatchJobExitReason.FAILED) or (
@@ -1855,7 +1862,11 @@ class KubernetesBatchSystem(BatchSystemCleanupSupport):
 
         # Return the one finished job we found
         return UpdatedBatchJobInfo(
-            jobID=jobID, exitStatus=exitCode, wallTime=runtime, exitReason=None
+            jobID=jobID,
+            exitStatus=exitCode,
+            wallTime=runtime,
+            exitReason=None,
+            backing_id=jobObject.metadata.name,
         )
 
     def _waitForJobDeath(self, jobName: str) -> None:

toil/batchSystems/mesos/batchSystem.py

@@ -103,6 +103,9 @@ class MesosBatchSystem(BatchSystemLocalSupport, AbstractScalableBatchSystem, Sch
         if config.mesos_framework_id is not None:
             self.mesos_framework_id = config.mesos_framework_id
 
+        # How long in seconds to wait to register before declaring Mesos unreachable.
+        self.mesos_timeout = 60
+
         # Written to when Mesos kills tasks, as directed by Toil.
         # Jobs must not enter this set until they are removed from runningJobMap.
         self.killedJobIds = set()
@@ -345,17 +348,38 @@ class MesosBatchSystem(BatchSystemLocalSupport, AbstractScalableBatchSystem, Sch
             framework.roles = config.mesos_role
             framework.capabilities = [dict(type="MULTI_ROLE")]
 
+        endpoint = self._resolveAddress(self.mesos_endpoint)
+        log.info("Connecting to Mesos at %s...", self.mesos_endpoint)
+
         # Make the driver which implements most of the scheduler logic and calls back to us for the user-defined parts.
         # Make sure it will call us with nice namespace-y addicts
         self.driver = MesosSchedulerDriver(
             self,
             framework,
-            self._resolveAddress(self.mesos_endpoint),
+            endpoint,
             use_addict=True,
             implicit_acknowledgements=True,
         )
         self.driver.start()
 
+        wait_count = 0
+        while self.frameworkId is None:
+            # Wait to register with Mesos, and eventually fail if it just isn't
+            # responding.
+
+            # TODO: Use a condition instead of a spin wait.
+
+            if wait_count >= self.mesos_timeout:
+                error_message = f"Could not connect to Mesos endpoint at {self.mesos_endpoint}"
+                log.error(error_message)
+                self.shutdown()
+                raise RuntimeError(error_message)
+            elif wait_count > 1 and wait_count % 10 == 0:
+                log.warning("Waiting for Mesos registration (try %s/%s)", wait_count, self.mesos_timeout)
+            time.sleep(1)
+            wait_count += 1
+
+
     @staticmethod
     def _resolveAddress(address):
         """
@@ -394,10 +418,17 @@ class MesosBatchSystem(BatchSystemLocalSupport, AbstractScalableBatchSystem, Sch
         """
         Invoked when the scheduler successfully registers with a Mesos master
         """
-        log.debug("Registered with framework ID %s", frameworkId.value)
+        log.info("Registered with Mesos as framework ID %s", frameworkId.value)
         # Save the framework ID
         self.frameworkId = frameworkId.value
 
+    def error(self, driver, message):
+        """
+        Invoked when Mesos reports an unrecoverable error.
+        """
+        log.error("Mesos error: %s", message)
+        super().error(driver, message)
+
     def _declineAllOffers(self, driver, offers):
         for offer in offers:
             driver.declineOffer(offer.id)

toil/batchSystems/registry.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import importlib
 import logging
 import pkgutil
 import warnings
@@ -21,6 +20,7 @@ from typing import TYPE_CHECKING, Callable
 
 from toil.lib.compatibility import deprecated
 from toil.lib.memoize import memoize
+import toil.lib.plugins
 
 if TYPE_CHECKING:
     from toil.batchSystems.abstractBatchSystem import AbstractBatchSystem
@@ -40,17 +40,14 @@ def add_batch_system_factory(
 
     :param class_factory: A function that returns a batch system class (NOT an instance), which implements :class:`toil.batchSystems.abstractBatchSystem.AbstractBatchSystem`.
     """
-    _registry_keys.append(key)
-    _registry[key] = class_factory
+    toil.lib.plugins.register_plugin("batch_system", key, class_factory)
 
 
 def get_batch_systems() -> Sequence[str]:
     """
-    Get the names of all the availsble batch systems.
+    Get the names of all the available batch systems.
     """
-    _load_all_plugins()
-
-    return _registry_keys
+    return toil.lib.plugins.get_plugin_names("batch_system")
 
 
 def get_batch_system(key: str) -> type["AbstractBatchSystem"]:
@@ -60,8 +57,7 @@ def get_batch_system(key: str) -> type["AbstractBatchSystem"]:
     :raises: KeyError if the key is not the name of a batch system, and
              ImportError if the batch system's class cannot be loaded.
     """
-
-    return _registry[key]()
+    return toil.lib.plugins.get_plugin("batch_system", key)()
 
 
 DEFAULT_BATCH_SYSTEM = "single_machine"
@@ -126,114 +122,15 @@ def kubernetes_batch_system_factory():
 
 
 #####
-# Registry implementation
-#####
-
-_registry: dict[str, Callable[[], type["AbstractBatchSystem"]]] = {
-    "aws_batch": aws_batch_batch_system_factory,
-    "single_machine": single_machine_batch_system_factory,
-    "grid_engine": gridengine_batch_system_factory,
-    "lsf": lsf_batch_system_factory,
-    "mesos": mesos_batch_system_factory,
-    "slurm": slurm_batch_system_factory,
-    "torque": torque_batch_system_factory,
-    "htcondor": htcondor_batch_system_factory,
-    "kubernetes": kubernetes_batch_system_factory,
-}
-_registry_keys = list(_registry.keys())
-
-# We will load any packages starting with this prefix and let them call
-# add_batch_system_factory()
-_PLUGIN_NAME_PREFIX = "toil_batch_system_"
-
-
-@memoize
-def _load_all_plugins() -> None:
-    """
-    Load all the batch system plugins that are installed.
-    """
-
-    for finder, name, is_pkg in pkgutil.iter_modules():
-        # For all installed packages
-        if name.startswith(_PLUGIN_NAME_PREFIX):
-            # If it is a Toil batch system plugin, import it
-            importlib.import_module(name)
-
-
-#####
-# Deprecated API
+# Registers all built-in batch system
 #####
 
-# We used to directly access these constants, but now the Right Way to use this
-# module is add_batch_system_factory() to register and get_batch_systems() to
-# get the list/get_batch_system() to get a class by name.
-
-
-def __getattr__(name):
-    """
-    Implement a fallback attribute getter to handle deprecated constants.
-
-    See <https://stackoverflow.com/a/48242860>.
-    """
-    if name == "BATCH_SYSTEM_FACTORY_REGISTRY":
-        warnings.warn(
-            "BATCH_SYSTEM_FACTORY_REGISTRY is deprecated; use get_batch_system() or add_batch_system_factory()",
-            DeprecationWarning,
-        )
-        return _registry
-    elif name == "BATCH_SYSTEMS":
-        warnings.warn(
-            "BATCH_SYSTEMS is deprecated; use get_batch_systems()", DeprecationWarning
-        )
-        return _registry_keys
-    else:
-        raise AttributeError(f"Module {__name__} ahs no attribute {name}")
-
-
-@deprecated(new_function_name="add_batch_system_factory")
-def addBatchSystemFactory(
-    key: str, batchSystemFactory: Callable[[], type["AbstractBatchSystem"]]
-):
-    """
-    Deprecated method to add a batch system.
-    """
-    return add_batch_system_factory(key, batchSystemFactory)
-
-
-#####
-# Testing utilities
-#####
-
-# We need a snapshot save/restore system for testing. We can't just tamper with
-# the globals because module-level globals are their own references, so we
-# can't touch this module's global name bindings from a client module.
-
-
-def save_batch_system_plugin_state() -> (
-    tuple[list[str], dict[str, Callable[[], type["AbstractBatchSystem"]]]]
-):
-    """
-    Return a snapshot of the plugin registry that can be restored to remove
-    added plugins. Useful for testing the plugin system in-process with other
-    tests.
-    """
-
-    snapshot = (list(_registry_keys), dict(_registry))
-    return snapshot
-
-
-def restore_batch_system_plugin_state(
-    snapshot: tuple[list[str], dict[str, Callable[[], type["AbstractBatchSystem"]]]]
-):
-    """
-    Restore the batch system registry state to a snapshot from
-    save_batch_system_plugin_state().
-    """
-
-    # We need to apply the snapshot without rebinding the names, because that
-    # won't affect modules that imported the names.
-    wanted_batch_systems, wanted_registry = snapshot
-    _registry_keys.clear()
-    _registry_keys.extend(wanted_batch_systems)
-    _registry.clear()
-    _registry.update(wanted_registry)
+add_batch_system_factory("aws_batch", aws_batch_batch_system_factory)
+add_batch_system_factory("single_machine", single_machine_batch_system_factory)
+add_batch_system_factory("grid_engine", gridengine_batch_system_factory)
+add_batch_system_factory("lsf", lsf_batch_system_factory)
+add_batch_system_factory("mesos", mesos_batch_system_factory)
+add_batch_system_factory("slurm", slurm_batch_system_factory)
+add_batch_system_factory("torque", torque_batch_system_factory)
+add_batch_system_factory("htcondor", htcondor_batch_system_factory)
+add_batch_system_factory("kubernetes", kubernetes_batch_system_factory)

toil/batchSystems/slurm.py

@@ -18,9 +18,11 @@ import logging
 import math
 import os
 import sys
-from argparse import SUPPRESS, ArgumentParser, _ArgumentGroup
 import shlex
-from typing import Callable, NamedTuple, TypeVar
+
+from argparse import SUPPRESS, ArgumentParser, _ArgumentGroup
+from datetime import datetime, timedelta, timezone
+from typing import Callable, NamedTuple, Optional, TypeVar
 
 from toil.batchSystems.abstractBatchSystem import (
     EXIT_STATUS_UNAVAILABLE_VALUE,
@@ -350,9 +352,18 @@ class SlurmBatchSystem(AbstractGridEngineBatchSystem):
         ) -> list[int | tuple[int, BatchJobExitReason | None] | None]:
             """
             Collect all job exit codes in a single call.
-            :param batch_job_id_list: list of Job ID strings, where each string has the form
-            "<job>[.<task>]".
-            :return: list of job exit codes or exit code, exit reason pairs associated with the list of job IDs.
+            
+            :param batch_job_id_list: list of Job ID strings, where each string
+                has the form ``<job>[.<task>]``.
+            
+            :return: list of job exit codes or exit code, exit reason pairs
+                associated with the list of job IDs.
+            
+            :raises CalledProcessErrorStderr: if communicating with Slurm went
+                wrong.
+            
+            :raises OSError: if job details are not available becasue a Slurm
+                command could not start.
             """
             logger.log(
                 TRACE, "Getting exit codes for slurm jobs: %s", batch_job_id_list
@@ -387,15 +398,54 @@ class SlurmBatchSystem(AbstractGridEngineBatchSystem):
             Helper function for `getJobExitCode` and `coalesce_job_exit_codes`.
             Fetch job details from Slurm's accounting system or job control system.
             :param job_id_list: list of integer Job IDs.
-            :return: dict of job statuses, where key is the integer job ID, and value is a tuple
-            containing the job's state and exit code.
+            :return: dict of job statuses, where key is the integer job ID, and
+                value is a tuple containing the job's state and exit code.
+            :raises CalledProcessErrorStderr: if communicating with Slurm went
+                wrong.
+            :raises OSError: if job details are not available becasue a Slurm
+                command could not start.
             """
+
+            status_dict = {}
+            scontrol_problem: Optional[Exception] = None
+
+            try:
+                # Get all the job details we can from scontrol, which we think
+                # might be faster/less dangerous than sacct searching, even
+                # though it can't be aimed at more than one job.
+                status_dict.update(self._getJobDetailsFromScontrol(job_id_list))
+            except (CalledProcessErrorStderr, OSError) as e:
+                if isinstance(e, OSError):
+                    logger.warning("Could not run scontrol: %s", e)
+                else:
+                    logger.warning("Error from scontrol: %s", e)
+                scontrol_problem = e
+
+            logger.debug("After scontrol, got statuses: %s", status_dict)
+
+            # See what's not handy in scontrol (or everything if we couldn't
+            # call it).
+            sacct_job_id_list = self._remaining_jobs(job_id_list, status_dict)
+
+            logger.debug("Remaining jobs to find out about: %s", sacct_job_id_list)
+
             try:
-                status_dict = self._getJobDetailsFromSacct(job_id_list)
+                # Ask sacct about those jobs
+                status_dict.update(self._getJobDetailsFromSacct(sacct_job_id_list))
             except (CalledProcessErrorStderr, OSError) as e:
                 if isinstance(e, OSError):
                     logger.warning("Could not run sacct: %s", e)
-                status_dict = self._getJobDetailsFromScontrol(job_id_list)
+                else:
+                    logger.warning("Error from sacct: %s", e)
+                if scontrol_problem is not None:
+                    # Neither approach worked at all
+                    raise
+
+            # One of the methods worked, so we have at least (None, None)
+            # values filled in for all jobs.
+            assert len(status_dict) == len(job_id_list)
+
+
             return status_dict
 
         def _get_job_return_code(
@@ -466,15 +516,123 @@ class SlurmBatchSystem(AbstractGridEngineBatchSystem):
 
             return state_token
 
+        def _remaining_jobs(self, job_id_list: list[int], job_details: dict[int, tuple[str | None, int | None]]) -> list[int]:
+            """
+            Given a list of job IDs and a list of job details (state and exit
+            code), get the list of job IDs where the details are (None, None)
+            (or are missing).
+            """
+            return [
+                j
+                for j in job_id_list
+                if job_details.get(j, (None, None)) == (None, None)
+            ]
+
         def _getJobDetailsFromSacct(
-            self, job_id_list: list[int]
+            self,
+            job_id_list: list[int],
+        ) -> dict[int, tuple[str | None, int | None]]:
+            """
+            Get SLURM job exit codes for the jobs in `job_id_list` by running `sacct`.
+
+            Handles querying manageable time periods until all jobs have information.
+
+            There is no guarantee of inter-job consistency: one job may really
+            finish after another, but we might see the earlier-finishing job
+            still running and the later-finishing job finished.
+
+            :param job_id_list: list of integer batch job IDs.
+            :return: dict of job statuses, where key is the job-id, and value
+                is a tuple containing the job's state and exit code. Jobs with
+                no information reported from Slurm will have (None, None).
+            """
+
+            # Pick a now
+            now = datetime.now().astimezone(None)
+            # Decide when to start the search (first copy of past midnight)
+            begin_time = now.replace(
+                hour=0,
+                minute=0,
+                second=0,
+                microsecond=0,
+                fold=0
+            )
+            # And when to end (a day after that)
+            end_time = begin_time + timedelta(days=1)
+            while end_time < now:
+                # If something goes really weird, advance up to our chosen now
+                end_time += timedelta(days=1)
+            # If we don't go around the loop at least once, we might end up
+            # with an empty dict being returned, which shouldn't happen. We
+            # need the (None, None) entries for jobs we can't find.
+            assert end_time >= self.boss.start_time
+
+            results: dict[int, tuple[str | None, int | None]] = {}
+
+            while len(job_id_list) > 0 and end_time >= self.boss.start_time:
+                # There are still jobs to look for and our search isn't
+                # exclusively for stuff that only existed before our workflow
+                # started.
+                results.update(
+                    self._get_job_details_from_sacct_for_range(
+                        job_id_list,
+                        begin_time,
+                        end_time
+                    )
+                )
+                job_id_list = self._remaining_jobs(job_id_list, results)
+                # If we have to search again, search the previous day. But
+                # overlap a tiny bit so the endpoints don't exactly match, in
+                # case Slurm is not working with inclusive intervals.
+                # TODO: is Slurm working with inclusive intervals?
+                end_time = begin_time + timedelta(seconds=1)
+                begin_time = end_time - timedelta(days=1, seconds=1)
+
+
+            if end_time < self.boss.start_time and len(job_id_list) > 0:
+                # This is suspicious.
+                logger.warning(
+                    "Could not find any information from sacct after "
+                    "workflow start at %s about jobs: %s",
+                    self.boss.start_time.isoformat(),
+                    job_id_list
+                )
+
+            return results
+
+        def _get_job_details_from_sacct_for_range(
+            self,
+            job_id_list: list[int],
+            begin_time: datetime,
+            end_time: datetime,
         ) -> dict[int, tuple[str | None, int | None]]:
             """
             Get SLURM job exit codes for the jobs in `job_id_list` by running `sacct`.
+
+            Internally, Slurm's accounting thinks in wall clock time, so for
+            efficiency you need to only search relevant real-time periods.
+
             :param job_id_list: list of integer batch job IDs.
-            :return: dict of job statuses, where key is the job-id, and value is a tuple
-            containing the job's state and exit code.
+            :param begin_time: An aware datetime of the earliest time to search
+            :param end_time: An aware datetime of the latest time to search
+            :return: dict of job statuses, where key is the job-id, and value
+                is a tuple containing the job's state and exit code. Jobs with
+                no information reported from Slurm will have (None, None).
             """
+
+            assert begin_time.tzinfo is not None, "begin_time must be aware"
+            assert end_time.tzinfo is not None, "end_time must be aware"
+            def stringify(t: datetime) -> str:
+                """
+                Convert an aware time local time, and format it *without* a
+                trailing time zone indicator.
+                """
+                # TODO: What happens when we get an aware time that's ambiguous
+                # in local time? Or when the local timezone changes while we're
+                # sending things to Slurm or doing a progressive search back?
+                naive_t = t.astimezone(None).replace(tzinfo=None)
+                return naive_t.isoformat(timespec="seconds")
+
             job_ids = ",".join(str(id) for id in job_id_list)
             args = [
                 "sacct",
@@ -485,8 +643,10 @@ class SlurmBatchSystem(AbstractGridEngineBatchSystem):
                 "JobIDRaw,State,ExitCode",  # specify output columns
                 "-P",  # separate columns with pipes
                 "-S",
-                "1970-01-01",
-            ]  # override start time limit
+                stringify(begin_time),
+                "-E",
+                stringify(end_time),
+            ]
 
             # Collect the job statuses in a dict; key is the job-id, value is a tuple containing
             # job state and exit status. Initialize dict before processing output of `sacct`.
@@ -500,8 +660,20 @@ class SlurmBatchSystem(AbstractGridEngineBatchSystem):
                     if len(job_id_list) == 1:
                         # 1 is too big, we can't recurse further, bail out
                         raise
-                    job_statuses.update(self._getJobDetailsFromSacct(job_id_list[:len(job_id_list)//2]))
-                    job_statuses.update(self._getJobDetailsFromSacct(job_id_list[len(job_id_list)//2:]))
+                    job_statuses.update(
+                        self._get_job_details_from_sacct_for_range(
+                            job_id_list[:len(job_id_list)//2],
+                            begin_time,
+                            end_time,
+                        )
+                    )
+                    job_statuses.update(
+                        self._get_job_details_from_sacct_for_range(
+                            job_id_list[len(job_id_list)//2:],
+                            begin_time,
+                            end_time,
+                        )
+                    )
                     return job_statuses
                 else:
                     raise
@@ -847,6 +1019,9 @@ class SlurmBatchSystem(AbstractGridEngineBatchSystem):
     ) -> None:
         super().__init__(config, maxCores, maxMemory, maxDisk)
         self.partitions = SlurmBatchSystem.PartitionSet()
+        # Record when the workflow started, so we know when to stop looking for
+        # jobs we ran.
+        self.start_time = datetime.now().astimezone(None)
 
     # Override issuing jobs so we can check if we need to use Slurm's magic
     # whole-node-memory feature.