toil 7.0.0__py3-none-any.whl → 8.0.0__py3-none-any.whl

toil/serviceManager.py

@@ -15,9 +15,10 @@
 
 import logging
 import time
+from collections.abc import Iterable
 from queue import Empty, Queue
 from threading import Event, Thread
-from typing import Iterable, Optional, Set
+from typing import Optional
 
 from toil.job import ServiceJobDescription
 from toil.jobStores.abstractJobStore import AbstractJobStore
@@ -40,7 +41,7 @@ class ServiceManager:
 
         # These are all the client jobs that are waiting for their services to
         # start.
-        self.__waiting_clients: Set[str] = set()
+        self.__waiting_clients: set[str] = set()
 
         # This is used to terminate the thread associated with the service
         # manager
@@ -123,7 +124,9 @@ class ServiceManager:
             client_id = self.__clients_out.get(timeout=maxWait)
             self.__waiting_clients.remove(client_id)
             if self.__service_manager_jobs < 0:
-                raise RuntimeError("The number of jobs scheduled by the service manager cannot be negative.")
+                raise RuntimeError(
+                    "The number of jobs scheduled by the service manager cannot be negative."
+                )
             self.__service_manager_jobs -= 1
             return client_id
         except Empty:
@@ -141,7 +144,9 @@ class ServiceManager:
             client_id = self.__failed_clients_out.get(timeout=maxWait)
             self.__waiting_clients.remove(client_id)
             if self.__service_manager_jobs < 0:
-                raise RuntimeError("The number of jobs scheduled by the service manager cannot be negative.")
+                raise RuntimeError(
+                    "The number of jobs scheduled by the service manager cannot be negative."
+                )
             self.__service_manager_jobs -= 1
             return client_id
         except Empty:
@@ -157,7 +162,9 @@ class ServiceManager:
         try:
             service_id = self.__services_out.get(timeout=maxWait)
             if self.__service_manager_jobs < 0:
-                raise RuntimeError("The number of jobs scheduled by the service manager cannot be negative.")
+                raise RuntimeError(
+                    "The number of jobs scheduled by the service manager cannot be negative."
+                )
             self.__service_manager_jobs -= 1
             return service_id
         except Empty:
@@ -226,7 +233,7 @@ class ServiceManager:
 
         Will block until all services are started and blocked.
         """
-        logger.debug('Waiting for service manager thread to finish ...')
+        logger.debug("Waiting for service manager thread to finish ...")
         start_time = time.time()
         self.__terminate.set()
         self.__service_starter.join()
@@ -251,13 +258,17 @@ class ServiceManager:
         while True:
             with throttle(1.0):
                 if self.__terminate.is_set():
-                    logger.debug('Received signal to quit starting services.')
+                    logger.debug("Received signal to quit starting services.")
                     break
                 try:
                     client_id = self.__clients_in.get_nowait()
                     client = self.__toil_state.get_job(client_id)
                     host_id_batches = list(client.serviceHostIDsInBatches())
-                    logger.debug("Service manager processing client %s with %d batches of services", client, len(host_id_batches))
+                    logger.debug(
+                        "Service manager processing client %s with %d batches of services",
+                        client,
+                        len(host_id_batches),
+                    )
                     if len(host_id_batches) > 1:
                         # Have to fall back to the old blocking behavior to
                         # ensure entire service "groups" are issued as a whole.
@@ -288,7 +299,7 @@ class ServiceManager:
 
                 pending_service_count = len(starting_services)
                 if pending_service_count > 0 and log_limiter.throttle(False):
-                    logger.debug('%d services are starting...', pending_service_count)
+                    logger.debug("%d services are starting...", pending_service_count)
 
                 for service_id in list(starting_services):
                     service_job_desc = self._get_service_job(service_id)
@@ -297,7 +308,9 @@ class ServiceManager:
                         or service_job_desc.errorJobStoreID is None
                     ):
                         raise Exception("Must be a registered ServiceJobDescription")
-                    if not self.__job_store.file_exists(service_job_desc.startJobStoreID):
+                    if not self.__job_store.file_exists(
+                        service_job_desc.startJobStoreID
+                    ):
                         # Service has started (or failed)
                         logger.debug(
                             "Service %s has removed %s and is therefore started",
@@ -308,9 +321,13 @@ class ServiceManager:
                         client_id = service_to_client[service_id]
                         remaining_services_by_client[client_id] -= 1
                         if remaining_services_by_client[client_id] < 0:
-                            raise RuntimeError("The number of remaining services cannot be negative.")
+                            raise RuntimeError(
+                                "The number of remaining services cannot be negative."
+                            )
                         del service_to_client[service_id]
-                        if not self.__job_store.file_exists(service_job_desc.errorJobStoreID):
+                        if not self.__job_store.file_exists(
+                            service_job_desc.errorJobStoreID
+                        ):
                             logger.error(
                                 "Service %s has immediately failed before it could be used",
                                 service_job_desc,
@@ -321,13 +338,22 @@ class ServiceManager:
 
                 # Find if any clients have had *all* their services started.
                 ready_clients = set()
-                for client_id, remainingServices in remaining_services_by_client.items():
+                for (
+                    client_id,
+                    remainingServices,
+                ) in remaining_services_by_client.items():
                     if remainingServices == 0:
                         if client_id in clients_with_failed_services:
-                            logger.error('Job %s has had all its services try to start, but at least one failed', self.__toil_state.get_job(client_id))
+                            logger.error(
+                                "Job %s has had all its services try to start, but at least one failed",
+                                self.__toil_state.get_job(client_id),
+                            )
                             self.__failed_clients_out.put(client_id)
                         else:
-                            logger.debug('Job %s has all its services started', self.__toil_state.get_job(client_id))
+                            logger.debug(
+                                "Job %s has all its services started",
+                                self.__toil_state.get_job(client_id),
+                            )
                             self.__clients_out.put(client_id)
                         ready_clients.add(client_id)
                 for client_id in ready_clients:
@@ -344,7 +370,9 @@ class ServiceManager:
 
         # Start the service jobs in batches, waiting for each batch
         # to become established before starting the next batch
-        for service_job_list in self.__toil_state.get_job(client_id).serviceHostIDsInBatches():
+        for service_job_list in self.__toil_state.get_job(
+            client_id
+        ).serviceHostIDsInBatches():
             # When we get the job descriptions we store them here to go over them again.
             wait_on = []
             for service_id in service_job_list:
@@ -361,9 +389,13 @@ class ServiceManager:
                     service_job_desc.startJobStoreID,
                 )
                 if not self.__job_store.file_exists(service_job_desc.startJobStoreID):
-                    raise RuntimeError(f"Service manager attempted to start service {service_job_desc} that has already started")
+                    raise RuntimeError(
+                        f"Service manager attempted to start service {service_job_desc} that has already started"
+                    )
                 if not self.__toil_state.job_exists(str(service_job_desc.jobStoreID)):
-                    raise RuntimeError(f"Service manager attempted to start service {service_job_desc} that is not in the job store")
+                    raise RuntimeError(
+                        f"Service manager attempted to start service {service_job_desc} that is not in the job store"
+                    )
                 # At this point the terminateJobStoreID and errorJobStoreID
                 # could have been deleted, since the service can be killed at
                 # any time! So we can't assert their presence here.
@@ -382,7 +414,7 @@ class ServiceManager:
                     time.sleep(1.0)
 
                     if log_limiter.throttle(False):
-                        logger.info('Service %s is starting...', service_job_desc)
+                        logger.info("Service %s is starting...", service_job_desc)
 
                     # Check if the thread should quit
                     if self.__terminate.is_set():
@@ -395,9 +427,14 @@ class ServiceManager:
                     ):
                         # The service job has gone away but the service never flipped its start flag.
                         # That's not what the worker is supposed to do when running a service at all.
-                        logger.error('Service %s has completed and been removed without ever starting', service_job_desc)
+                        logger.error(
+                            "Service %s has completed and been removed without ever starting",
+                            service_job_desc,
+                        )
                         # Stop everything.
-                        raise RuntimeError(f"Service {service_job_desc} is in an inconsistent state")
+                        raise RuntimeError(
+                            f"Service {service_job_desc} is in an inconsistent state"
+                        )
 
                 # We don't bail out early here.
 
@@ -409,6 +446,5 @@ class ServiceManager:
                 # though, so they should stop immediately when we run them. TODO:
                 # this is a bad design!
 
-
         # Add the JobDescription to the output queue of jobs whose services have been started
         self.__clients_out.put(client_id)

toil/statsAndLogging.py

@@ -20,7 +20,7 @@ import time
 from argparse import ArgumentParser, Namespace
 from logging.handlers import RotatingFileHandler
 from threading import Event, Thread
-from typing import IO, TYPE_CHECKING, Any, Callable, List, Optional, Union
+from typing import IO, TYPE_CHECKING, Any, Callable, Optional, Union
 
 from toil.lib.conversions import strtobool
 from toil.lib.expando import Expando
@@ -32,27 +32,36 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 root_logger = logging.getLogger()
-toil_logger = logging.getLogger('toil')
+toil_logger = logging.getLogger("toil")
 
 DEFAULT_LOGLEVEL = logging.INFO
 __loggingFiles = []
 
+# We have some logging that belongs at a TRACE level, below DEBUG
+TRACE = logging.DEBUG - 5
+
+logging.addLevelName(TRACE, "TRACE")
+
 
 class StatsAndLogging:
     """A thread to aggregate statistics and logging."""
 
-    def __init__(self, jobStore: 'AbstractJobStore', config: 'Config') -> None:
+    def __init__(self, jobStore: "AbstractJobStore", config: "Config") -> None:
         self._stop = Event()
-        self._worker = Thread(target=self.statsAndLoggingAggregator,
-                              args=(jobStore, self._stop, config),
-                              daemon=True)
+        self._worker = Thread(
+            target=self.statsAndLoggingAggregator,
+            args=(jobStore, self._stop, config),
+            daemon=True,
+        )
 
     def start(self) -> None:
         """Start the stats and logging thread."""
         self._worker.start()
 
     @classmethod
-    def formatLogStream(cls, stream: Union[IO[str], IO[bytes]], stream_name: str) -> str:
+    def formatLogStream(
+        cls, stream: Union[IO[str], IO[bytes]], stream_name: str
+    ) -> str:
         """
         Given a stream of text or bytes, and the job name, job itself, or some
         other optional stringifyable identity info for the job, return a big
@@ -65,21 +74,25 @@ class StatsAndLogging:
 
         :param stream: The stream of text or bytes to print for the user.
         """
-        lines = [f'{stream_name} follows:', '=========>']
+        lines = [f"{stream_name} follows:", "=========>"]
 
         for line in stream:
             if isinstance(line, bytes):
-                line = line.decode('utf-8', errors='replace')
-            lines.append('\t' + line.rstrip('\n'))
+                line = line.decode("utf-8", errors="replace")
+            lines.append("\t" + line.rstrip("\n"))
 
-        lines.append('<=========')
+        lines.append("<=========")
 
-        return '\n'.join(lines)
+        return "\n".join(lines)
 
     @classmethod
-    def logWithFormatting(cls, stream_name: str, jobLogs: Union[IO[str], IO[bytes]],
-                          method: Callable[[str], None] = logger.debug,
-                          message: Optional[str] = None) -> None:
+    def logWithFormatting(
+        cls,
+        stream_name: str,
+        jobLogs: Union[IO[str], IO[bytes]],
+        method: Callable[[str], None] = logger.debug,
+        message: Optional[str] = None,
+    ) -> None:
         if message is not None:
             method(message)
 
@@ -87,28 +100,36 @@ class StatsAndLogging:
         method(cls.formatLogStream(jobLogs, stream_name))
 
     @classmethod
-    def writeLogFiles(cls, jobNames: List[str], jobLogList: List[str], config: 'Config', failed: bool = False) -> None:
-        def createName(logPath: str, jobName: str, logExtension: str, failed: bool = False) -> str:
-            logName = jobName.replace('-', '--')
-            logName = logName.replace('/', '-')
-            logName = logName.replace(' ', '_')
-            logName = logName.replace("'", '')
-            logName = logName.replace('"', '')
+    def writeLogFiles(
+        cls,
+        jobNames: list[str],
+        jobLogList: list[str],
+        config: "Config",
+        failed: bool = False,
+    ) -> None:
+        def createName(
+            logPath: str, jobName: str, logExtension: str, failed: bool = False
+        ) -> str:
+            logName = jobName.replace("-", "--")
+            logName = logName.replace("/", "-")
+            logName = logName.replace(" ", "_")
+            logName = logName.replace("'", "")
+            logName = logName.replace('"', "")
             # Add a "failed_" prefix to logs from failed jobs.
-            logName = ('failed_' if failed else '') + logName
+            logName = ("failed_" if failed else "") + logName
             counter = 0
             while True:
-                suffix = '_' + str(counter).zfill(3) + logExtension
+                suffix = "_" + str(counter).zfill(3) + logExtension
                 fullName = os.path.join(logPath, logName + suffix)
                 #  The maximum file name size in the default HFS+ file system is 255 UTF-16 encoding units, so basically 255 characters
                 if len(fullName) >= 255:
-                    return fullName[:(255 - len(suffix))] + suffix
+                    return fullName[: (255 - len(suffix))] + suffix
                 if not os.path.exists(fullName):
                     return fullName
                 counter += 1
 
         mainFileName = jobNames[0]
-        extension = '.log'
+        extension = ".log"
         writeFn: Callable[..., Any]
         if config.writeLogs:
             path = config.writeLogs
@@ -116,7 +137,7 @@ class StatsAndLogging:
         elif config.writeLogsGzip:
             path = config.writeLogsGzip
             writeFn = gzip.open
-            extension += '.gz'
+            extension += ".gz"
         else:
             # we don't have anywhere to write the logs, return now
             return
@@ -125,13 +146,13 @@ class StatsAndLogging:
         os.makedirs(path, exist_ok=True)
 
         fullName = createName(path, mainFileName, extension, failed)
-        with writeFn(fullName, 'wb') as f:
+        with writeFn(fullName, "wb") as f:
             for l in jobLogList:
                 if isinstance(l, bytes):
-                    l = l.decode('utf-8')
-                if not l.endswith('\n'):
-                    l += '\n'
-                f.write(l.encode('utf-8'))
+                    l = l.decode("utf-8")
+                if not l.endswith("\n"):
+                    l += "\n"
+                f.write(l.encode("utf-8"))
         for alternateName in jobNames[1:]:
             # There are chained jobs in this output - indicate this with a symlink
             # of the job's name to this file
@@ -140,7 +161,9 @@ class StatsAndLogging:
                 os.symlink(os.path.relpath(fullName, path), name)
 
     @classmethod
-    def statsAndLoggingAggregator(cls, jobStore: 'AbstractJobStore', stop: Event, config: 'Config') -> None:
+    def statsAndLoggingAggregator(
+        cls, jobStore: "AbstractJobStore", stop: Event, config: "Config"
+    ) -> None:
         """
         The following function is used for collating stats/reporting log messages from the workers.
         Works inside of a thread, collates as long as the stop flag is not True.
@@ -165,9 +188,12 @@ class StatsAndLogging:
                 pass
             else:
                 for message in logs:
-                    logger.log(int(message.level),
-                               'Got message from job at time %s: %s',
-                               time.strftime('%m-%d-%Y %H:%M:%S'), message.text)
+                    logger.log(
+                        int(message.level),
+                        "Got message from job at time %s: %s",
+                        time.strftime("%m-%d-%Y %H:%M:%S"),
+                        message.text,
+                    )
 
             try:
                 # Handle all the user-level text streams reported back (command output, etc.)
@@ -198,8 +224,11 @@ class StatsAndLogging:
                 # we may have multiple jobs per worker
                 jobNames = logs.names
                 messages = logs.messages
-                cls.logWithFormatting(f'Log from job "{jobNames[0]}"', messages,
-                                      message='Received Toil worker log. Disable debug level logging to hide this output')
+                cls.logWithFormatting(
+                    f'Log from job "{jobNames[0]}"',
+                    messages,
+                    message="Received Toil worker log. Disable debug level logging to hide this output",
+                )
                 cls.writeLogFiles(jobNames, messages, config=config)
 
         while True:
@@ -211,8 +240,13 @@ class StatsAndLogging:
                 time.sleep(0.5)  # Avoid cycling too fast
 
         # Finish the stats file
-        text = json.dumps(dict(total_time=str(time.time() - startTime),
-                               total_clock=str(ResourceMonitor.get_total_cpu_time() - startClock)), ensure_ascii=True)
+        text = json.dumps(
+            dict(
+                total_time=str(time.time() - startTime),
+                total_clock=str(ResourceMonitor.get_total_cpu_time() - startClock),
+            ),
+            ensure_ascii=True,
+        )
         jobStore.write_logs(text)
 
     def check(self) -> None:
@@ -225,11 +259,14 @@ class StatsAndLogging:
 
     def shutdown(self) -> None:
         """Finish up the stats/logging aggregation thread."""
-        logger.debug('Waiting for stats and logging collator thread to finish ...')
+        logger.debug("Waiting for stats and logging collator thread to finish ...")
         startTime = time.time()
         self._stop.set()
         self._worker.join()
-        logger.debug('... finished collating stats and logs. Took %s seconds', time.time() - startTime)
+        logger.debug(
+            "... finished collating stats and logs. Took %s seconds",
+            time.time() - startTime,
+        )
         # in addition to cleaning on exceptions, onError should clean if there are any failed jobs
 
 
@@ -250,6 +287,11 @@ def install_log_color(set_logger: Optional[logging.Logger] = None) -> None:
     import coloredlogs  # type: ignore[import-untyped]
 
     level_styles = dict(coloredlogs.DEFAULT_LEVEL_STYLES)
+    level_styles["trace"] = dict(level_styles["debug"])
+
+    # TODO: What if these fixed colors aren't right for the terminal background?
+    # It might be light or dark or even grey.
+    level_styles["trace"]["color"] = 242
     level_styles["debug"]["color"] = 242
     level_styles["notice"] = {"color": "green", "bold": True}
     level_styles["error"]["bold"] = True
@@ -272,7 +314,9 @@ def install_log_color(set_logger: Optional[logging.Logger] = None) -> None:
     )
 
 
-def add_logging_options(parser: ArgumentParser, default_level: Optional[int] = None) -> None:
+def add_logging_options(
+    parser: ArgumentParser, default_level: Optional[int] = None
+) -> None:
     """
     Add logging options to set the global log level.
 
@@ -285,23 +329,51 @@ def add_logging_options(parser: ArgumentParser, default_level: Optional[int] = N
 
     group = parser.add_argument_group("Logging Options")
 
-    levels = ['Critical', 'Error', 'Warning', 'Debug', 'Info']
+    levels = ["Critical", "Error", "Warning", "Info", "Debug", "Trace"]
     for level in levels:
-        group.add_argument(f"--log{level}", dest="logLevel", default=default_level_name, action="store_const",
-                           const=level, help=f"Turn on loglevel {level}.  Default: {default_level_name}.")
+        group.add_argument(
+            f"--log{level}",
+            dest="logLevel",
+            default=default_level_name,
+            action="store_const",
+            const=level,
+            help=f"Set logging level to {level}. Default: {default_level_name}.",
+        )
 
     levels += [l.lower() for l in levels] + [l.upper() for l in levels]
-    group.add_argument("--logOff", dest="logLevel", default=default_level_name,
-                       action="store_const", const="CRITICAL", help="Same as --logCRITICAL.")
+    group.add_argument(
+        "--logOff",
+        dest="logLevel",
+        default=default_level_name,
+        action="store_const",
+        const="CRITICAL",
+        help="Same as --logCritical.",
+    )
     # Maybe deprecate the above in favor of --logLevel?
 
-    group.add_argument("--logLevel", dest="logLevel", default=default_level_name, choices=levels,
-                       help=f"Set the log level. Default: {default_level_name}.  Options: {levels}.")
+    group.add_argument(
+        "--logLevel",
+        dest="logLevel",
+        default=default_level_name,
+        choices=levels,
+        help=f"Set the log level. Default: {default_level_name}.  Options: {levels}.",
+    )
     group.add_argument("--logFile", dest="logFile", help="File to log in.")
-    group.add_argument("--rotatingLogging", dest="logRotating", action="store_true", default=False,
-                       help="Turn on rotating logging, which prevents log files from getting too big.")
-    group.add_argument("--logColors", dest="colored_logs", default=True, type=strtobool, metavar="BOOL",
-                       help="Enable or disable colored logging. Default: %(default)s")
+    group.add_argument(
+        "--rotatingLogging",
+        dest="logRotating",
+        action="store_true",
+        default=False,
+        help="Turn on rotating logging, which prevents log files from getting too big.",
+    )
+    group.add_argument(
+        "--logColors",
+        dest="colored_logs",
+        default=True,
+        type=strtobool,
+        metavar="BOOL",
+        help="Enable or disable colored logging. Default: %(default)s",
+    )
 
 
 def configure_root_logger() -> None:
@@ -311,8 +383,10 @@ def configure_root_logger() -> None:
     Should be called before any entry point tries to log anything,
     to ensure consistent formatting.
     """
-    logging.basicConfig(format='[%(asctime)s] [%(threadName)-10s] [%(levelname).1s] [%(name)s] %(message)s',
-                        datefmt='%Y-%m-%dT%H:%M:%S%z')
+    logging.basicConfig(
+        format="[%(asctime)s] [%(threadName)-10s] [%(levelname).1s] [%(name)s] %(message)s",
+        datefmt="%Y-%m-%dT%H:%M:%S%z",
+    )
     root_logger.setLevel(DEFAULT_LOGLEVEL)
 
 
@@ -330,12 +404,16 @@ def log_to_file(log_file: Optional[str], log_rotation: bool) -> None:
 
 def set_logging_from_options(options: Union["Config", Namespace]) -> None:
     configure_root_logger()
-    options.logLevel = options.logLevel or logging.getLevelName(root_logger.getEffectiveLevel())
+    options.logLevel = options.logLevel or logging.getLevelName(
+        root_logger.getEffectiveLevel()
+    )
     set_log_level(options.logLevel)
     if options.colored_logs:
         install_log_color()
-    logger.debug(f"Root logger is at level '{logging.getLevelName(root_logger.getEffectiveLevel())}', "
-                 f"'toil' logger at level '{logging.getLevelName(toil_logger.getEffectiveLevel())}'.")
+    logger.debug(
+        f"Root logger is at level '{logging.getLevelName(root_logger.getEffectiveLevel())}', "
+        f"'toil' logger at level '{logging.getLevelName(toil_logger.getEffectiveLevel())}'."
+    )
 
     # start logging to log file if specified
     log_to_file(options.logFile, options.logRotating)
@@ -353,18 +431,24 @@ def suppress_exotic_logging(local_logger: str) -> None:
     This is important because some packages, particularly boto3, are not always instantiated yet in the
     environment when this is run, and so we create the logger and set the level preemptively.
     """
-    never_suppress = ['toil', '__init__', '__main__', 'toil-rt', 'cwltool']
-    always_suppress = ['boto3', 'boto', 'botocore']  # ensure we suppress even before instantiated
+    never_suppress = ["toil", "__init__", "__main__", "toil-rt", "cwltool"]
+    always_suppress = [
+        "boto3",
+        "boto",
+        "botocore",
+    ]  # ensure we suppress even before instantiated
 
-    top_level_loggers: List[str] = []
+    top_level_loggers: list[str] = []
 
     # Due to https://stackoverflow.com/questions/61683713
     for pkg_logger in list(logging.Logger.manager.loggerDict.keys()) + always_suppress:
         if pkg_logger != local_logger:
             # many sub-loggers may exist, like "boto.a", "boto.b", "boto.c"; we only want the top_level: "boto"
-            top_level_logger = pkg_logger.split('.')[0] if '.' in pkg_logger else pkg_logger
+            top_level_logger = (
+                pkg_logger.split(".")[0] if "." in pkg_logger else pkg_logger
+            )
 
             if top_level_logger not in top_level_loggers + never_suppress:
                 top_level_loggers.append(top_level_logger)
                 logging.getLogger(top_level_logger).setLevel(logging.CRITICAL)
-    logger.debug(f'Suppressing the following loggers: {set(top_level_loggers)}')
+    logger.debug(f"Suppressing the following loggers: {set(top_level_loggers)}")