toil 7.0.0__py3-none-any.whl → 8.1.0b1__py3-none-any.whl

toil/utils/toilStatus.py

@@ -15,13 +15,12 @@
 import logging
 import os
 import sys
-from typing import Any, Dict, List, Optional, Set
+from typing import Any, Optional
 
 from toil.bus import replay_message_bus
-from toil.common import Config, Toil, parser_with_common_options
+from toil.common import Toil, parser_with_common_options
 from toil.job import JobDescription, JobException, ServiceJobDescription
-from toil.jobStores.abstractJobStore import (NoSuchFileException,
-                                             NoSuchJobStoreException)
+from toil.jobStores.abstractJobStore import NoSuchFileException, NoSuchJobStoreException
 from toil.statsAndLogging import StatsAndLogging, set_logging_from_options
 
 logger = logging.getLogger(__name__)
@@ -30,33 +29,59 @@ logger = logging.getLogger(__name__)
 class ToilStatus:
     """Tool for reporting on job status."""
 
-    def __init__(self, jobStoreName: str, specifiedJobs: Optional[List[str]] = None):
+    def __init__(self, jobStoreName: str, specifiedJobs: Optional[list[str]] = None):
         self.jobStoreName = jobStoreName
         self.jobStore = Toil.resumeJobStore(jobStoreName)
 
         if specifiedJobs is None:
-            rootJob = self.fetchRootJob()
-            logger.info('Traversing the job graph gathering jobs. This may take a couple of minutes.')
-            self.jobsToReport = self.traverseJobGraph(rootJob)
+            try:
+                rootJob = self.fetchRootJob()
+                logger.info(
+                    "Traversing the job graph gathering jobs. This may take a couple of minutes."
+                )
+                self.jobsToReport = self.traverseJobGraph(rootJob)
+            except JobException:
+                # Root job isn't set.
+                logger.warning("Workflow does not have a root job (yet? anymore?). Cannot look for jobs.")
+                self.jobsToReport = []
+
         else:
             self.jobsToReport = self.fetchUserJobs(specifiedJobs)
 
         self.message_bus_path = self.jobStore.config.write_messages
+
     def print_dot_chart(self) -> None:
         """Print a dot output graph representing the workflow."""
         print("digraph toil_graph {")
         print("# This graph was created from job-store: %s" % self.jobStoreName)
 
         # Make job IDs to node names map
-        jobsToNodeNames: Dict[str, str] = dict(
-            map(lambda job: (str(job.jobStoreID), job.jobName), self.jobsToReport)
-        )
+        def id_to_name(job_id: str) -> str:
+            """
+            Change a job ID into a GraphViz node name.
+            """
+            replacements = [
+                ("_", "_u_"),
+                ("/", "_s_"),
+                ("-", "_d_")
+            ]
+            result = job_id
+            for char, replacement in replacements:
+                result = result.replace(char, replacement)
+            return result
+        id_strings = [str(job.jobStoreID) for job in self.jobsToReport]
+        jobsToNodeNames = {
+            s: id_to_name(s) for s in id_strings
+        }
 
         # Print the nodes
         for job in set(self.jobsToReport):
             print(
-                '{} [label="{} {}"];'.format(
-                    jobsToNodeNames[str(job.jobStoreID)], job.jobName, job.jobStoreID
+                '{} [label="{} {}" color="{}"];'.format(
+                    jobsToNodeNames[str(job.jobStoreID)],
+                    job.jobName,
+                    job.displayName,
+                    "black" if job.has_body() else "green"
                 )
             )
 
@@ -82,7 +107,11 @@ class ToilStatus:
                 with job.getLogFileHandle(self.jobStore) as fH:
                     # TODO: This looks intended to be machine-readable, but the format is
                     #  unspecified and no escaping is done. But keep these tags around.
-                    print(StatsAndLogging.formatLogStream(fH, stream_name=f"LOG_FILE_OF_JOB:{job} LOG:"))
+                    print(
+                        StatsAndLogging.formatLogStream(
+                            fH, stream_name=f"LOG_FILE_OF_JOB:{job} LOG:"
+                        )
+                    )
             else:
                 print(f"LOG_FILE_OF_JOB: {job} LOG: Job has no log file")
 
@@ -94,22 +123,33 @@ class ToilStatus:
                 children += "\t(CHILD_JOB:%s,PRECEDENCE:%i)" % (childJob, level)
             print(children)
 
-    def printAggregateJobStats(self, properties: List[Set[str]], childNumber: List[int]) -> None:
+    def printAggregateJobStats(
+        self, properties: list[set[str]], childNumber: list[int]
+    ) -> None:
         """
         Prints each job's ID, log file, remaining tries, and other properties.
 
         :param properties: A set of string flag names for each job in self.jobsToReport.
         :param childNumber: A list of child counts for each job in self.jobsToReport.
         """
-        for job, job_properties, job_child_number in zip(self.jobsToReport, properties, childNumber):
+        for job, job_properties, job_child_number in zip(
+            self.jobsToReport, properties, childNumber
+        ):
 
             def lf(x: str) -> str:
                 return f"{x}:{str(x in job_properties)}"
+
             # We use a sort of not-really-machine-readable key:value TSV format here.
             # But we only include important keys to help the humans, and flags
             # don't have a value, just a key.
             parts = [f"JOB:{job}"]
-            for flag in ["COMPLETELY_FAILED", "READY_TO_RUN", "IS_ZOMBIE", "HAS_SERVICES", "IS_SERVICE"]:
+            for flag in [
+                "COMPLETELY_FAILED",
+                "READY_TO_RUN",
+                "IS_ZOMBIE",
+                "HAS_SERVICES",
+                "IS_SERVICE",
+            ]:
                 if flag in job_properties:
                     parts.append(flag)
             if job.logJobStoreFileID:
@@ -121,7 +161,7 @@ class ToilStatus:
 
             print("\t".join(parts))
 
-    def report_on_jobs(self) -> Dict[str, Any]:
+    def report_on_jobs(self) -> dict[str, Any]:
         """
         Gathers information about jobs such as its child jobs and status.
 
@@ -132,20 +172,20 @@ class ToilStatus:
         hasChildren = []
         readyToRun = []
         zombies = []
-        hasLogFile: List[JobDescription] = []
+        hasLogFile: list[JobDescription] = []
         hasServices = []
-        services: List[ServiceJobDescription] = []
+        services: list[ServiceJobDescription] = []
         completely_failed = []
 
         # These are stats for jobs in self.jobsToReport
-        child_number: List[int] = []
-        properties: List[Set[str]] = []
+        child_number: list[int] = []
+        properties: list[set[str]] = []
 
         # TODO: This mix of semantics is confusing and made per-job status be
         # wrong for multiple years because it was not understood. Redesign it!
 
         for job in self.jobsToReport:
-            job_properties: Set[str] = set()
+            job_properties: set[str] = set()
             if job.logJobStoreFileID is not None:
                 hasLogFile.append(job)
 
@@ -176,16 +216,16 @@ class ToilStatus:
 
         jobStats = {
             # These are lists of the mathcing jobs
-            'hasChildren': hasChildren,
-            'readyToRun': readyToRun,
-            'zombies': zombies,
-            'hasServices': hasServices,
-            'services': services,
-            'hasLogFile': hasLogFile,
-            'completelyFailed': completely_failed,
+            "hasChildren": hasChildren,
+            "readyToRun": readyToRun,
+            "zombies": zombies,
+            "hasServices": hasServices,
+            "services": services,
+            "hasLogFile": hasLogFile,
+            "completelyFailed": completely_failed,
             # These are stats for jobs in self.jobsToReport
-            'properties': properties,
-            'childNumber': child_number
+            "properties": properties,
+            "childNumber": child_number,
         }
         return jobStats
 
@@ -202,21 +242,21 @@ class ToilStatus:
         try:
             jobstore = Toil.resumeJobStore(jobStoreName)
         except NoSuchJobStoreException:
-            return 'QUEUED'
+            return "QUEUED"
         except NoSuchFileException:
-            return 'QUEUED'
+            return "QUEUED"
 
         try:
             pid = jobstore.read_leader_pid()
             try:
                 os.kill(pid, 0)  # Does not kill process when 0 is passed.
             except OSError:  # Process not found, must be done.
-                return 'COMPLETED'
+                return "COMPLETED"
             else:
-                return 'RUNNING'
+                return "RUNNING"
         except NoSuchFileException:
             pass
-        return 'QUEUED'
+        return "QUEUED"
 
     @staticmethod
     def getStatus(jobStoreName: str) -> str:
@@ -235,38 +275,45 @@ class ToilStatus:
         try:
             jobstore = Toil.resumeJobStore(jobStoreName)
         except NoSuchJobStoreException:
-            return 'QUEUED'
+            return "QUEUED"
         except NoSuchFileException:
-            return 'QUEUED'
+            return "QUEUED"
 
         try:
-            with jobstore.read_shared_file_stream('succeeded.log') as successful:
+            with jobstore.read_shared_file_stream("succeeded.log") as successful:
                 pass
-            return 'COMPLETED'
+            return "COMPLETED"
         except NoSuchFileException:
             try:
-                with jobstore.read_shared_file_stream('failed.log') as failed:
+                with jobstore.read_shared_file_stream("failed.log") as failed:
                     pass
-                return 'ERROR'
+                return "ERROR"
             except NoSuchFileException:
                 pass
-        return 'RUNNING'
+        return "RUNNING"
 
-    def print_bus_messages(self) -> None:
+    def print_running_jobs(self) -> None:
         """
-        Goes through bus messages, returns a list of tuples which have correspondence between
-        PID on assigned batch system and
-
         Prints a list of the currently running jobs
         """
 
         print("\nMessage bus path: ", self.message_bus_path)
         if self.message_bus_path is not None:
             if os.path.exists(self.message_bus_path):
-                replayed_messages = replay_message_bus(self.message_bus_path)
-                for key in replayed_messages:
-                    if replayed_messages[key].exit_code != 0:
-                        print(replayed_messages[key])
+                all_job_statuses = replay_message_bus(self.message_bus_path)
+
+                for job_status in all_job_statuses.values():
+                    if job_status.is_running():
+                        status_line = [
+                            f"Job ID {job_status.job_store_id} with name {job_status.name} is running"
+                        ]
+                        if job_status.batch_system != "":
+                            # batch system exists
+                            status_line.append(
+                                f" on {job_status.batch_system} as ID {job_status.external_batch_id}"
+                            )
+                        status_line.append(".")
+                        print("".join(status_line))
             else:
                 print("Message bus file is missing!")
 
@@ -287,10 +334,12 @@ class ToilStatus:
             return self.jobStore.load_root_job()
         except JobException as e:
             logger.info(e)
-            print('Root job is absent. The workflow has may have completed successfully.')
+            print(
+                "Root job is absent. The workflow has may have completed successfully."
+            )
             raise
 
-    def fetchUserJobs(self, jobs: List[str]) -> List[JobDescription]:
+    def fetchUserJobs(self, jobs: list[str]) -> list[JobDescription]:
         """
         Takes a user input array of jobs, verifies that they are in the jobStore
         and returns the array of jobsToReport.
@@ -303,16 +352,16 @@ class ToilStatus:
             try:
                 jobsToReport.append(self.jobStore.load_job(jobID))
             except JobException:
-                print('The job %s could not be found.' % jobID, file=sys.stderr)
+                print("The job %s could not be found." % jobID, file=sys.stderr)
                 raise
         return jobsToReport
 
     def traverseJobGraph(
         self,
         rootJob: JobDescription,
-        jobsToReport: Optional[List[JobDescription]] = None,
-        foundJobStoreIDs: Optional[Set[str]] = None,
-    ) -> List[JobDescription]:
+        jobsToReport: Optional[list[JobDescription]] = None,
+        foundJobStoreIDs: Optional[set[str]] = None,
+    ) -> list[JobDescription]:
         """
         Find all current jobs in the jobStore and return them as an Array.
 
@@ -335,15 +384,24 @@ class ToilStatus:
         jobsToReport.append(rootJob)
         # Traverse jobs in stack
         for successorJobStoreID in rootJob.allSuccessors():
-            if successorJobStoreID not in foundJobStoreIDs and self.jobStore.job_exists(successorJobStoreID):
-                self.traverseJobGraph(self.jobStore.load_job(successorJobStoreID), jobsToReport, foundJobStoreIDs)
+            if (
+                successorJobStoreID not in foundJobStoreIDs
+                and self.jobStore.job_exists(successorJobStoreID)
+            ):
+                self.traverseJobGraph(
+                    self.jobStore.load_job(successorJobStoreID),
+                    jobsToReport,
+                    foundJobStoreIDs,
+                )
 
         # Traverse service jobs
         for jobs in rootJob.services:
             for serviceJobStoreID in jobs:
                 if self.jobStore.job_exists(serviceJobStoreID):
                     if serviceJobStoreID in foundJobStoreIDs:
-                        raise RuntimeError('Service job was unexpectedly found while traversing ')
+                        raise RuntimeError(
+                            "Service job was unexpectedly found while traversing "
+                        )
                     foundJobStoreIDs.add(serviceJobStoreID)
                     jobsToReport.append(self.jobStore.load_job(serviceJobStoreID))
 
@@ -353,40 +411,80 @@ class ToilStatus:
 def main() -> None:
     """Reports the state of a Toil workflow."""
     parser = parser_with_common_options(prog="toil status")
-    parser.add_argument("--failIfNotComplete", action="store_true",
-                        help="Return exit value of 1 if toil jobs not all completed. default=%(default)s",
-                        default=False)
-
-    parser.add_argument("--noAggStats", dest="stats", action="store_false",
-                        help="Do not print overall, aggregate status of workflow.",
-                        default=True)
-
-    parser.add_argument("--dot", "--printDot", dest="print_dot", action="store_true",
-                        help="Print dot formatted description of the graph. If using --jobs will "
-                             "restrict to subgraph including only those jobs. default=%(default)s",
-                        default=False)
-
-    parser.add_argument("--jobs", nargs='+',
-                        help="Restrict reporting to the following jobs (allows subsetting of the report).",
-                        default=None)
-
-    parser.add_argument("--perJob", "--printPerJobStats", dest="print_per_job_stats", action="store_true",
-                        help="Print info about each job. default=%(default)s",
-                        default=False)
-
-    parser.add_argument("--logs", "--printLogs", dest="print_logs", action="store_true",
-                        help="Print the log files of jobs (if they exist). default=%(default)s",
-                        default=False)
-
-    parser.add_argument("--children", "--printChildren", dest="print_children", action="store_true",
-                        help="Print children of each job. default=%(default)s",
-                        default=False)
-
-    parser.add_argument("--status", "--printStatus", dest="print_status", action="store_true",
-                        help="Determine which jobs are currently running and the associated batch system ID")
-
-    parser.add_argument("--failed", "--printFailed", dest="print_failed", action="store_true",
-                        help="List jobs which seem to have failed to run")
+    parser.add_argument(
+        "--failIfNotComplete",
+        action="store_true",
+        help="Return exit value of 1 if toil jobs not all completed. default=%(default)s",
+        default=False,
+    )
+
+    parser.add_argument(
+        "--noAggStats",
+        dest="stats",
+        action="store_false",
+        help="Do not print overall, aggregate status of workflow.",
+        default=True,
+    )
+
+    parser.add_argument(
+        "--dot",
+        "--printDot",
+        dest="print_dot",
+        action="store_true",
+        help="Print dot formatted description of the graph. If using --jobs will "
+        "restrict to subgraph including only those jobs. default=%(default)s",
+        default=False,
+    )
+
+    parser.add_argument(
+        "--jobs",
+        nargs="+",
+        help="Restrict reporting to the following jobs (allows subsetting of the report).",
+        default=None,
+    )
+
+    parser.add_argument(
+        "--perJob",
+        "--printPerJobStats",
+        dest="print_per_job_stats",
+        action="store_true",
+        help="Print info about each job. default=%(default)s",
+        default=False,
+    )
+
+    parser.add_argument(
+        "--logs",
+        "--printLogs",
+        dest="print_logs",
+        action="store_true",
+        help="Print the log files of jobs (if they exist). default=%(default)s",
+        default=False,
+    )
+
+    parser.add_argument(
+        "--children",
+        "--printChildren",
+        dest="print_children",
+        action="store_true",
+        help="Print children of each job. default=%(default)s",
+        default=False,
+    )
+
+    parser.add_argument(
+        "--status",
+        "--printStatus",
+        dest="print_status",
+        action="store_true",
+        help="Determine which jobs are currently running and the associated batch system ID, if any",
+    )
+
+    parser.add_argument(
+        "--failed",
+        "--printFailed",
+        dest="print_failed",
+        action="store_true",
+        help="List jobs which seem to have failed to run",
+    )
 
     options = parser.parse_args()
     set_logging_from_options(options)
@@ -398,7 +496,7 @@ def main() -> None:
     try:
         status = ToilStatus(options.jobStore, options.jobs)
     except NoSuchJobStoreException:
-        print(f'The job store {options.jobStore} was not found.')
+        print(f"The job store {options.jobStore} was not found.")
         return
     except JobException:  # Workflow likely complete, user informed in ToilStatus()
         return
@@ -407,16 +505,16 @@ def main() -> None:
 
     # Info to be reported.
     # These are lists of matching jobs.
-    hasChildren = jobStats['hasChildren']
-    readyToRun = jobStats['readyToRun']
-    zombies = jobStats['zombies']
-    hasServices = jobStats['hasServices']
-    services = jobStats['services']
-    hasLogFile = jobStats['hasLogFile']
-    completely_failed = jobStats['completelyFailed']
+    hasChildren = jobStats["hasChildren"]
+    readyToRun = jobStats["readyToRun"]
+    zombies = jobStats["zombies"]
+    hasServices = jobStats["hasServices"]
+    services = jobStats["services"]
+    hasLogFile = jobStats["hasLogFile"]
+    completely_failed = jobStats["completelyFailed"]
     # These are results for corresponding jobs in status.jobsToReport
-    properties = jobStats['properties']
-    childNumber = jobStats['childNumber']
+    properties = jobStats["properties"]
+    childNumber = jobStats["childNumber"]
 
     if options.print_per_job_stats:
         status.printAggregateJobStats(properties, childNumber)
@@ -431,21 +529,30 @@ def main() -> None:
         for job in completely_failed:
             print(job)
     if options.stats:
-        print('Of the %i jobs considered, '
-              'there are '
-              '%i completely failed jobs, '
-              '%i jobs with children, '
-              '%i jobs ready to run, '
-              '%i zombie jobs, '
-              '%i jobs with services, '
-              '%i services, '
-              'and %i jobs with log files currently in %s.' %
-              (len(status.jobsToReport), len(completely_failed), len(hasChildren),
-               len(readyToRun), len(zombies), len(hasServices), len(services),
-               len(hasLogFile), status.jobStore))
+        print(
+            "Of the %i jobs considered, "
+            "there are "
+            "%i completely failed jobs, "
+            "%i jobs with children, "
+            "%i jobs ready to run, "
+            "%i zombie jobs, "
+            "%i jobs with services, "
+            "%i services, "
+            "and %i jobs with log files currently in %s."
+            % (
+                len(status.jobsToReport),
+                len(completely_failed),
+                len(hasChildren),
+                len(readyToRun),
+                len(zombies),
+                len(hasServices),
+                len(services),
+                len(hasLogFile),
+                status.jobStore,
+            )
+        )
     if options.print_status:
-        status.print_bus_messages()
+        status.print_running_jobs()
     if len(status.jobsToReport) > 0 and options.failIfNotComplete:
         # Upon workflow completion, all jobs will have been removed from job store
         exit(1)
-

toil/utils/toilUpdateEC2Instances.py

@@ -31,7 +31,9 @@ def internet_connection() -> bool:
 
 def main() -> None:
     if not internet_connection():
-        raise RuntimeError('No internet.  Updating the EC2 Instance list requires internet.')
+        raise RuntimeError(
+            "No internet.  Updating the EC2 Instance list requires internet."
+        )
     updateStaticEC2Instances()
 
 

toil/version.py

@@ -1,14 +1,14 @@
-baseVersion = '7.0.0'
+baseVersion = '8.1.0b1'
 cgcloudVersion = '1.6.0a1.dev393'
-version = '7.0.0-d569ea5711eb310ffd5703803f7250ebf7c19576'
+version = '8.1.0b1-4bb05349c027096ab4785259e39b2648118b5dd7'
 cacheTag = 'cache-local-py3.9'
 mainCacheTag = 'cache-master-py3.9'
-distVersion = '7.0.0'
+distVersion = '8.1.0b1'
 exactPython = 'python3.9'
 python = 'python3.9'
-dockerTag = '7.0.0-d569ea5711eb310ffd5703803f7250ebf7c19576-py3.9'
-currentCommit = 'd569ea5711eb310ffd5703803f7250ebf7c19576'
+dockerTag = '8.1.0b1-4bb05349c027096ab4785259e39b2648118b5dd7-py3.9'
+currentCommit = '4bb05349c027096ab4785259e39b2648118b5dd7'
 dockerRegistry = 'quay.io/ucsc_cgl'
 dockerName = 'toil'
 dirty = False
-cwltool_version = '3.1.20240508115724'
+cwltool_version = '3.1.20250110105449'

toil/wdl/utils.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from typing import Iterable
+from collections.abc import Iterable
 
 
 def get_version(iterable: Iterable[str]) -> str:
@@ -22,14 +22,14 @@ def get_version(iterable: Iterable[str]) -> str:
     :return: The WDL version used in the workflow.
     """
     if isinstance(iterable, str):
-        iterable = iterable.split('\n')
+        iterable = iterable.split("\n")
 
     for line in iterable:
         line = line.strip()
         # check if the first non-empty, non-comment line is the version statement
-        if line and not line.startswith('#'):
-            if line.startswith('version '):
+        if line and not line.startswith("#"):
+            if line.startswith("version "):
                 return line[8:].strip()
             break
     # only draft-2 doesn't contain the version declaration
-    return 'draft-2'
+    return "draft-2"