siliconcompiler 0.35.1__py3-none-any.whl → 0.35.3__py3-none-any.whl

siliconcompiler/scheduler/scheduler.py

@@ -1,14 +1,18 @@
 import io
 import logging
+import multiprocessing
 import os
 import re
 import shutil
 import sys
+import tempfile
 import traceback
 
 import os.path
 
-from typing import Union, TYPE_CHECKING
+from datetime import datetime
+
+from typing import Union, Dict, Optional, Tuple, List, TYPE_CHECKING
 
 from siliconcompiler import NodeStatus
 from siliconcompiler.schema import Journal
@@ -17,7 +21,8 @@ from siliconcompiler.scheduler import SchedulerNode
 from siliconcompiler.scheduler import SlurmSchedulerNode
 from siliconcompiler.scheduler import DockerSchedulerNode
 from siliconcompiler.scheduler import TaskScheduler
-from siliconcompiler.scheduler.schedulernode import SchedulerFlowReset
+from siliconcompiler.scheduler.schedulernode import SchedulerFlowReset, SchedulerNodeReset
+from siliconcompiler.tool import TaskExecutableNotFound, TaskExecutableNotReceived
 
 from siliconcompiler import utils
 from siliconcompiler.utils.logging import SCLoggerFormatter
@@ -56,7 +61,7 @@ class Scheduler:
             SCRuntimeError: If the specified flow is not defined or fails validation.
         """
         self.__project = project
-        self.__logger: logging.Logger = project.logger
+        self.__logger: logging.Logger = project.logger.getChild("scheduler")
         self.__name = project.name
 
         flow = self.__project.get("option", "flow")
@@ -95,13 +100,20 @@ class Scheduler:
         self.__record: "RecordSchema" = self.__project.get("record", field="schema")
         self.__metrics: "MetricSchema" = self.__project.get("metric", field="schema")
 
-        self.__tasks = {}
+        self.__tasks: Dict[Tuple[str, str], SchedulerNode] = {}
 
         # Create dummy handler
         self.__joblog_handler = logging.NullHandler()
         self.__org_job_name = self.__project.get("option", "jobname")
         self.__logfile = None
 
+    @property
+    def manifest(self) -> str:
+        """
+        Returns the path to the job manifest
+        """
+        return os.path.join(jobdir(self.__project), f"{self.__name}.pkg.json")
+
     @property
     def log(self) -> Union[None, str]:
         """
@@ -129,7 +141,7 @@ class Scheduler:
         Args:
             header (str): A header message to print before the status list.
         """
-        self.__logger.debug(f"#### {header}")
+        self.__logger.debug(f"#### {header} : {datetime.now().strftime('%H:%M:%S')}")
         for step, index in self.__flow.get_nodes():
             self.__logger.debug(f"({step}, {index}) -> "
                                 f"{self.__record.get('status', step=step, index=index)}")
@@ -261,6 +273,10 @@ class Scheduler:
             self.__run_setup()
             self.configure_nodes()
 
+            # Verify tool setups
+            if not self.__check_tool_versions():
+                raise SCRuntimeError("Tools did not meet version requirements")
+
             # Verify tool setups
             if not self.__check_tool_requirements():
                 raise SCRuntimeError("Tools requirements not met")
@@ -278,8 +294,7 @@ class Scheduler:
             self.__project._record_history()
 
             # Record final manifest
-            filepath = os.path.join(jobdir(self.__project), f"{self.__name}.pkg.json")
-            self.__project.write_manifest(filepath)
+            self.__project.write_manifest(self.manifest)
 
             send_messages.send(self.__project, 'summary', None, None)
         finally:
@@ -366,6 +381,8 @@ class Scheduler:
         nodes = self.__flow_runtime.get_nodes()
         error = False
 
+        manifest_name = os.path.basename(self.manifest)
+
         for (step, index) in nodes:
             # Get files we receive from input nodes.
             in_nodes = self.__flow_runtime.get_node_inputs(step, index, record=self.__record)
@@ -390,9 +407,7 @@ class Scheduler:
                         inputs = []
                         continue
 
-                    design = self.__project.get("option", 'design')
-                    manifest = f'{design}.pkg.json'
-                    inputs = [inp for inp in os.listdir(in_step_out_dir) if inp != manifest]
+                    inputs = [inp for inp in os.listdir(in_step_out_dir) if inp != manifest_name]
                 else:
                     in_tool = self.__flow.get(in_step, in_index, "tool")
                     in_task = self.__flow.get(in_step, in_index, "task")
@@ -534,6 +549,8 @@ class Scheduler:
             for delfile in os.listdir(cur_job_dir):
                 if delfile == "job.log" and recheck:
                     continue
+                if delfile.startswith("sc_") and recheck:
+                    continue
                 if os.path.isfile(os.path.join(cur_job_dir, delfile)):
                     os.remove(os.path.join(cur_job_dir, delfile))
                 else:
@@ -548,19 +565,17 @@ class Scheduler:
         node's runtime context and invokes its clean_directory method to perform
         node-specific cleanup.
         """
-        protected_dirs = {os.path.basename(collectiondir(self.__project))}
-
         keep_steps = set([step for step, _ in self.__flow.get_nodes()])
         cur_job_dir = jobdir(self.__project)
         for step in os.listdir(cur_job_dir):
-            if step in protected_dirs:
+            if step.startswith("sc_"):
                 continue
             if not os.path.isdir(os.path.join(cur_job_dir, step)):
                 continue
             if step not in keep_steps:
                 shutil.rmtree(os.path.join(cur_job_dir, step))
         for step in os.listdir(cur_job_dir):
-            if step in protected_dirs:
+            if step.startswith("sc_"):
                 continue
             if not os.path.isdir(os.path.join(cur_job_dir, step)):
                 continue
@@ -576,32 +591,23 @@ class Scheduler:
                 with self.__tasks[(step, index)].runtime():
                     self.__tasks[(step, index)].clean_directory()
 
-    def configure_nodes(self) -> None:
-        """
-        Prepare and configure all flow nodes before execution, including loading prior run state,
-        running per-node setup, and marking nodes that require rerun.
+    def __configure_collect_previous_information(self) -> Dict[Tuple[str, str], "Project"]:
+        """Collects information from previous runs for nodes that won't be re-executed.
 
-        This method:
-        - Loads available node manifests from previous jobs and uses them to populate setup data
-          where appropriate.
-        - Runs each node's setup routine to initialize tools and runtime state.
-        - For nodes whose parameters or inputs have changed, marks them and all downstream nodes
-          as pending so they will be re-executed.
-        - Replays preserved journaled results for nodes that remain valid to reuse previous outputs.
-        - On a SchedulerFlowReset, forces a full build-directory recheck and marks every node
-          as pending.
-        - Persists the resulting manifest for the current job before returning.
+        This method identifies nodes that are marked for loading (not cleaning) and
+        are not part of the current 'from' execution path. For each of these
+        nodes, it attempts to load its manifest from a previous run.
+
+        Returns:
+            Dict[Tuple[str, str], "Project"]: A dictionary mapping (step, index)
+                tuples to their corresponding loaded Project objects from
+                previous runs.
         """
         from siliconcompiler import Project
+        self.__print_status("Start - collect")
 
-        from_nodes = []
         extra_setup_nodes = {}
-
-        journal = Journal.access(self.__project)
-        journal.start()
-
-        self.__print_status("Start")
-
+        from_nodes = []
         if self.__project.get('option', 'clean'):
             if self.__project.get("option", "from"):
                 from_nodes = self.__flow_runtime.get_entry_nodes()
@@ -620,16 +626,37 @@ class Scheduler:
                 # Node will be run so no need to load
                 continue
 
-            manifest = os.path.join(workdir(self.__project, step=step, index=index),
-                                    'outputs',
-                                    f'{self.__name}.pkg.json')
+            manifest = self.__tasks[(step, index)].get_manifest()
             if os.path.exists(manifest):
                 # ensure we setup these nodes again
                 try:
                     extra_setup_nodes[(step, index)] = Project.from_manifest(filepath=manifest)
-                except Exception:
+                except Exception as e:
+                    self.__logger.debug(f"Reading {manifest} caused: {e}")
                     pass
 
+        self.__print_status("End - collect")
+
+        return extra_setup_nodes
+
+    def __configure_run_setup(self, extra_setup_nodes: Dict[Tuple[str, str], "Project"]) -> None:
+        """Runs the setup() method for all flow nodes and forwards previous status.
+
+        This method iterates through all nodes in execution order and calls
+        their respective `setup()` methods.
+
+        It also uses the `extra_setup_nodes` to:
+        1. Prune nodes from `extra_setup_nodes` if their `setup()` method
+           returns False (indicating the node is no longer valid).
+        2. Forward the 'status' from a valid, previously-run node (found in
+           `extra_setup_nodes`) into the current job's records.
+
+        Args:
+            extra_setup_nodes (Dict[Tuple[str, str], "Project"]): A dictionary
+                of loaded Project objects from previous runs. This dictionary
+                may be modified in-place (nodes may be removed).
+        """
+        self.__print_status("Start - setup")
         # Setup tools for all nodes to run
         for layer_nodes in self.__flow.get_execution_order():
             for step, index in layer_nodes:
@@ -649,28 +676,155 @@ class Scheduler:
                     if node_status:
                         # Forward old status
                         self.__record.set('status', node_status, step=step, index=index)
+        self.__print_status("End - setup")
+
+    @staticmethod
+    def _configure_run_required(task: SchedulerNode) \
+            -> Optional[Union[SchedulerFlowReset, SchedulerNodeReset]]:
+        """
+        Helper method to run requires_run() with threads.
+        """
+        with task.runtime():
+            try:
+                task.requires_run()
+            except (SchedulerFlowReset, SchedulerNodeReset) as e:
+                return e
+        return None
+
+    def __configure_check_run_required(self) -> List[Tuple[str, str]]:
+        """Checks which nodes require a re-run and which can be replayed.
+
+        This method iterates through all nodes that are currently marked as
+        'SUCCESS' (typically from a previous run). It calls `requires_run()`
+        on each to determine if inputs, parameters, or other dependencies
+        have changed.
+
+        - If `requires_run()` is True, the node is marked as 'pending' (and
+          will be re-executed).
+        - If `requires_run()` is False, the node is added to the 'replay' list,
+          indicating its previous results can be reused.
+
+        Returns:
+            List[Tuple[str, str]]: A list of (step, index) tuples for nodes
+                that do *not* require a re-run and whose results can be
+                replayed from the journal.
+        """
+        self.__print_status("Start - check")
+
+        replay: List[Tuple[str, str]] = []
+
+        nodes: List[Tuple[str, str]] = []
+
+        def filter_nodes(nodes: List[Tuple[str, str]]) -> None:
+            for step, index in tuple(nodes):
+                # Only look at successful nodes
+                if self.__record.get("status", step=step, index=index) != NodeStatus.SUCCESS:
+                    nodes.remove((step, index))
+
+        def create_node_group(nodes: List[Tuple[str, str]], size: int) -> List[Tuple[str, str]]:
+            group = []
+            for _ in range(size):
+                if nodes:
+                    group.append(nodes.pop(0))
+            return group
 
-        self.__print_status("After setup")
+        # Collect initial list of nodes to process
+        for layer_nodes in self.__flow.get_execution_order():
+            nodes.extend(layer_nodes)
+
+        # Determine pool size
+        cores = utils.get_cores()
+        pool_size = self.project.option.scheduler.get_maxthreads() or cores
+        pool_size = max(1, min(cores, pool_size))
+
+        # Limit based on number of nodes if less than number of cores
+        filter_nodes(nodes)
+        if not nodes:
+            # No nodes left so just return
+            return []
+
+        pool_size = min(pool_size, len(nodes))
+
+        self.__logger.debug(f"Check pool size: {pool_size}")
+
+        # Call this in case this was invoked without __main__
+        multiprocessing.freeze_support()
+
+        with multiprocessing.get_context("spawn").Pool(pool_size) as pool:
+            while True:
+                # Filter nodes
+                filter_nodes(nodes)
+
+                # Generate a group of nodes to run
+                group = create_node_group(nodes, pool_size)
+                self.__logger.debug(f"Group to check: {group}")
+                if not group:
+                    # Group is empty
+                    break
+
+                tasks = [self.__tasks[(step, index)] for step, index in group]
+                # Suppress excess info messages during checks
+                cur_level = self.project.logger.level
+                self.project.logger.setLevel(logging.WARNING)
+                try:
+                    runcheck = pool.map(Scheduler._configure_run_required, tasks)
+                finally:
+                    self.project.logger.setLevel(cur_level)
+
+                for node, runrequired in zip(group, runcheck):
+                    if self.__record.get("status", step=node[0], index=node[1]) != \
+                            NodeStatus.SUCCESS:
+                        continue
+
+                    self.__logger.debug(f"  Result: {node} -> {runrequired}")
+
+                    if runrequired is not None:
+                        runrequired.log(self.__logger)
+
+                        if isinstance(runrequired, SchedulerFlowReset):
+                            raise runrequired from None
+
+                        # This node must be run
+                        self.__mark_pending(*node)
+                    else:
+                        # import old information
+                        replay.append(node)
+
+        self.__print_status("End - check")
+
+        return replay
+
+    def configure_nodes(self) -> None:
+        """
+        Prepare and configure all flow nodes before execution, including loading prior run state,
+        running per-node setup, and marking nodes that require rerun.
+
+        This method:
+        - Loads available node manifests from previous jobs and uses them to populate setup data
+          where appropriate.
+        - Runs each node's setup routine to initialize tools and runtime state.
+        - For nodes whose parameters or inputs have changed, marks them and all downstream nodes
+          as pending so they will be re-executed.
+        - Replays preserved journaled results for nodes that remain valid to reuse previous outputs.
+        - On a SchedulerFlowReset, forces a full build-directory recheck and marks every node
+          as pending.
+        - Persists the resulting manifest for the current job before returning.
+        """
+        journal = Journal.access(self.__project)
+        journal.start()
+
+        extra_setup_nodes = self.__configure_collect_previous_information()
+
+        self.__configure_run_setup(extra_setup_nodes)
 
         # Check for modified information
         try:
-            replay = []
-            for layer_nodes in self.__flow.get_execution_order():
-                for step, index in layer_nodes:
-                    # Only look at successful nodes
-                    if self.__record.get("status", step=step, index=index) != NodeStatus.SUCCESS:
-                        continue
+            replay = self.__configure_check_run_required()
 
-                    with self.__tasks[(step, index)].runtime():
-                        if self.__tasks[(step, index)].requires_run():
-                            # This node must be run
-                            self.__mark_pending(step, index)
-                        elif (step, index) in extra_setup_nodes:
-                            # import old information
-                            replay.append((step, index))
             # Replay previous information
             for step, index in replay:
-                Journal.access(extra_setup_nodes[(step, index)]).replay(self.__project)
+                if (step, index) in extra_setup_nodes:
+                    Journal.access(extra_setup_nodes[(step, index)]).replay(self.__project)
         except SchedulerFlowReset:
             # Mark all nodes as pending
             self.__clean_build_dir_full(recheck=True)
@@ -678,7 +832,7 @@ class Scheduler:
             for step, index in self.__flow.get_nodes():
                 self.__mark_pending(step, index)
 
-        self.__print_status("After requires run")
+        self.__print_status("Before ensure")
 
         # Ensure all nodes are marked as pending if needed
         for layer_nodes in self.__flow_runtime.get_execution_order():
@@ -687,11 +841,12 @@ class Scheduler:
                 if NodeStatus.is_waiting(status) or NodeStatus.is_error(status):
                     self.__mark_pending(step, index)
 
-        self.__print_status("After ensure")
+        self.__print_status("FINAL")
+
+        # Write configured manifest
+        os.makedirs(os.path.dirname(self.manifest), exist_ok=True)
+        self.__project.write_manifest(self.manifest)
 
-        os.makedirs(jobdir(self.__project), exist_ok=True)
-        self.__project.write_manifest(os.path.join(jobdir(self.__project),
-                                                   f"{self.__name}.pkg.json"))
         journal.stop()
 
     def __check_display(self) -> None:
@@ -740,3 +895,73 @@ class Scheduler:
             self.__project.set('option', 'jobname', f'{stem}{jobid + 1}')
             return True
         return False
+
+    def __check_tool_versions(self) -> bool:
+        """
+        Validates tool executables and versions for all local nodes.
+
+        This method iterates through all nodes defined in the flow runtime.
+        It performs checks only for nodes scheduled to run locally. Nodes
+        configured to run on a remote scheduler (e.g., LSF, Slurm) are
+        skipped. The entire check is also skipped if the project is
+        configured for remote execution.
+
+        For each local node, it:
+        1.  Runs from within a temporary directory to avoid conflicts.
+        2.  Enters the node's specific runtime context (e.g., sets env vars).
+        3.  Tries to resolve the executable path. Logs an error if not found.
+        4.  Calls `node.check_version()` to validate the tool version.
+        5.  Caches version results to avoid re-checking the same executable
+            for different nodes.
+
+        It logs an error for any node that fails validation (missing executable
+        or failed version check) and returns an overall status.
+
+        Returns:
+            bool: True if all local nodes pass validation or if checks
+                are skipped. False if any local node fails.
+        """
+        if self.__project.option.get_remote():
+            return True
+
+        error = False
+
+        cwd = os.getcwd()
+        with tempfile.TemporaryDirectory(prefix="sc_tool_check") as d:
+            try:
+                versions: Dict[str, Optional[str]] = {}
+
+                self.__logger.debug(f"Executing tool checks in: {d}")
+                os.chdir(d)
+                for (step, index) in self.__flow_runtime.get_nodes():
+                    if self.__project.option.scheduler.get_name(step=step, index=index) is not None:
+                        continue
+
+                    node = SchedulerNode(self.__project, step, index)
+                    with node.runtime():
+                        try:
+                            exe = node.get_exe_path()
+                        except TaskExecutableNotReceived:
+                            continue
+                        except TaskExecutableNotFound:
+                            exe = node.task.get("exe")
+                            self.__logger.error(f"Executable for {step}/{index} could not "
+                                                f"be found: {exe}")
+                            error = True
+                            continue
+
+                        try:
+                            if exe:
+                                version: Optional[str] = versions.get(exe, None)
+                                version, check = node.check_version(version)
+                                versions[exe] = version
+                                if not check:
+                                    self.__logger.error(f"Executable for {step}/{index} did not "
+                                                        "meet version checks")
+                                    error = True
+                        except NotImplementedError:
+                            self.__logger.error(f"Unable to process version for {step}/{index}")
+            finally:
+                os.chdir(cwd)
+
+        return not error