siliconcompiler 0.35.2__py3-none-any.whl → 0.35.4__py3-none-any.whl

siliconcompiler/scheduler/scheduler.py

@@ -1,5 +1,6 @@
 import io
 import logging
+import multiprocessing
 import os
 import re
 import shutil
@@ -9,7 +10,9 @@ import traceback
 
 import os.path
 
-from typing import Union, Dict, Optional, TYPE_CHECKING
+from datetime import datetime
+
+from typing import Union, Dict, Optional, Tuple, List, Set, TYPE_CHECKING
 
 from siliconcompiler import NodeStatus
 from siliconcompiler.schema import Journal
@@ -18,7 +21,7 @@ from siliconcompiler.scheduler import SchedulerNode
 from siliconcompiler.scheduler import SlurmSchedulerNode
 from siliconcompiler.scheduler import DockerSchedulerNode
 from siliconcompiler.scheduler import TaskScheduler
-from siliconcompiler.scheduler.schedulernode import SchedulerFlowReset
+from siliconcompiler.scheduler.schedulernode import SchedulerFlowReset, SchedulerNodeReset
 from siliconcompiler.tool import TaskExecutableNotFound, TaskExecutableNotReceived
 
 from siliconcompiler import utils
@@ -26,6 +29,7 @@ from siliconcompiler.utils.logging import SCLoggerFormatter
 from siliconcompiler.utils.multiprocessing import MPManager
 from siliconcompiler.scheduler import send_messages, SCRuntimeError
 from siliconcompiler.utils.paths import collectiondir, jobdir, workdir
+from siliconcompiler.utils.curation import collect
 
 if TYPE_CHECKING:
     from siliconcompiler.project import Project
@@ -58,7 +62,7 @@ class Scheduler:
             SCRuntimeError: If the specified flow is not defined or fails validation.
         """
         self.__project = project
-        self.__logger: logging.Logger = project.logger
+        self.__logger: logging.Logger = project.logger.getChild("scheduler")
         self.__name = project.name
 
         flow = self.__project.get("option", "flow")
@@ -97,13 +101,35 @@ class Scheduler:
         self.__record: "RecordSchema" = self.__project.get("record", field="schema")
         self.__metrics: "MetricSchema" = self.__project.get("metric", field="schema")
 
-        self.__tasks = {}
+        self.__tasks: Dict[Tuple[str, str], SchedulerNode] = {}
+        self.__skippedtasks: Set[Tuple[str, str]] = set()
 
         # Create dummy handler
         self.__joblog_handler = logging.NullHandler()
         self.__org_job_name = self.__project.get("option", "jobname")
         self.__logfile = None
 
+        # Create tasks
+        for step, index in self.__flow.get_nodes():
+            node_cls = SchedulerNode
+
+            node_scheduler = self.__project.get('option', 'scheduler', 'name',
+                                                step=step, index=index)
+            if node_scheduler == 'slurm':
+                node_cls = SlurmSchedulerNode
+            elif node_scheduler == 'docker':
+                node_cls = DockerSchedulerNode
+            self.__tasks[(step, index)] = node_cls(self.__project, step, index)
+            if self.__flow.get(step, index, "tool") == "builtin":
+                self.__tasks[(step, index)].set_builtin()
+
+    @property
+    def manifest(self) -> str:
+        """
+        Returns the path to the job manifest
+        """
+        return os.path.join(jobdir(self.__project), f"{self.__name}.pkg.json")
+
     @property
     def log(self) -> Union[None, str]:
         """
@@ -131,7 +157,7 @@ class Scheduler:
         Args:
             header (str): A header message to print before the status list.
         """
-        self.__logger.debug(f"#### {header}")
+        self.__logger.debug(f"#### {header} : {datetime.now().strftime('%H:%M:%S')}")
         for step, index in self.__flow.get_nodes():
             self.__logger.debug(f"({step}, {index}) -> "
                                 f"{self.__record.get('status', step=step, index=index)}")
@@ -260,6 +286,9 @@ class Scheduler:
             if not self.check_manifest():
                 raise SCRuntimeError("check_manifest() failed")
 
+            # Initialize schedulers
+            self.__init_schedulers()
+
             self.__run_setup()
             self.configure_nodes()
 
@@ -278,16 +307,23 @@ class Scheduler:
             if not self.__check_flowgraph_io():
                 raise SCRuntimeError("Flowgraph file IO constrains errors")
 
-            self.run_core()
+            # Collect files for remote runs
+            if self.__check_collect_files():
+                collect(self.project)
 
-            # Store run in history
-            self.__project._record_history()
+            try:
+                self.run_core()
+            except SCRuntimeError as e:
+                raise e
 
-            # Record final manifest
-            filepath = os.path.join(jobdir(self.__project), f"{self.__name}.pkg.json")
-            self.__project.write_manifest(filepath)
+            finally:
+                # Store run in history
+                self.__project._record_history()
 
-            send_messages.send(self.__project, 'summary', None, None)
+                # Record final manifest
+                self.__project.write_manifest(self.manifest)
+
+                send_messages.send(self.__project, 'summary', None, None)
         finally:
             if self.__joblog_handler is not None:
                 self.__logger.removeHandler(self.__joblog_handler)
@@ -310,50 +346,10 @@ class Scheduler:
         error = False
 
         for (step, index) in self.__flow_runtime.get_nodes():
-            scheduler = self.__project.option.scheduler.get_name(step=step, index=index)
-            check_file_access = not self.__project.option.get_remote() and scheduler is None
-
-            node = SchedulerNode(self.__project, step, index)
-            requires = []
-            with node.runtime():
-                requires = node.task.get('require')
-
-            for item in sorted(set(requires)):
-                keypath = item.split(',')
-                if not self.__project.valid(*keypath):
-                    self.__logger.error(f'Cannot resolve required keypath [{",".join(keypath)}] '
-                                        f'for {step}/{index}.')
-                    error = True
-                    continue
+            node = self.__tasks[(step, index)]
 
-                param = self.__project.get(*keypath, field=None)
-                check_step, check_index = step, index
-                if param.get(field='pernode').is_never():
-                    check_step, check_index = None, None
-
-                if not param.has_value(step=check_step, index=check_index):
-                    self.__logger.error('No value set for required keypath '
-                                        f'[{",".join(keypath)}] for {step}/{index}.')
-                    error = True
-                    continue
-
-                paramtype = param.get(field='type')
-                if check_file_access and (('file' in paramtype) or ('dir' in paramtype)):
-                    abspath = self.__project.find_files(*keypath,
-                                                        missing_ok=True,
-                                                        step=check_step, index=check_index)
-
-                    unresolved_paths = param.get(step=check_step, index=check_index)
-                    if not isinstance(abspath, list):
-                        abspath = [abspath]
-                        unresolved_paths = [unresolved_paths]
-
-                    for path, setpath in zip(abspath, unresolved_paths):
-                        if path is None:
-                            self.__logger.error(f'Cannot resolve path {setpath} in '
-                                                f'required file keypath [{",".join(keypath)}] '
-                                                f'for {step}/{index}.')
-                            error = True
+            error |= not node.check_required_values()
+            error |= not node.check_required_paths()
 
         return not error
 
@@ -372,6 +368,8 @@ class Scheduler:
         nodes = self.__flow_runtime.get_nodes()
         error = False
 
+        manifest_name = os.path.basename(self.manifest)
+
         for (step, index) in nodes:
             # Get files we receive from input nodes.
             in_nodes = self.__flow_runtime.get_node_inputs(step, index, record=self.__record)
@@ -396,21 +394,17 @@ class Scheduler:
                         inputs = []
                         continue
 
-                    design = self.__project.get("option", 'design')
-                    manifest = f'{design}.pkg.json'
-                    inputs = [inp for inp in os.listdir(in_step_out_dir) if inp != manifest]
+                    inputs = [inp for inp in os.listdir(in_step_out_dir) if inp != manifest_name]
                 else:
                     in_tool = self.__flow.get(in_step, in_index, "tool")
                     in_task = self.__flow.get(in_step, in_index, "task")
                     in_task_class = self.__project.get("tool", in_tool, "task", in_task,
                                                        field="schema")
 
-                    with in_task_class.runtime(SchedulerNode(self.__project,
-                                                             in_step, in_index)) as task:
+                    with in_task_class.runtime(self.__tasks[(in_step, in_index)]) as task:
                         inputs = task.get_output_files()
 
-                with task_class.runtime(SchedulerNode(self.__project,
-                                                      step, index)) as task:
+                with task_class.runtime(self.__tasks[(step, index)]) as task:
                     for inp in inputs:
                         node_inp = task.compute_input_file_node_name(inp, in_step, in_index)
                         if node_inp in requirements:
@@ -446,7 +440,7 @@ class Scheduler:
 
         self.__record.set('status', NodeStatus.PENDING, step=step, index=index)
         for next_step, next_index in self.__flow_runtime.get_nodes_starting_at(step, index):
-            if self.__record.get('status', step=next_step, index=next_index) == NodeStatus.SKIPPED:
+            if (next_step, next_index) in self.__skippedtasks:
                 continue
 
             # Mark following steps as pending
@@ -466,18 +460,6 @@ class Scheduler:
         copy_from_nodes = set(self.__flow_load_runtime.get_nodes()).difference(
             self.__flow_runtime.get_entry_nodes())
         for step, index in self.__flow.get_nodes():
-            node_cls = SchedulerNode
-
-            node_scheduler = self.__project.get('option', 'scheduler', 'name',
-                                                step=step, index=index)
-            if node_scheduler == 'slurm':
-                node_cls = SlurmSchedulerNode
-            elif node_scheduler == 'docker':
-                node_cls = DockerSchedulerNode
-            self.__tasks[(step, index)] = node_cls(self.__project, step, index)
-            if self.__flow.get(step, index, "tool") == "builtin":
-                self.__tasks[(step, index)].set_builtin()
-
             if self.__org_job_name and (step, index) in copy_from_nodes:
                 self.__tasks[(step, index)].copy_from(self.__org_job_name)
 
@@ -581,33 +563,28 @@ class Scheduler:
             if NodeStatus.is_waiting(self.__record.get('status', step=step, index=index)):
                 with self.__tasks[(step, index)].runtime():
                     self.__tasks[(step, index)].clean_directory()
+                    parent_dir = os.path.dirname(self.__tasks[(step, index)].workdir)
+                    if os.path.exists(parent_dir) and len(os.listdir(parent_dir)) == 0:
+                        # Step directory is empty so safe to remove
+                        os.rmdir(parent_dir)
 
-    def configure_nodes(self) -> None:
-        """
-        Prepare and configure all flow nodes before execution, including loading prior run state,
-        running per-node setup, and marking nodes that require rerun.
+    def __configure_collect_previous_information(self) -> Dict[Tuple[str, str], "Project"]:
+        """Collects information from previous runs for nodes that won't be re-executed.
 
-        This method:
-        - Loads available node manifests from previous jobs and uses them to populate setup data
-          where appropriate.
-        - Runs each node's setup routine to initialize tools and runtime state.
-        - For nodes whose parameters or inputs have changed, marks them and all downstream nodes
-          as pending so they will be re-executed.
-        - Replays preserved journaled results for nodes that remain valid to reuse previous outputs.
-        - On a SchedulerFlowReset, forces a full build-directory recheck and marks every node
-          as pending.
-        - Persists the resulting manifest for the current job before returning.
+        This method identifies nodes that are marked for loading (not cleaning) and
+        are not part of the current 'from' execution path. For each of these
+        nodes, it attempts to load its manifest from a previous run.
+
+        Returns:
+            Dict[Tuple[str, str], "Project"]: A dictionary mapping (step, index)
+                tuples to their corresponding loaded Project objects from
+                previous runs.
         """
         from siliconcompiler import Project
+        self.__print_status("Start - collect")
 
-        from_nodes = []
         extra_setup_nodes = {}
-
-        journal = Journal.access(self.__project)
-        journal.start()
-
-        self.__print_status("Start")
-
+        from_nodes = []
         if self.__project.get('option', 'clean'):
             if self.__project.get("option", "from"):
                 from_nodes = self.__flow_runtime.get_entry_nodes()
@@ -626,25 +603,49 @@ class Scheduler:
                 # Node will be run so no need to load
                 continue
 
-            manifest = os.path.join(workdir(self.__project, step=step, index=index),
-                                    'outputs',
-                                    f'{self.__name}.pkg.json')
+            manifest = self.__tasks[(step, index)].get_manifest()
             if os.path.exists(manifest):
                 # ensure we setup these nodes again
                 try:
                     extra_setup_nodes[(step, index)] = Project.from_manifest(filepath=manifest)
-                except Exception:
+                except Exception as e:
+                    self.__logger.debug(f"Reading {manifest} caused: {e}")
                     pass
 
+        self.__print_status("End - collect")
+
+        return extra_setup_nodes
+
+    def __configure_run_setup(self, extra_setup_nodes: Dict[Tuple[str, str], "Project"]) -> None:
+        """Runs the setup() method for all flow nodes and forwards previous status.
+
+        This method iterates through all nodes in execution order and calls
+        their respective `setup()` methods.
+
+        It also uses the `extra_setup_nodes` to:
+        1. Prune nodes from `extra_setup_nodes` if their `setup()` method
+           returns False (indicating the node is no longer valid).
+        2. Forward the 'status' from a valid, previously-run node (found in
+           `extra_setup_nodes`) into the current job's records.
+
+        Args:
+            extra_setup_nodes (Dict[Tuple[str, str], "Project"]): A dictionary
+                of loaded Project objects from previous runs. This dictionary
+                may be modified in-place (nodes may be removed).
+        """
+        self.__print_status("Start - setup")
         # Setup tools for all nodes to run
         for layer_nodes in self.__flow.get_execution_order():
             for step, index in layer_nodes:
                 with self.__tasks[(step, index)].runtime():
                     node_kept = self.__tasks[(step, index)].setup()
+                if not node_kept:
+                    self.__skippedtasks.add((step, index))
                 if not node_kept and (step, index) in extra_setup_nodes:
                     # remove from previous node data
                     del extra_setup_nodes[(step, index)]
 
+                # Copy in old status information, this will be overwritten if needed
                 if (step, index) in extra_setup_nodes:
                     schema = extra_setup_nodes[(step, index)]
                     node_status = None
@@ -655,28 +656,155 @@ class Scheduler:
                     if node_status:
                         # Forward old status
                         self.__record.set('status', node_status, step=step, index=index)
+        self.__print_status("End - setup")
+
+    @staticmethod
+    def _configure_run_required(task: SchedulerNode) \
+            -> Optional[Union[SchedulerFlowReset, SchedulerNodeReset]]:
+        """
+        Helper method to run requires_run() with threads.
+        """
+        with task.runtime():
+            try:
+                task.requires_run()
+            except (SchedulerFlowReset, SchedulerNodeReset) as e:
+                return e
+        return None
+
+    def __configure_check_run_required(self) -> List[Tuple[str, str]]:
+        """Checks which nodes require a re-run and which can be replayed.
+
+        This method iterates through all nodes that are currently marked as
+        'SUCCESS' (typically from a previous run). It calls `requires_run()`
+        on each to determine if inputs, parameters, or other dependencies
+        have changed.
+
+        - If `requires_run()` is True, the node is marked as 'pending' (and
+          will be re-executed).
+        - If `requires_run()` is False, the node is added to the 'replay' list,
+          indicating its previous results can be reused.
+
+        Returns:
+            List[Tuple[str, str]]: A list of (step, index) tuples for nodes
+                that do *not* require a re-run and whose results can be
+                replayed from the journal.
+        """
+        self.__print_status("Start - check")
+
+        replay: List[Tuple[str, str]] = []
 
-        self.__print_status("After setup")
+        nodes: List[Tuple[str, str]] = []
+
+        def filter_nodes(nodes: List[Tuple[str, str]]) -> None:
+            for step, index in tuple(nodes):
+                # Only look at successful nodes
+                if self.__record.get("status", step=step, index=index) != NodeStatus.SUCCESS:
+                    nodes.remove((step, index))
+
+        def create_node_group(nodes: List[Tuple[str, str]], size: int) -> List[Tuple[str, str]]:
+            group = []
+            for _ in range(size):
+                if nodes:
+                    group.append(nodes.pop(0))
+            return group
+
+        # Collect initial list of nodes to process
+        for layer_nodes in self.__flow.get_execution_order():
+            nodes.extend(layer_nodes)
+
+        # Determine pool size
+        cores = utils.get_cores()
+        pool_size = self.project.option.scheduler.get_maxthreads() or cores
+        pool_size = max(1, min(cores, pool_size))
+
+        # Limit based on number of nodes if less than number of cores
+        filter_nodes(nodes)
+        if not nodes:
+            # No nodes left so just return
+            return []
+
+        pool_size = min(pool_size, len(nodes))
+
+        self.__logger.debug(f"Check pool size: {pool_size}")
+
+        # Call this in case this was invoked without __main__
+        multiprocessing.freeze_support()
+
+        with multiprocessing.get_context("spawn").Pool(pool_size) as pool:
+            while True:
+                # Filter nodes
+                filter_nodes(nodes)
+
+                # Generate a group of nodes to run
+                group = create_node_group(nodes, pool_size)
+                self.__logger.debug(f"Group to check: {group}")
+                if not group:
+                    # Group is empty
+                    break
+
+                tasks = [self.__tasks[(step, index)] for step, index in group]
+                # Suppress excess info messages during checks
+                cur_level = self.project.logger.level
+                self.project.logger.setLevel(logging.WARNING)
+                try:
+                    runcheck = pool.map(Scheduler._configure_run_required, tasks)
+                finally:
+                    self.project.logger.setLevel(cur_level)
+
+                for node, runrequired in zip(group, runcheck):
+                    if self.__record.get("status", step=node[0], index=node[1]) != \
+                            NodeStatus.SUCCESS:
+                        continue
+
+                    self.__logger.debug(f"  Result: {node} -> {runrequired}")
+
+                    if runrequired is not None:
+                        runrequired.log(self.__logger)
+
+                        if isinstance(runrequired, SchedulerFlowReset):
+                            raise runrequired from None
+
+                        # This node must be run
+                        self.__mark_pending(*node)
+                    else:
+                        # import old information
+                        replay.append(node)
+
+        self.__print_status("End - check")
+
+        return replay
+
+    def configure_nodes(self) -> None:
+        """
+        Prepare and configure all flow nodes before execution, including loading prior run state,
+        running per-node setup, and marking nodes that require rerun.
+
+        This method:
+        - Loads available node manifests from previous jobs and uses them to populate setup data
+          where appropriate.
+        - Runs each node's setup routine to initialize tools and runtime state.
+        - For nodes whose parameters or inputs have changed, marks them and all downstream nodes
+          as pending so they will be re-executed.
+        - Replays preserved journaled results for nodes that remain valid to reuse previous outputs.
+        - On a SchedulerFlowReset, forces a full build-directory recheck and marks every node
+          as pending.
+        - Persists the resulting manifest for the current job before returning.
+        """
+        journal = Journal.access(self.__project)
+        journal.start()
+
+        extra_setup_nodes = self.__configure_collect_previous_information()
+
+        self.__configure_run_setup(extra_setup_nodes)
 
         # Check for modified information
         try:
-            replay = []
-            for layer_nodes in self.__flow.get_execution_order():
-                for step, index in layer_nodes:
-                    # Only look at successful nodes
-                    if self.__record.get("status", step=step, index=index) != NodeStatus.SUCCESS:
-                        continue
+            replay = self.__configure_check_run_required()
 
-                    with self.__tasks[(step, index)].runtime():
-                        if self.__tasks[(step, index)].requires_run():
-                            # This node must be run
-                            self.__mark_pending(step, index)
-                        elif (step, index) in extra_setup_nodes:
-                            # import old information
-                            replay.append((step, index))
             # Replay previous information
             for step, index in replay:
-                Journal.access(extra_setup_nodes[(step, index)]).replay(self.__project)
+                if (step, index) in extra_setup_nodes:
+                    Journal.access(extra_setup_nodes[(step, index)]).replay(self.__project)
         except SchedulerFlowReset:
             # Mark all nodes as pending
             self.__clean_build_dir_full(recheck=True)
@@ -684,7 +812,7 @@ class Scheduler:
             for step, index in self.__flow.get_nodes():
                 self.__mark_pending(step, index)
 
-        self.__print_status("After requires run")
+        self.__print_status("Before ensure")
 
         # Ensure all nodes are marked as pending if needed
         for layer_nodes in self.__flow_runtime.get_execution_order():
@@ -693,11 +821,12 @@ class Scheduler:
                 if NodeStatus.is_waiting(status) or NodeStatus.is_error(status):
                     self.__mark_pending(step, index)
 
-        self.__print_status("After ensure")
+        self.__print_status("FINAL")
+
+        # Write configured manifest
+        os.makedirs(os.path.dirname(self.manifest), exist_ok=True)
+        self.__project.write_manifest(self.manifest)
 
-        os.makedirs(jobdir(self.__project), exist_ok=True)
-        self.__project.write_manifest(os.path.join(jobdir(self.__project),
-                                                   f"{self.__name}.pkg.json"))
         journal.stop()
 
     def __check_display(self) -> None:
@@ -744,6 +873,8 @@ class Scheduler:
                 if m:
                     jobid = max(jobid, int(m.group(1)))
             self.__project.set('option', 'jobname', f'{stem}{jobid + 1}')
+            for task in self.__tasks.values():
+                task._update_job()
             return True
         return False
 
@@ -788,7 +919,7 @@ class Scheduler:
                     if self.__project.option.scheduler.get_name(step=step, index=index) is not None:
                         continue
 
-                    node = SchedulerNode(self.__project, step, index)
+                    node = self.__tasks[(step, index)]
                     with node.runtime():
                         try:
                             exe = node.get_exe_path()
@@ -816,3 +947,35 @@ class Scheduler:
                 os.chdir(cwd)
 
         return not error
+
+    def __check_collect_files(self) -> bool:
+        """
+        Iterates through all tasks in the scheduler, and checks if the there
+        are files or directories that need to be collected
+
+        Returns:
+            bool: True if there is something to be collected, False otherwise.
+        """
+        do_collect = False
+        for task in self.__tasks.values():
+            if task.mark_copy():
+                do_collect = True
+
+        return do_collect
+
+    def __init_schedulers(self) -> None:
+        """
+        Collect and invoke unique initialization callbacks from all task schedulers.
+
+        This method gathers init functions from all SchedulerNode instances, deduplicates them
+        (since multiple tasks may share the same scheduler class), and invokes each once to
+        perform early validation (e.g., checking Docker/Slurm availability).
+        """
+        self.__logger.debug("Collecting unique scheduler initialization callbacks")
+        init_funcs = set()
+        for step, index in self.__flow_runtime.get_nodes():
+            init_funcs.add(self.__tasks[(step, index)].init)
+
+        for init in sorted(init_funcs, key=lambda func: func.__qualname__):
+            self.__logger.debug(f"Initializing scheduler: {init.__qualname__}")
+            init(self.__project)