siliconcompiler 0.35.3__py3-none-any.whl → 0.35.4__py3-none-any.whl

siliconcompiler/scheduler/scheduler.py

@@ -12,7 +12,7 @@ import os.path
 
 from datetime import datetime
 
-from typing import Union, Dict, Optional, Tuple, List, TYPE_CHECKING
+from typing import Union, Dict, Optional, Tuple, List, Set, TYPE_CHECKING
 
 from siliconcompiler import NodeStatus
 from siliconcompiler.schema import Journal
@@ -29,6 +29,7 @@ from siliconcompiler.utils.logging import SCLoggerFormatter
 from siliconcompiler.utils.multiprocessing import MPManager
 from siliconcompiler.scheduler import send_messages, SCRuntimeError
 from siliconcompiler.utils.paths import collectiondir, jobdir, workdir
+from siliconcompiler.utils.curation import collect
 
 if TYPE_CHECKING:
     from siliconcompiler.project import Project
@@ -101,12 +102,27 @@ class Scheduler:
         self.__metrics: "MetricSchema" = self.__project.get("metric", field="schema")
 
         self.__tasks: Dict[Tuple[str, str], SchedulerNode] = {}
+        self.__skippedtasks: Set[Tuple[str, str]] = set()
 
         # Create dummy handler
         self.__joblog_handler = logging.NullHandler()
         self.__org_job_name = self.__project.get("option", "jobname")
         self.__logfile = None
 
+        # Create tasks
+        for step, index in self.__flow.get_nodes():
+            node_cls = SchedulerNode
+
+            node_scheduler = self.__project.get('option', 'scheduler', 'name',
+                                                step=step, index=index)
+            if node_scheduler == 'slurm':
+                node_cls = SlurmSchedulerNode
+            elif node_scheduler == 'docker':
+                node_cls = DockerSchedulerNode
+            self.__tasks[(step, index)] = node_cls(self.__project, step, index)
+            if self.__flow.get(step, index, "tool") == "builtin":
+                self.__tasks[(step, index)].set_builtin()
+
     @property
     def manifest(self) -> str:
         """
@@ -270,6 +286,9 @@ class Scheduler:
             if not self.check_manifest():
                 raise SCRuntimeError("check_manifest() failed")
 
+            # Initialize schedulers
+            self.__init_schedulers()
+
             self.__run_setup()
             self.configure_nodes()
 
@@ -288,15 +307,23 @@ class Scheduler:
             if not self.__check_flowgraph_io():
                 raise SCRuntimeError("Flowgraph file IO constrains errors")
 
-            self.run_core()
+            # Collect files for remote runs
+            if self.__check_collect_files():
+                collect(self.project)
+
+            try:
+                self.run_core()
+            except SCRuntimeError as e:
+                raise e
 
-            # Store run in history
-            self.__project._record_history()
+            finally:
+                # Store run in history
+                self.__project._record_history()
 
-            # Record final manifest
-            self.__project.write_manifest(self.manifest)
+                # Record final manifest
+                self.__project.write_manifest(self.manifest)
 
-            send_messages.send(self.__project, 'summary', None, None)
+                send_messages.send(self.__project, 'summary', None, None)
         finally:
             if self.__joblog_handler is not None:
                 self.__logger.removeHandler(self.__joblog_handler)
@@ -319,50 +346,10 @@ class Scheduler:
         error = False
 
         for (step, index) in self.__flow_runtime.get_nodes():
-            scheduler = self.__project.option.scheduler.get_name(step=step, index=index)
-            check_file_access = not self.__project.option.get_remote() and scheduler is None
-
-            node = SchedulerNode(self.__project, step, index)
-            requires = []
-            with node.runtime():
-                requires = node.task.get('require')
-
-            for item in sorted(set(requires)):
-                keypath = item.split(',')
-                if not self.__project.valid(*keypath):
-                    self.__logger.error(f'Cannot resolve required keypath [{",".join(keypath)}] '
-                                        f'for {step}/{index}.')
-                    error = True
-                    continue
-
-                param = self.__project.get(*keypath, field=None)
-                check_step, check_index = step, index
-                if param.get(field='pernode').is_never():
-                    check_step, check_index = None, None
-
-                if not param.has_value(step=check_step, index=check_index):
-                    self.__logger.error('No value set for required keypath '
-                                        f'[{",".join(keypath)}] for {step}/{index}.')
-                    error = True
-                    continue
+            node = self.__tasks[(step, index)]
 
-                paramtype = param.get(field='type')
-                if check_file_access and (('file' in paramtype) or ('dir' in paramtype)):
-                    abspath = self.__project.find_files(*keypath,
-                                                        missing_ok=True,
-                                                        step=check_step, index=check_index)
-
-                    unresolved_paths = param.get(step=check_step, index=check_index)
-                    if not isinstance(abspath, list):
-                        abspath = [abspath]
-                        unresolved_paths = [unresolved_paths]
-
-                    for path, setpath in zip(abspath, unresolved_paths):
-                        if path is None:
-                            self.__logger.error(f'Cannot resolve path {setpath} in '
-                                                f'required file keypath [{",".join(keypath)}] '
-                                                f'for {step}/{index}.')
-                            error = True
+            error |= not node.check_required_values()
+            error |= not node.check_required_paths()
 
         return not error
 
@@ -414,12 +401,10 @@ class Scheduler:
                     in_task_class = self.__project.get("tool", in_tool, "task", in_task,
                                                        field="schema")
 
-                    with in_task_class.runtime(SchedulerNode(self.__project,
-                                                             in_step, in_index)) as task:
+                    with in_task_class.runtime(self.__tasks[(in_step, in_index)]) as task:
                         inputs = task.get_output_files()
 
-                with task_class.runtime(SchedulerNode(self.__project,
-                                                      step, index)) as task:
+                with task_class.runtime(self.__tasks[(step, index)]) as task:
                     for inp in inputs:
                         node_inp = task.compute_input_file_node_name(inp, in_step, in_index)
                         if node_inp in requirements:
@@ -455,7 +440,7 @@ class Scheduler:
 
         self.__record.set('status', NodeStatus.PENDING, step=step, index=index)
         for next_step, next_index in self.__flow_runtime.get_nodes_starting_at(step, index):
-            if self.__record.get('status', step=next_step, index=next_index) == NodeStatus.SKIPPED:
+            if (next_step, next_index) in self.__skippedtasks:
                 continue
 
             # Mark following steps as pending
@@ -475,18 +460,6 @@ class Scheduler:
         copy_from_nodes = set(self.__flow_load_runtime.get_nodes()).difference(
             self.__flow_runtime.get_entry_nodes())
         for step, index in self.__flow.get_nodes():
-            node_cls = SchedulerNode
-
-            node_scheduler = self.__project.get('option', 'scheduler', 'name',
-                                                step=step, index=index)
-            if node_scheduler == 'slurm':
-                node_cls = SlurmSchedulerNode
-            elif node_scheduler == 'docker':
-                node_cls = DockerSchedulerNode
-            self.__tasks[(step, index)] = node_cls(self.__project, step, index)
-            if self.__flow.get(step, index, "tool") == "builtin":
-                self.__tasks[(step, index)].set_builtin()
-
             if self.__org_job_name and (step, index) in copy_from_nodes:
                 self.__tasks[(step, index)].copy_from(self.__org_job_name)
 
@@ -590,6 +563,10 @@ class Scheduler:
             if NodeStatus.is_waiting(self.__record.get('status', step=step, index=index)):
                 with self.__tasks[(step, index)].runtime():
                     self.__tasks[(step, index)].clean_directory()
+                    parent_dir = os.path.dirname(self.__tasks[(step, index)].workdir)
+                    if os.path.exists(parent_dir) and len(os.listdir(parent_dir)) == 0:
+                        # Step directory is empty so safe to remove
+                        os.rmdir(parent_dir)
 
     def __configure_collect_previous_information(self) -> Dict[Tuple[str, str], "Project"]:
         """Collects information from previous runs for nodes that won't be re-executed.
@@ -662,10 +639,13 @@ class Scheduler:
             for step, index in layer_nodes:
                 with self.__tasks[(step, index)].runtime():
                     node_kept = self.__tasks[(step, index)].setup()
+                if not node_kept:
+                    self.__skippedtasks.add((step, index))
                 if not node_kept and (step, index) in extra_setup_nodes:
                     # remove from previous node data
                     del extra_setup_nodes[(step, index)]
 
+                # Copy in old status information, this will be overwritten if needed
                 if (step, index) in extra_setup_nodes:
                     schema = extra_setup_nodes[(step, index)]
                     node_status = None
@@ -893,6 +873,8 @@ class Scheduler:
                 if m:
                     jobid = max(jobid, int(m.group(1)))
             self.__project.set('option', 'jobname', f'{stem}{jobid + 1}')
+            for task in self.__tasks.values():
+                task._update_job()
             return True
         return False
 
@@ -937,7 +919,7 @@ class Scheduler:
                     if self.__project.option.scheduler.get_name(step=step, index=index) is not None:
                         continue
 
-                    node = SchedulerNode(self.__project, step, index)
+                    node = self.__tasks[(step, index)]
                     with node.runtime():
                         try:
                             exe = node.get_exe_path()
@@ -965,3 +947,35 @@ class Scheduler:
                 os.chdir(cwd)
 
         return not error
+
+    def __check_collect_files(self) -> bool:
+        """
+        Iterates through all tasks in the scheduler, and checks if the there
+        are files or directories that need to be collected
+
+        Returns:
+            bool: True if there is something to be collected, False otherwise.
+        """
+        do_collect = False
+        for task in self.__tasks.values():
+            if task.mark_copy():
+                do_collect = True
+
+        return do_collect
+
+    def __init_schedulers(self) -> None:
+        """
+        Collect and invoke unique initialization callbacks from all task schedulers.
+
+        This method gathers init functions from all SchedulerNode instances, deduplicates them
+        (since multiple tasks may share the same scheduler class), and invokes each once to
+        perform early validation (e.g., checking Docker/Slurm availability).
+        """
+        self.__logger.debug("Collecting unique scheduler initialization callbacks")
+        init_funcs = set()
+        for step, index in self.__flow_runtime.get_nodes():
+            init_funcs.add(self.__tasks[(step, index)].init)
+
+        for init in sorted(init_funcs, key=lambda func: func.__qualname__):
+            self.__logger.debug(f"Initializing scheduler: {init.__qualname__}")
+            init(self.__project)

siliconcompiler/scheduler/schedulernode.py

@@ -9,7 +9,7 @@ import time
 
 import os.path
 
-from logging.handlers import QueueHandler
+from siliconcompiler.utils.multiprocessing import MPQueueHandler as QueueHandler
 
 from typing import List, Optional, Set, Tuple, TYPE_CHECKING
 
@@ -19,7 +19,7 @@ from siliconcompiler.utils.logging import get_console_formatter, SCInRunLoggerFo
 
 from siliconcompiler.package import Resolver
 from siliconcompiler.schema_support.record import RecordTime, RecordTool
-from siliconcompiler.schema import Journal
+from siliconcompiler.schema import Journal, Parameter
 from siliconcompiler.scheduler import send_messages
 from siliconcompiler.utils.paths import workdir, jobdir, collectiondir, cwdir
 
@@ -67,9 +67,10 @@ class SchedulerNode:
     This class encapsulates the state and logic required to run a specific
     step and index, including setting up directories, handling file I/O,
     executing the associated tool, and recording results.
-
     """
 
+    __MAX_LOG_PRINT = 100  # Maximum number of warnings/error to print to log
+
     def __init__(self, project: "Project", step: str, index: str, replay: bool = False):
         """
         Initializes a SchedulerNode.
@@ -100,7 +101,6 @@ class SchedulerNode:
             self.__project.get("option", "fileset")[0],
             "topmodule")
 
-        self.__job: str = self.__project.get('option', 'jobname')
         self.__record_user_info: bool = self.__project.get(
             "option", "track", step=self.__step, index=self.__index)
         self.__pipe = None
@@ -116,24 +116,12 @@ class SchedulerNode:
         self.__enforce_inputfiles = True
         self.__enforce_outputfiles = True
 
+        self._update_job()
+
         flow: str = self.__project.get('option', 'flow')
         self.__is_entry_node: bool = (self.__step, self.__index) in \
             self.__project.get("flowgraph", flow, field="schema").get_entry_nodes()
 
-        self.__cwd = cwdir(self.__project)
-        self.__jobworkdir = jobdir(self.__project)
-        self.__workdir = workdir(self.__project, step=self.__step, index=self.__index)
-        self.__manifests = {
-            "input": os.path.join(self.__workdir, "inputs", f"{self.__name}.pkg.json"),
-            "output": os.path.join(self.__workdir, "outputs", f"{self.__name}.pkg.json")
-        }
-        self.__logs = {
-            "sc": os.path.join(self.__workdir, f"sc_{self.__step}_{self.__index}.log"),
-            "exe": os.path.join(self.__workdir, f"{self.__step}.log")
-        }
-        self.__replay_script = os.path.join(self.__workdir, "replay.sh")
-        self.__collection_path = collectiondir(self.__project)
-
         self.set_queue(None, None)
         self.__setup_schema_access()
 
@@ -258,6 +246,22 @@ class SchedulerNode:
         """Task: The task object associated with this node."""
         return self.__task
 
+    def _update_job(self):
+        self.__job: str = self.__project.get('option', 'jobname')
+        self.__cwd = cwdir(self.__project)
+        self.__jobworkdir = jobdir(self.__project)
+        self.__workdir = workdir(self.__project, step=self.__step, index=self.__index)
+        self.__manifests = {
+            "input": os.path.join(self.__workdir, "inputs", f"{self.__name}.pkg.json"),
+            "output": os.path.join(self.__workdir, "outputs", f"{self.__name}.pkg.json")
+        }
+        self.__logs = {
+            "sc": os.path.join(self.__workdir, f"sc_{self.__step}_{self.__index}.log"),
+            "exe": os.path.join(self.__workdir, f"{self.__step}.log")
+        }
+        self.__replay_script = os.path.join(self.__workdir, "replay.sh")
+        self.__collection_path = collectiondir(self.__project)
+
     def get_manifest(self, input: bool = False) -> str:
         """
         Gets the path to the input or output manifest file for this node.
@@ -1156,11 +1160,13 @@ class SchedulerNode:
         if 'errors' in checks:
             ordered_suffixes.append('errors')
 
+        print_paths = {}
         # Looping through patterns for each line
         with sc_open(self.__logs["exe"]) as f:
             line_count = sum(1 for _ in f)
             right_align = len(str(line_count))
             for suffix in ordered_suffixes:
+                print_paths[suffix] = False
                 # Start at the beginning of file again
                 f.seek(0)
                 for num, line in enumerate(f, start=1):
@@ -1169,7 +1175,7 @@ class SchedulerNode:
                         if string is None:
                             break
                         else:
-                            string = utils.grep(self.__project, item, string)
+                            string = utils.grep(self.__project.logger, item, string)
                     if string is not None:
                         matches[suffix] += 1
                         # always print to file
@@ -1177,11 +1183,21 @@ class SchedulerNode:
                         print(line_with_num, file=checks[suffix]['report'])
                         # selectively print to display
                         if checks[suffix]["display"]:
-                            checks[suffix]["display"](suffix, line_with_num)
+                            if matches[suffix] <= SchedulerNode.__MAX_LOG_PRINT:
+                                checks[suffix]["display"](suffix, line_with_num)
+                            else:
+                                if not print_paths[suffix]:
+                                    checks[suffix]["display"](suffix, "print limit reached")
+                                print_paths[suffix] = True
 
         for check in checks.values():
             check['report'].close()
 
+        for suffix in ordered_suffixes:
+            if print_paths[suffix]:
+                self.logger.info(f"All {suffix} can be viewed at: "
+                                 f"{os.path.abspath(f'{self.__step}.{suffix}')}")
+
         for metric in ("errors", "warnings"):
             if metric in matches:
                 value = self.__metrics.get(metric, step=self.__step, index=self.__index)
@@ -1370,3 +1386,100 @@ class SchedulerNode:
             for logfile in self.__logs.values():
                 if os.path.isfile(logfile):
                     tar.add(logfile, arcname=arcname(logfile))
+
+    def get_required_keys(self) -> Set[Tuple[str, ...]]:
+        """
+        This function walks through the 'require' keys and returns the
+        keys.
+        """
+        path_keys = set()
+        with self.runtime():
+            task = self.task
+            for key in task.get('require'):
+                path_keys.add(tuple(key.split(",")))
+            if task.has_prescript():
+                path_keys.add((*task._keypath, "prescript"))
+            if task.has_postscript():
+                path_keys.add((*task._keypath, "postscript"))
+            if task.get("refdir"):
+                path_keys.add((*task._keypath, "refdir"))
+            if task.get("script"):
+                path_keys.add((*task._keypath, "script"))
+            if task.get("exe"):
+                path_keys.add((*task._keypath, "exe"))
+
+        return path_keys
+
+    def get_required_path_keys(self) -> Set[Tuple[str, ...]]:
+        """
+        This function walks through the 'require' keys and returns the
+        keys that are of type path (file/dir).
+        """
+        path_keys = set()
+        for key in self.get_required_keys():
+            try:
+                param_type: str = self.__project.get(*key, field="type")
+                if "file" in param_type or "dir" in param_type:
+                    path_keys.add(key)
+            except KeyError:
+                # Key does not exist
+                pass
+
+        return path_keys
+
+    def mark_copy(self) -> bool:
+        """Marks files from the 'require' path keys for copying."""
+        return False
+
+    def check_required_values(self) -> bool:
+        requires = self.get_required_keys()
+
+        error = False
+        for key in sorted(requires):
+            if not self.__project.valid(*key):
+                self.logger.error(f'Cannot resolve required keypath [{",".join(key)}] '
+                                  f'for {self.step}/{self.index}.')
+                error = True
+                continue
+
+            param: Parameter = self.__project.get(*key, field=None)
+            check_step, check_index = self.step, self.index
+            if param.get(field='pernode').is_never():
+                check_step, check_index = None, None
+
+            if not param.has_value(step=check_step, index=check_index):
+                self.logger.error('No value set for required keypath '
+                                  f'[{",".join(key)}] for {self.step}/{self.index}.')
+                error = True
+                continue
+        return not error
+
+    def check_required_paths(self) -> bool:
+        if self.__project.option.get_remote():
+            return True
+
+        requires = self.get_required_path_keys()
+
+        error = False
+        for key in sorted(requires):
+            param: Parameter = self.__project.get(*key, field=None)
+            check_step, check_index = self.step, self.index
+            if param.get(field='pernode').is_never():
+                check_step, check_index = None, None
+
+            abspath = self.__project.find_files(*key,
+                                                missing_ok=True,
+                                                step=check_step, index=check_index)
+
+            unresolved_paths = param.get(step=check_step, index=check_index)
+            if not isinstance(abspath, list):
+                abspath = [abspath]
+                unresolved_paths = [unresolved_paths]
+
+            for path, setpath in zip(abspath, unresolved_paths):
+                if path is None:
+                    self.logger.error(f'Cannot resolve path {setpath} in '
+                                      f'required file keypath [{",".join(key)}] '
+                                      f'for {self.step}/{self.index}.')
+                    error = True
+        return not error