siliconcompiler 0.35.2__py3-none-any.whl → 0.35.4__py3-none-any.whl

siliconcompiler/scheduler/schedulernode.py

@@ -9,7 +9,7 @@ import time
 
 import os.path
 
-from logging.handlers import QueueHandler
+from siliconcompiler.utils.multiprocessing import MPQueueHandler as QueueHandler
 
 from typing import List, Optional, Set, Tuple, TYPE_CHECKING
 
@@ -19,7 +19,7 @@ from siliconcompiler.utils.logging import get_console_formatter, SCInRunLoggerFo
 
 from siliconcompiler.package import Resolver
 from siliconcompiler.schema_support.record import RecordTime, RecordTool
-from siliconcompiler.schema import Journal
+from siliconcompiler.schema import Journal, Parameter
 from siliconcompiler.scheduler import send_messages
 from siliconcompiler.utils.paths import workdir, jobdir, collectiondir, cwdir
 
@@ -30,10 +30,36 @@ if TYPE_CHECKING:
     from siliconcompiler.schema_support.metric import MetricSchema
 
 
-class SchedulerFlowReset(Exception):
+class _SchedulerReset(Exception):
+    def __init__(self, msg: str, *args: object) -> None:
+        super().__init__(msg, *args)
+        self.__msg = msg
+
+    @property
+    def msg(self) -> str:
+        return self.__msg
+
+    def log(self, logger: logging.Logger) -> None:
+        logger.debug(self.msg)
+
+
+class SchedulerFlowReset(_SchedulerReset):
     pass
 
 
+class SchedulerNodeReset(_SchedulerReset):
+    def log(self, logger: logging.Logger) -> None:
+        logger.warning(self.msg)
+
+
+class SchedulerNodeResetSilent(SchedulerNodeReset):
+    def __init__(self, msg: str, *args: object) -> None:
+        super().__init__(msg, *args)
+
+    def log(self, logger: logging.Logger) -> None:
+        _SchedulerReset.log(self, logger)
+
+
 class SchedulerNode:
     """
     A class for managing and executing a single node in the compilation flow graph.
@@ -41,9 +67,10 @@ class SchedulerNode:
     This class encapsulates the state and logic required to run a specific
     step and index, including setting up directories, handling file I/O,
     executing the associated tool, and recording results.
-
     """
 
+    __MAX_LOG_PRINT = 100  # Maximum number of warnings/error to print to log
+
     def __init__(self, project: "Project", step: str, index: str, replay: bool = False):
         """
         Initializes a SchedulerNode.
@@ -74,7 +101,6 @@ class SchedulerNode:
             self.__project.get("option", "fileset")[0],
             "topmodule")
 
-        self.__job: str = self.__project.get('option', 'jobname')
         self.__record_user_info: bool = self.__project.get(
             "option", "track", step=self.__step, index=self.__index)
         self.__pipe = None
@@ -90,24 +116,12 @@ class SchedulerNode:
         self.__enforce_inputfiles = True
         self.__enforce_outputfiles = True
 
+        self._update_job()
+
         flow: str = self.__project.get('option', 'flow')
         self.__is_entry_node: bool = (self.__step, self.__index) in \
             self.__project.get("flowgraph", flow, field="schema").get_entry_nodes()
 
-        self.__cwd = cwdir(self.__project)
-        self.__jobworkdir = jobdir(self.__project)
-        self.__workdir = workdir(self.__project, step=self.__step, index=self.__index)
-        self.__manifests = {
-            "input": os.path.join(self.__workdir, "inputs", f"{self.__name}.pkg.json"),
-            "output": os.path.join(self.__workdir, "outputs", f"{self.__name}.pkg.json")
-        }
-        self.__logs = {
-            "sc": os.path.join(self.__workdir, f"sc_{self.__step}_{self.__index}.log"),
-            "exe": os.path.join(self.__workdir, f"{self.__step}.log")
-        }
-        self.__replay_script = os.path.join(self.__workdir, "replay.sh")
-        self.__collection_path = collectiondir(self.__project)
-
         self.set_queue(None, None)
         self.__setup_schema_access()
 
@@ -120,10 +134,12 @@ class SchedulerNode:
         are directed to the correct task's schema.
         """
         prev_task = self.__task
-        with self.__task.runtime(self) as runtask:
-            self.__task = runtask
-            yield
-        self.__task = prev_task
+        try:
+            with self.__task.runtime(self) as runtask:
+                self.__task = runtask
+                yield
+        finally:
+            self.__task = prev_task
 
     @staticmethod
     def init(project: "Project") -> None:
@@ -230,6 +246,22 @@ class SchedulerNode:
         """Task: The task object associated with this node."""
         return self.__task
 
+    def _update_job(self):
+        self.__job: str = self.__project.get('option', 'jobname')
+        self.__cwd = cwdir(self.__project)
+        self.__jobworkdir = jobdir(self.__project)
+        self.__workdir = workdir(self.__project, step=self.__step, index=self.__index)
+        self.__manifests = {
+            "input": os.path.join(self.__workdir, "inputs", f"{self.__name}.pkg.json"),
+            "output": os.path.join(self.__workdir, "outputs", f"{self.__name}.pkg.json")
+        }
+        self.__logs = {
+            "sc": os.path.join(self.__workdir, f"sc_{self.__step}_{self.__index}.log"),
+            "exe": os.path.join(self.__workdir, f"{self.__step}.log")
+        }
+        self.__replay_script = os.path.join(self.__workdir, "replay.sh")
+        self.__collection_path = collectiondir(self.__project)
+
     def get_manifest(self, input: bool = False) -> str:
         """
         Gets the path to the input or output manifest file for this node.
@@ -382,7 +414,7 @@ class SchedulerNode:
 
             return True
 
-    def check_previous_run_status(self, previous_run: "SchedulerNode") -> bool:
+    def check_previous_run_status(self, previous_run: "SchedulerNode") -> None:
         """
         Determine whether a prior run is compatible and completed successfully for use as
         an incremental build starting point.
@@ -406,25 +438,19 @@ class SchedulerNode:
 
         # Tool name
         if self.__task.tool() != previous_run.__task.tool():
-            self.logger.debug("Tool name changed")
-            return False
+            raise SchedulerNodeResetSilent("Tool name changed")
 
         # Task name
         if self.__task.task() != previous_run.__task.task():
-            self.logger.debug("Task name changed")
-            return False
+            raise SchedulerNodeResetSilent("Task name changed")
 
         previous_status = previous_run.__project.get("record", "status",
                                                      step=self.__step, index=self.__index)
         if not NodeStatus.is_done(previous_status):
-            self.logger.debug("Previous step did not complete")
-            # Not complete
-            return False
+            raise SchedulerNodeResetSilent("Previous step did not complete")
 
         if not NodeStatus.is_success(previous_status):
-            self.logger.debug("Previous step was not successful")
-            # Not a success
-            return False
+            raise SchedulerNodeResetSilent("Previous step was not successful")
 
         # Check input nodes
         log_level = self.logger.level
@@ -433,16 +459,11 @@ class SchedulerNode:
         self.logger.setLevel(log_level)
         if set(previous_run.__project.get("record", "inputnode",
                                           step=self.__step, index=self.__index)) != set(sel_inputs):
-            self.logger.warning(f'inputs to {self.__step}/{self.__index} has been modified from '
-                                'previous run')
-            return False
-
-        # Check that all output files are present?
-
-        return True
+            raise SchedulerNodeReset(f'inputs to {self.__step}/{self.__index} has been '
+                                     'modified from previous run')
 
     def check_values_changed(self, previous_run: "SchedulerNode", keys: Set[Tuple[str, ...]]) \
-            -> bool:
+            -> None:
         """
         Checks if any specified schema parameter values have changed.
 
@@ -453,15 +474,14 @@ class SchedulerNode:
         Returns:
             bool: True if any value has changed, False otherwise.
         """
-        def print_warning(key):
-            self.logger.warning(f'[{",".join(key)}] in {self.__step}/{self.__index} has been '
-                                'modified from previous run')
+        def gen_reset(key):
+            raise SchedulerNodeReset(f'[{",".join(key)}] in {self.__step}/{self.__index} has been '
+                                     'modified from previous run')
 
         for key in sorted(keys):
             if not self.__project.valid(*key) or not previous_run.__project.valid(*key):
                 # Key is missing in either run
-                print_warning(key)
-                return True
+                gen_reset(key)
 
             param = self.__project.get(*key, field=None)
             step, index = self.__step, self.__index
@@ -472,13 +492,10 @@ class SchedulerNode:
             prev_val = previous_run.__project.get(*key, step=step, index=index)
 
             if check_val != prev_val:
-                print_warning(key)
-                return True
-
-        return False
+                gen_reset(key)
 
     def check_files_changed(self, previous_run: "SchedulerNode",
-                            previous_time: float, keys: Set[Tuple[str, ...]]) -> bool:
+                            previous_time: float, keys: Set[Tuple[str, ...]]) -> None:
         """
         Checks if any specified file-based parameters have changed.
 
@@ -494,9 +511,9 @@ class SchedulerNode:
         """
         use_hash = self.__hash and previous_run.__hash
 
-        def print_warning(key, reason):
-            self.logger.warning(f'[{",".join(key)}] ({reason}) in {self.__step}/{self.__index} has '
-                                'been modified from previous run')
+        def gen_warning(key, reason):
+            raise SchedulerNodeReset(f'[{",".join(key)}] ({reason}) in {self.__step}/'
+                                     f'{self.__index} has been modified from previous run')
 
         def get_file_time(path):
             times = [os.path.getmtime(path)]
@@ -521,8 +538,7 @@ class SchedulerNode:
                                                        step=step, index=index)
 
                 if check_hash != prev_hash:
-                    print_warning(key, "file hash")
-                    return True
+                    gen_warning(key, "file hash")
             else:
                 # check package values
                 check_val = self.__project.get(*key, field='dataroot',
@@ -531,8 +547,7 @@ class SchedulerNode:
                                                       step=step, index=index)
 
                 if check_val != prev_val:
-                    print_warning(key, "file dataroot")
-                    return True
+                    gen_warning(key, "file dataroot")
 
                 files = self.__project.find_files(*key, step=step, index=index)
                 if not isinstance(files, (list, set, tuple)):
@@ -540,10 +555,7 @@ class SchedulerNode:
 
                 for check_file in files:
                     if get_file_time(check_file) > previous_time:
-                        print_warning(key, "timestamp")
-                        return True
-
-        return False
+                        gen_warning(key, "timestamp")
 
     def get_check_changed_keys(self) -> Tuple[Set[Tuple[str, ...]], Set[Tuple[str, ...]]]:
         """
@@ -585,7 +597,7 @@ class SchedulerNode:
 
         return value_keys, path_keys
 
-    def requires_run(self) -> bool:
+    def requires_run(self) -> None:
         """
         Determines if the node needs to be re-run.
 
@@ -601,8 +613,7 @@ class SchedulerNode:
 
         if self.__breakpoint:
             # Breakpoint is set to must run
-            self.logger.debug("Breakpoint is set")
-            return True
+            raise SchedulerNodeResetSilent(f"Breakpoint is set on {self.__step}/{self.__index}")
 
         # Load previous manifest
         previous_node = None
@@ -612,33 +623,27 @@ class SchedulerNode:
             try:
                 i_project: Project = Project.from_manifest(filepath=self.__manifests["input"])
             except:  # noqa E722
-                self.logger.debug("Input manifest failed to load")
-                return True
+                raise SchedulerNodeResetSilent("Input manifest failed to load")
             previous_node = SchedulerNode(i_project, self.__step, self.__index)
         else:
             # No manifest found so assume rerun is needed
-            self.logger.debug("Previous run did not generate input manifest")
-            return True
+            raise SchedulerNodeResetSilent("Previous run did not generate input manifest")
 
         previous_node_end = None
         if os.path.exists(self.__manifests["output"]):
             try:
                 o_project = Project.from_manifest(filepath=self.__manifests["output"])
             except:  # noqa E722
-                self.logger.debug("Output manifest failed to load")
-                return True
+                raise SchedulerNodeResetSilent("Output manifest failed to load")
             previous_node_end = SchedulerNode(o_project, self.__step, self.__index)
         else:
             # No manifest found so assume rerun is needed
-            self.logger.debug("Previous run did not generate output manifest")
-            return True
+            raise SchedulerNodeResetSilent("Previous run did not generate output manifest")
 
         with self.runtime():
             if previous_node_end:
                 with previous_node_end.runtime():
-                    if not self.check_previous_run_status(previous_node_end):
-                        self.logger.debug("Previous run state failed")
-                        return True
+                    self.check_previous_run_status(previous_node_end)
 
             if previous_node:
                 with previous_node.runtime():
@@ -650,18 +655,10 @@ class SchedulerNode:
                         value_keys.update(previous_value_keys)
                         path_keys.update(previous_path_keys)
                     except KeyError:
-                        self.logger.debug("Failed to acquire keys")
-                        return True
-
-                    if self.check_values_changed(previous_node, value_keys.union(path_keys)):
-                        self.logger.debug("Key values changed")
-                        return True
+                        raise SchedulerNodeResetSilent("Failed to acquire keys")
 
-                    if self.check_files_changed(previous_node, previous_node_time, path_keys):
-                        self.logger.debug("Files changed")
-                        return True
-
-        return False
+                    self.check_values_changed(previous_node, value_keys.union(path_keys))
+                    self.check_files_changed(previous_node, previous_node_time, path_keys)
 
     def setup_input_directory(self) -> None:
         """
@@ -1163,11 +1160,13 @@ class SchedulerNode:
         if 'errors' in checks:
             ordered_suffixes.append('errors')
 
+        print_paths = {}
         # Looping through patterns for each line
         with sc_open(self.__logs["exe"]) as f:
             line_count = sum(1 for _ in f)
             right_align = len(str(line_count))
             for suffix in ordered_suffixes:
+                print_paths[suffix] = False
                 # Start at the beginning of file again
                 f.seek(0)
                 for num, line in enumerate(f, start=1):
@@ -1176,7 +1175,7 @@ class SchedulerNode:
                         if string is None:
                             break
                         else:
-                            string = utils.grep(self.__project, item, string)
+                            string = utils.grep(self.__project.logger, item, string)
                     if string is not None:
                         matches[suffix] += 1
                         # always print to file
@@ -1184,11 +1183,21 @@ class SchedulerNode:
                         print(line_with_num, file=checks[suffix]['report'])
                         # selectively print to display
                         if checks[suffix]["display"]:
-                            checks[suffix]["display"](suffix, line_with_num)
+                            if matches[suffix] <= SchedulerNode.__MAX_LOG_PRINT:
+                                checks[suffix]["display"](suffix, line_with_num)
+                            else:
+                                if not print_paths[suffix]:
+                                    checks[suffix]["display"](suffix, "print limit reached")
+                                print_paths[suffix] = True
 
         for check in checks.values():
             check['report'].close()
 
+        for suffix in ordered_suffixes:
+            if print_paths[suffix]:
+                self.logger.info(f"All {suffix} can be viewed at: "
+                                 f"{os.path.abspath(f'{self.__step}.{suffix}')}")
+
         for metric in ("errors", "warnings"):
             if metric in matches:
                 value = self.__metrics.get(metric, step=self.__step, index=self.__index)
@@ -1377,3 +1386,100 @@ class SchedulerNode:
             for logfile in self.__logs.values():
                 if os.path.isfile(logfile):
                     tar.add(logfile, arcname=arcname(logfile))
+
+    def get_required_keys(self) -> Set[Tuple[str, ...]]:
+        """
+        This function walks through the 'require' keys and returns the
+        keys.
+        """
+        path_keys = set()
+        with self.runtime():
+            task = self.task
+            for key in task.get('require'):
+                path_keys.add(tuple(key.split(",")))
+            if task.has_prescript():
+                path_keys.add((*task._keypath, "prescript"))
+            if task.has_postscript():
+                path_keys.add((*task._keypath, "postscript"))
+            if task.get("refdir"):
+                path_keys.add((*task._keypath, "refdir"))
+            if task.get("script"):
+                path_keys.add((*task._keypath, "script"))
+            if task.get("exe"):
+                path_keys.add((*task._keypath, "exe"))
+
+        return path_keys
+
+    def get_required_path_keys(self) -> Set[Tuple[str, ...]]:
+        """
+        This function walks through the 'require' keys and returns the
+        keys that are of type path (file/dir).
+        """
+        path_keys = set()
+        for key in self.get_required_keys():
+            try:
+                param_type: str = self.__project.get(*key, field="type")
+                if "file" in param_type or "dir" in param_type:
+                    path_keys.add(key)
+            except KeyError:
+                # Key does not exist
+                pass
+
+        return path_keys
+
+    def mark_copy(self) -> bool:
+        """Marks files from the 'require' path keys for copying."""
+        return False
+
+    def check_required_values(self) -> bool:
+        requires = self.get_required_keys()
+
+        error = False
+        for key in sorted(requires):
+            if not self.__project.valid(*key):
+                self.logger.error(f'Cannot resolve required keypath [{",".join(key)}] '
+                                  f'for {self.step}/{self.index}.')
+                error = True
+                continue
+
+            param: Parameter = self.__project.get(*key, field=None)
+            check_step, check_index = self.step, self.index
+            if param.get(field='pernode').is_never():
+                check_step, check_index = None, None
+
+            if not param.has_value(step=check_step, index=check_index):
+                self.logger.error('No value set for required keypath '
+                                  f'[{",".join(key)}] for {self.step}/{self.index}.')
+                error = True
+                continue
+        return not error
+
+    def check_required_paths(self) -> bool:
+        if self.__project.option.get_remote():
+            return True
+
+        requires = self.get_required_path_keys()
+
+        error = False
+        for key in sorted(requires):
+            param: Parameter = self.__project.get(*key, field=None)
+            check_step, check_index = self.step, self.index
+            if param.get(field='pernode').is_never():
+                check_step, check_index = None, None
+
+            abspath = self.__project.find_files(*key,
+                                                missing_ok=True,
+                                                step=check_step, index=check_index)
+
+            unresolved_paths = param.get(step=check_step, index=check_index)
+            if not isinstance(abspath, list):
+                abspath = [abspath]
+                unresolved_paths = [unresolved_paths]
+
+            for path, setpath in zip(abspath, unresolved_paths):
+                if path is None:
+                    self.logger.error(f'Cannot resolve path {setpath} in '
+                                      f'required file keypath [{",".join(key)}] '
+                                      f'for {self.step}/{self.index}.')
+                    error = True
+        return not error