siliconcompiler 0.35.3__py3-none-any.whl → 0.36.0__py3-none-any.whl

siliconcompiler/scheduler/scheduler.py

@@ -12,9 +12,9 @@ import os.path
 
 from datetime import datetime
 
-from typing import Union, Dict, Optional, Tuple, List, TYPE_CHECKING
+from typing import Union, Dict, Optional, Tuple, List, Set, TYPE_CHECKING
 
-from siliconcompiler import NodeStatus
+from siliconcompiler import NodeStatus, Task
 from siliconcompiler.schema import Journal
 from siliconcompiler.flowgraph import RuntimeFlowgraph
 from siliconcompiler.scheduler import SchedulerNode
@@ -29,6 +29,7 @@ from siliconcompiler.utils.logging import SCLoggerFormatter
 from siliconcompiler.utils.multiprocessing import MPManager
 from siliconcompiler.scheduler import send_messages, SCRuntimeError
 from siliconcompiler.utils.paths import collectiondir, jobdir, workdir
+from siliconcompiler.utils.curation import collect
 
 if TYPE_CHECKING:
     from siliconcompiler.project import Project
@@ -101,12 +102,27 @@ class Scheduler:
         self.__metrics: "MetricSchema" = self.__project.get("metric", field="schema")
 
         self.__tasks: Dict[Tuple[str, str], SchedulerNode] = {}
+        self.__skippedtasks: Set[Tuple[str, str]] = set()
 
         # Create dummy handler
         self.__joblog_handler = logging.NullHandler()
         self.__org_job_name = self.__project.get("option", "jobname")
         self.__logfile = None
 
+        # Create tasks
+        for step, index in self.__flow.get_nodes():
+            node_cls = SchedulerNode
+
+            node_scheduler = self.__project.get('option', 'scheduler', 'name',
+                                                step=step, index=index)
+            if node_scheduler == 'slurm':
+                node_cls = SlurmSchedulerNode
+            elif node_scheduler == 'docker':
+                node_cls = DockerSchedulerNode
+            self.__tasks[(step, index)] = node_cls(self.__project, step, index)
+            if self.__flow.get(step, index, "tool") == "builtin":
+                self.__tasks[(step, index)].set_builtin()
+
     @property
     def manifest(self) -> str:
         """
@@ -270,9 +286,16 @@ class Scheduler:
             if not self.check_manifest():
                 raise SCRuntimeError("check_manifest() failed")
 
+            # Initialize schedulers
+            self.__init_schedulers()
+
             self.__run_setup()
             self.configure_nodes()
 
+            # Verify task classes
+            if not self.__check_task_classes():
+                raise SCRuntimeError("Task classes are missing")
+
             # Verify tool setups
             if not self.__check_tool_versions():
                 raise SCRuntimeError("Tools did not meet version requirements")
@@ -288,15 +311,23 @@ class Scheduler:
             if not self.__check_flowgraph_io():
                 raise SCRuntimeError("Flowgraph file IO constrains errors")
 
-            self.run_core()
+            # Collect files for remote runs
+            if self.__check_collect_files():
+                collect(self.project)
+
+            try:
+                self.run_core()
+            except SCRuntimeError as e:
+                raise e
 
-            # Store run in history
-            self.__project._record_history()
+            finally:
+                # Store run in history
+                self.__project._record_history()
 
-            # Record final manifest
-            self.__project.write_manifest(self.manifest)
+                # Record final manifest
+                self.__project.write_manifest(self.manifest)
 
-            send_messages.send(self.__project, 'summary', None, None)
+                send_messages.send(self.__project, 'summary', None, None)
         finally:
             if self.__joblog_handler is not None:
                 self.__logger.removeHandler(self.__joblog_handler)
@@ -319,50 +350,37 @@ class Scheduler:
         error = False
 
         for (step, index) in self.__flow_runtime.get_nodes():
-            scheduler = self.__project.option.scheduler.get_name(step=step, index=index)
-            check_file_access = not self.__project.option.get_remote() and scheduler is None
-
-            node = SchedulerNode(self.__project, step, index)
-            requires = []
-            with node.runtime():
-                requires = node.task.get('require')
-
-            for item in sorted(set(requires)):
-                keypath = item.split(',')
-                if not self.__project.valid(*keypath):
-                    self.__logger.error(f'Cannot resolve required keypath [{",".join(keypath)}] '
-                                        f'for {step}/{index}.')
-                    error = True
-                    continue
+            node = self.__tasks[(step, index)]
 
-                param = self.__project.get(*keypath, field=None)
-                check_step, check_index = step, index
-                if param.get(field='pernode').is_never():
-                    check_step, check_index = None, None
+            error |= not node.check_required_values()
+            error |= not node.check_required_paths()
 
-                if not param.has_value(step=check_step, index=check_index):
-                    self.__logger.error('No value set for required keypath '
-                                        f'[{",".join(keypath)}] for {step}/{index}.')
-                    error = True
-                    continue
+        return not error
 
-                paramtype = param.get(field='type')
-                if check_file_access and (('file' in paramtype) or ('dir' in paramtype)):
-                    abspath = self.__project.find_files(*keypath,
-                                                        missing_ok=True,
-                                                        step=check_step, index=check_index)
-
-                    unresolved_paths = param.get(step=check_step, index=check_index)
-                    if not isinstance(abspath, list):
-                        abspath = [abspath]
-                        unresolved_paths = [unresolved_paths]
-
-                    for path, setpath in zip(abspath, unresolved_paths):
-                        if path is None:
-                            self.__logger.error(f'Cannot resolve path {setpath} in '
-                                                f'required file keypath [{",".join(keypath)}] '
-                                                f'for {step}/{index}.')
-                            error = True
+    def __check_task_classes(self) -> bool:
+        """
+        Verifies that all runtime nodes have loaded their specific Task implementation classes.
+
+        Iterates through all nodes in the execution flow and checks if the associated
+        task object is a generic instance of the base `Task` class. If so, it indicates
+        that the specific module for that tool/task was not loaded correctly.
+
+        Returns:
+            bool: `True` if all nodes are using specialized Task subclasses, `False` if any
+            node is using the base `Task` class.
+        """
+        nodes = self.__flow_runtime.get_nodes()
+        error = False
+
+        for (step, index) in nodes:
+            tool = self.__flow.get(step, index, "tool")
+            task = self.__flow.get(step, index, "task")
+
+            task_cls = self.project.get("tool", tool, "task", task, field="schema")
+            if type(task_cls) is Task:
+                self.__logger.error(f"Invalid task: {step}/{index} did not load "
+                                    "the correct class module")
+                error = True
 
         return not error
 
@@ -414,16 +432,16 @@ class Scheduler:
                     in_task_class = self.__project.get("tool", in_tool, "task", in_task,
                                                        field="schema")
 
-                    with in_task_class.runtime(SchedulerNode(self.__project,
-                                                             in_step, in_index)) as task:
+                    with in_task_class.runtime(self.__tasks[(in_step, in_index)]) as task:
                         inputs = task.get_output_files()
 
-                with task_class.runtime(SchedulerNode(self.__project,
-                                                      step, index)) as task:
+                with task_class.runtime(self.__tasks[(step, index)]) as task:
                     for inp in inputs:
                         node_inp = task.compute_input_file_node_name(inp, in_step, in_index)
                         if node_inp in requirements:
                             inp = node_inp
+                        if inp not in requirements:
+                            continue
                         if inp in all_inputs:
                             self.__logger.error(f'Invalid flow: {step}/{index} '
                                                 f'receives {inp} from multiple input tasks')
@@ -455,7 +473,7 @@ class Scheduler:
 
         self.__record.set('status', NodeStatus.PENDING, step=step, index=index)
         for next_step, next_index in self.__flow_runtime.get_nodes_starting_at(step, index):
-            if self.__record.get('status', step=next_step, index=next_index) == NodeStatus.SKIPPED:
+            if (next_step, next_index) in self.__skippedtasks:
                 continue
 
             # Mark following steps as pending
@@ -475,18 +493,6 @@ class Scheduler:
         copy_from_nodes = set(self.__flow_load_runtime.get_nodes()).difference(
             self.__flow_runtime.get_entry_nodes())
         for step, index in self.__flow.get_nodes():
-            node_cls = SchedulerNode
-
-            node_scheduler = self.__project.get('option', 'scheduler', 'name',
-                                                step=step, index=index)
-            if node_scheduler == 'slurm':
-                node_cls = SlurmSchedulerNode
-            elif node_scheduler == 'docker':
-                node_cls = DockerSchedulerNode
-            self.__tasks[(step, index)] = node_cls(self.__project, step, index)
-            if self.__flow.get(step, index, "tool") == "builtin":
-                self.__tasks[(step, index)].set_builtin()
-
             if self.__org_job_name and (step, index) in copy_from_nodes:
                 self.__tasks[(step, index)].copy_from(self.__org_job_name)
 
@@ -590,6 +596,10 @@ class Scheduler:
             if NodeStatus.is_waiting(self.__record.get('status', step=step, index=index)):
                 with self.__tasks[(step, index)].runtime():
                     self.__tasks[(step, index)].clean_directory()
+                    parent_dir = os.path.dirname(self.__tasks[(step, index)].workdir)
+                    if os.path.exists(parent_dir) and len(os.listdir(parent_dir)) == 0:
+                        # Step directory is empty so safe to remove
+                        os.rmdir(parent_dir)
 
     def __configure_collect_previous_information(self) -> Dict[Tuple[str, str], "Project"]:
         """Collects information from previous runs for nodes that won't be re-executed.
@@ -662,10 +672,13 @@ class Scheduler:
             for step, index in layer_nodes:
                 with self.__tasks[(step, index)].runtime():
                     node_kept = self.__tasks[(step, index)].setup()
+                if not node_kept:
+                    self.__skippedtasks.add((step, index))
                 if not node_kept and (step, index) in extra_setup_nodes:
                     # remove from previous node data
                     del extra_setup_nodes[(step, index)]
 
+                # Copy in old status information, this will be overwritten if needed
                 if (step, index) in extra_setup_nodes:
                     schema = extra_setup_nodes[(step, index)]
                     node_status = None
@@ -893,6 +906,8 @@ class Scheduler:
                 if m:
                     jobid = max(jobid, int(m.group(1)))
             self.__project.set('option', 'jobname', f'{stem}{jobid + 1}')
+            for task in self.__tasks.values():
+                task._update_job()
             return True
         return False
 
@@ -926,42 +941,70 @@ class Scheduler:
 
         error = False
 
-        cwd = os.getcwd()
-        with tempfile.TemporaryDirectory(prefix="sc_tool_check") as d:
-            try:
-                versions: Dict[str, Optional[str]] = {}
+        with tempfile.TemporaryDirectory(
+                prefix=f"sc_tool_check_{self.project.option.get_jobname()}_") as d:
+            versions: Dict[str, Optional[str]] = {}
 
-                self.__logger.debug(f"Executing tool checks in: {d}")
-                os.chdir(d)
-                for (step, index) in self.__flow_runtime.get_nodes():
-                    if self.__project.option.scheduler.get_name(step=step, index=index) is not None:
-                        continue
+            self.__logger.debug(f"Executing tool checks in: {d}")
+            for (step, index) in self.__flow_runtime.get_nodes():
+                if self.__project.option.scheduler.get_name(step=step, index=index) is not None:
+                    continue
 
-                    node = SchedulerNode(self.__project, step, index)
-                    with node.runtime():
-                        try:
-                            exe = node.get_exe_path()
-                        except TaskExecutableNotReceived:
-                            continue
-                        except TaskExecutableNotFound:
-                            exe = node.task.get("exe")
-                            self.__logger.error(f"Executable for {step}/{index} could not "
-                                                f"be found: {exe}")
-                            error = True
-                            continue
+                node = self.__tasks[(step, index)]
+                with node.runtime():
+                    try:
+                        exe = node.get_exe_path()
+                    except TaskExecutableNotReceived:
+                        continue
+                    except TaskExecutableNotFound:
+                        exe = node.task.get("exe")
+                        self.__logger.error(f"Executable for {step}/{index} could not "
+                                            f"be found: {exe}")
+                        error = True
+                        continue
 
-                        try:
-                            if exe:
-                                version: Optional[str] = versions.get(exe, None)
-                                version, check = node.check_version(version)
-                                versions[exe] = version
-                                if not check:
-                                    self.__logger.error(f"Executable for {step}/{index} did not "
-                                                        "meet version checks")
-                                    error = True
-                        except NotImplementedError:
-                            self.__logger.error(f"Unable to process version for {step}/{index}")
-            finally:
-                os.chdir(cwd)
+                    try:
+                        if exe:
+                            version: Optional[str] = versions.get(exe, None)
+                            version, check = node.check_version(version, workdir=d)
+                            versions[exe] = version
+                            if not check:
+                                self.__logger.error(f"Executable for {step}/{index} did not "
+                                                    "meet version checks")
+                                error = True
+                    except NotImplementedError:
+                        self.__logger.error(f"Unable to process version for {step}/{index}")
 
         return not error
+
+    def __check_collect_files(self) -> bool:
+        """
+        Iterates through all tasks in the scheduler, and checks if the there
+        are files or directories that need to be collected
+
+        Returns:
+            bool: True if there is something to be collected, False otherwise.
+        """
+        do_collect = False
+        for task in self.__tasks.values():
+            if task.mark_copy():
+                do_collect = True
+
+        return do_collect
+
+    def __init_schedulers(self) -> None:
+        """
+        Collect and invoke unique initialization callbacks from all task schedulers.
+
+        This method gathers init functions from all SchedulerNode instances, deduplicates them
+        (since multiple tasks may share the same scheduler class), and invokes each once to
+        perform early validation (e.g., checking Docker/Slurm availability).
+        """
+        self.__logger.debug("Collecting unique scheduler initialization callbacks")
+        init_funcs = set()
+        for step, index in self.__flow_runtime.get_nodes():
+            init_funcs.add(self.__tasks[(step, index)].init)
+
+        for init in sorted(init_funcs, key=lambda func: func.__qualname__):
+            self.__logger.debug(f"Initializing scheduler: {init.__qualname__}")
+            init(self.__project)

siliconcompiler/scheduler/schedulernode.py

@@ -9,7 +9,7 @@ import time
 
 import os.path
 
-from logging.handlers import QueueHandler
+from siliconcompiler.utils.multiprocessing import MPQueueHandler as QueueHandler
 
 from typing import List, Optional, Set, Tuple, TYPE_CHECKING
 
@@ -19,7 +19,7 @@ from siliconcompiler.utils.logging import get_console_formatter, SCInRunLoggerFo
 
 from siliconcompiler.package import Resolver
 from siliconcompiler.schema_support.record import RecordTime, RecordTool
-from siliconcompiler.schema import Journal
+from siliconcompiler.schema import Journal, Parameter
 from siliconcompiler.scheduler import send_messages
 from siliconcompiler.utils.paths import workdir, jobdir, collectiondir, cwdir
 
@@ -67,9 +67,10 @@ class SchedulerNode:
     This class encapsulates the state and logic required to run a specific
     step and index, including setting up directories, handling file I/O,
     executing the associated tool, and recording results.
-
     """
 
+    __MAX_LOG_PRINT = 100  # Maximum number of warnings/error to print to log
+
     def __init__(self, project: "Project", step: str, index: str, replay: bool = False):
         """
         Initializes a SchedulerNode.
@@ -100,7 +101,6 @@ class SchedulerNode:
             self.__project.get("option", "fileset")[0],
             "topmodule")
 
-        self.__job: str = self.__project.get('option', 'jobname')
         self.__record_user_info: bool = self.__project.get(
             "option", "track", step=self.__step, index=self.__index)
         self.__pipe = None
@@ -116,24 +116,12 @@ class SchedulerNode:
         self.__enforce_inputfiles = True
         self.__enforce_outputfiles = True
 
+        self._update_job()
+
         flow: str = self.__project.get('option', 'flow')
         self.__is_entry_node: bool = (self.__step, self.__index) in \
             self.__project.get("flowgraph", flow, field="schema").get_entry_nodes()
 
-        self.__cwd = cwdir(self.__project)
-        self.__jobworkdir = jobdir(self.__project)
-        self.__workdir = workdir(self.__project, step=self.__step, index=self.__index)
-        self.__manifests = {
-            "input": os.path.join(self.__workdir, "inputs", f"{self.__name}.pkg.json"),
-            "output": os.path.join(self.__workdir, "outputs", f"{self.__name}.pkg.json")
-        }
-        self.__logs = {
-            "sc": os.path.join(self.__workdir, f"sc_{self.__step}_{self.__index}.log"),
-            "exe": os.path.join(self.__workdir, f"{self.__step}.log")
-        }
-        self.__replay_script = os.path.join(self.__workdir, "replay.sh")
-        self.__collection_path = collectiondir(self.__project)
-
         self.set_queue(None, None)
         self.__setup_schema_access()
 
@@ -258,6 +246,22 @@ class SchedulerNode:
         """Task: The task object associated with this node."""
         return self.__task
 
+    def _update_job(self):
+        self.__job: str = self.__project.get('option', 'jobname')
+        self.__cwd = cwdir(self.__project)
+        self.__jobworkdir = jobdir(self.__project)
+        self.__workdir = workdir(self.__project, step=self.__step, index=self.__index)
+        self.__manifests = {
+            "input": os.path.join(self.__workdir, "inputs", f"{self.__name}.pkg.json"),
+            "output": os.path.join(self.__workdir, "outputs", f"{self.__name}.pkg.json")
+        }
+        self.__logs = {
+            "sc": os.path.join(self.__workdir, f"sc_{self.__step}_{self.__index}.log"),
+            "exe": os.path.join(self.__workdir, f"{self.__step}.log")
+        }
+        self.__replay_script = os.path.join(self.__workdir, "replay.sh")
+        self.__collection_path = collectiondir(self.__project)
+
     def get_manifest(self, input: bool = False) -> str:
         """
         Gets the path to the input or output manifest file for this node.
@@ -891,7 +895,8 @@ class SchedulerNode:
         with self.__set_env():
             return self.__task.get_exe()
 
-    def check_version(self, version: Optional[str] = None) -> Tuple[Optional[str], bool]:
+    def check_version(self, version: Optional[str] = None,
+                      workdir: Optional[str] = None) -> Tuple[Optional[str], bool]:
         """Checks the version of the tool for this task.
 
         Compares a version string against the tool's requirements. This check
@@ -904,6 +909,8 @@ class SchedulerNode:
         Args:
             version: The version string to check. If None, the task's
                 configured version is fetched and used.
+            workdir: The working directory to use for the version check. If None,
+                the current working directory is used.
 
         Returns:
             A tuple (version_str, check_passed):
@@ -917,7 +924,7 @@ class SchedulerNode:
 
         with self.__set_env():
             if version is None:
-                version = self.__task.get_exe_version()
+                version = self.__task.get_exe_version(workdir=workdir)
 
             check = self.__task.check_exe_version(version)
 
@@ -1156,11 +1163,13 @@ class SchedulerNode:
         if 'errors' in checks:
             ordered_suffixes.append('errors')
 
+        print_paths = {}
         # Looping through patterns for each line
         with sc_open(self.__logs["exe"]) as f:
             line_count = sum(1 for _ in f)
             right_align = len(str(line_count))
             for suffix in ordered_suffixes:
+                print_paths[suffix] = False
                 # Start at the beginning of file again
                 f.seek(0)
                 for num, line in enumerate(f, start=1):
@@ -1169,7 +1178,7 @@ class SchedulerNode:
                         if string is None:
                             break
                         else:
-                            string = utils.grep(self.__project, item, string)
+                            string = utils.grep(self.__project.logger, item, string)
                     if string is not None:
                         matches[suffix] += 1
                         # always print to file
@@ -1177,11 +1186,21 @@ class SchedulerNode:
                         print(line_with_num, file=checks[suffix]['report'])
                         # selectively print to display
                         if checks[suffix]["display"]:
-                            checks[suffix]["display"](suffix, line_with_num)
+                            if matches[suffix] <= SchedulerNode.__MAX_LOG_PRINT:
+                                checks[suffix]["display"](suffix, line_with_num)
+                            else:
+                                if not print_paths[suffix]:
+                                    checks[suffix]["display"](suffix, "print limit reached")
+                                print_paths[suffix] = True
 
         for check in checks.values():
             check['report'].close()
 
+        for suffix in ordered_suffixes:
+            if print_paths[suffix]:
+                self.logger.info(f"All {suffix} can be viewed at: "
+                                 f"{os.path.abspath(f'{self.__step}.{suffix}')}")
+
         for metric in ("errors", "warnings"):
             if metric in matches:
                 value = self.__metrics.get(metric, step=self.__step, index=self.__index)
@@ -1370,3 +1389,100 @@ class SchedulerNode:
             for logfile in self.__logs.values():
                 if os.path.isfile(logfile):
                     tar.add(logfile, arcname=arcname(logfile))
+
+    def get_required_keys(self) -> Set[Tuple[str, ...]]:
+        """
+        This function walks through the 'require' keys and returns the
+        keys.
+        """
+        path_keys = set()
+        with self.runtime():
+            task = self.task
+            for key in task.get('require'):
+                path_keys.add(tuple(key.split(",")))
+            if task.has_prescript():
+                path_keys.add((*task._keypath, "prescript"))
+            if task.has_postscript():
+                path_keys.add((*task._keypath, "postscript"))
+            if task.get("refdir"):
+                path_keys.add((*task._keypath, "refdir"))
+            if task.get("script"):
+                path_keys.add((*task._keypath, "script"))
+            if task.get("exe"):
+                path_keys.add((*task._keypath, "exe"))
+
+        return path_keys
+
+    def get_required_path_keys(self) -> Set[Tuple[str, ...]]:
+        """
+        This function walks through the 'require' keys and returns the
+        keys that are of type path (file/dir).
+        """
+        path_keys = set()
+        for key in self.get_required_keys():
+            try:
+                param_type: str = self.__project.get(*key, field="type")
+                if "file" in param_type or "dir" in param_type:
+                    path_keys.add(key)
+            except KeyError:
+                # Key does not exist
+                pass
+
+        return path_keys
+
+    def mark_copy(self) -> bool:
+        """Marks files from the 'require' path keys for copying."""
+        return False
+
+    def check_required_values(self) -> bool:
+        requires = self.get_required_keys()
+
+        error = False
+        for key in sorted(requires):
+            if not self.__project.valid(*key):
+                self.logger.error(f'Cannot resolve required keypath [{",".join(key)}] '
+                                  f'for {self.step}/{self.index}.')
+                error = True
+                continue
+
+            param: Parameter = self.__project.get(*key, field=None)
+            check_step, check_index = self.step, self.index
+            if param.get(field='pernode').is_never():
+                check_step, check_index = None, None
+
+            if not param.has_value(step=check_step, index=check_index):
+                self.logger.error('No value set for required keypath '
+                                  f'[{",".join(key)}] for {self.step}/{self.index}.')
+                error = True
+                continue
+        return not error
+
+    def check_required_paths(self) -> bool:
+        if self.__project.option.get_remote():
+            return True
+
+        requires = self.get_required_path_keys()
+
+        error = False
+        for key in sorted(requires):
+            param: Parameter = self.__project.get(*key, field=None)
+            check_step, check_index = self.step, self.index
+            if param.get(field='pernode').is_never():
+                check_step, check_index = None, None
+
+            abspath = self.__project.find_files(*key,
+                                                missing_ok=True,
+                                                step=check_step, index=check_index)
+
+            unresolved_paths = param.get(step=check_step, index=check_index)
+            if not isinstance(abspath, list):
+                abspath = [abspath]
+                unresolved_paths = [unresolved_paths]
+
+            for path, setpath in zip(abspath, unresolved_paths):
+                if path is None:
+                    self.logger.error(f'Cannot resolve path {setpath} in '
+                                      f'required file keypath [{",".join(key)}] '
+                                      f'for {self.step}/{self.index}.')
+                    error = True
+        return not error