siliconcompiler 0.34.2__py3-none-any.whl → 0.34.3__py3-none-any.whl

siliconcompiler/scheduler/send_messages.py

@@ -1,3 +1,12 @@
+"""
+A module for sending email notifications about SiliconCompiler job events.
+
+This module provides functionality to send detailed email updates at various
+stages of a compilation flow (e.g., on begin, failure, or a final summary).
+It loads SMTP server credentials from a configuration file, constructs
+HTML-formatted emails with relevant job data and attachments (logs, images),
+and sends them to specified recipients.
+"""
 from siliconcompiler.utils import default_email_credentials_file, get_file_template
 import smtplib
 from email.mime.multipart import MIMEMultipart
@@ -23,6 +32,19 @@ with open(api_dir / 'email_credentials.json') as schema:
 
 
 def __load_config(chip):
+    """
+    Loads and validates email credentials from the default configuration file.
+
+    This function locates the email credentials JSON file, loads its content,
+    and validates it against a predefined JSON schema.
+
+    Args:
+        chip (Chip): The Chip object, used for logging.
+
+    Returns:
+        dict: A dictionary containing the validated email credentials. Returns
+        an empty dictionary if the file is not found or is invalid.
+    """
     path = default_email_credentials_file()
     if not os.path.exists(path):
         chip.logger.warning(f'Email credentials are not available: {path}')
@@ -39,6 +61,23 @@ def __load_config(chip):
 
 
 def send(chip, msg_type, step, index):
+    """
+    Constructs and sends an email notification for a specific job event.
+
+    This function checks if a notification is required for the given event type
+    based on the chip's configuration. If so, it assembles an email with a
+    subject, HTML body, and relevant attachments (logs or images) and sends
+    it via the configured SMTP server.
+
+    Args:
+        chip (Chip): The Chip object containing all run data and configuration.
+        msg_type (str): The type of event triggering the message (e.g., 'begin',
+            'fail', 'summary').
+        step (str): The step name associated with the event. Can be None for
+            global events.
+        index (str): The index associated with the event. Can be None for
+            global events.
+    """
     chip_step, chip_index = step, index
     if step is None:
         chip_step = Schema.GLOBAL_KEY
@@ -82,6 +121,7 @@ def send(chip, msg_type, step, index):
 
     if cred["max_file_size"] > 0:
         if msg_type == "summary":
+            # Handle summary message: attach layout image and metrics summary
             layout_img = report_utils._find_summary_image(chip)
             if layout_img and os.path.isfile(layout_img):
                 with open(layout_img, 'rb') as img_file:
@@ -109,6 +149,7 @@ def send(chip, msg_type, step, index):
                 metrics_unit=metrics_unit,
                 metric_keys=metrics_to_show)
         else:
+            # Handle general node message: attach log files and node-specific data
             # Attach logs
             for log in (f'sc_{step}_{index}.log', f'{step}.log'):
                 log_file = f'{chip.getworkdir(step=step, index=index)}/{log}'
@@ -125,43 +166,48 @@ def send(chip, msg_type, step, index):
                                               filename=f'{log_name}.txt')
                         msg.attach(log_attach)
 
-        records = {}
-        for record in chip.getkeys('record'):
-            value = None
-            if chip.get('record', record, field='pernode').is_never():
-                value = chip.get('record', record)
-            else:
-                value = chip.get('record', record, step=step, index=index)
-
-            if value is not None:
-                records[record] = value
-
-        nodes, errors, metrics, metrics_unit, metrics_to_show, _ = \
-            report_utils._collect_data(chip, flow=flow, flowgraph_nodes=[(step, index)])
-
-        status = chip.get('record', 'status', step=step, index=index)
-
-        text_msg = get_file_template('email/general.j2').render(
-            design=chip.design,
-            job=jobname,
-            step=step,
-            index=index,
-            status=status,
-            records=records,
-            nodes=nodes,
-            errors=errors,
-            metrics=metrics,
-            metrics_unit=metrics_unit,
-            metric_keys=metrics_to_show)
+            # Collect records for the specific node
+            records = {}
+            for record in chip.getkeys('record'):
+                value = None
+                if chip.get('record', record, field='pernode').is_never():
+                    value = chip.get('record', record)
+                else:
+                    value = chip.get('record', record, step=step, index=index)
+
+                if value is not None:
+                    records[record] = value
+
+            # Collect metrics for the specific node
+            nodes, errors, metrics, metrics_unit, metrics_to_show, _ = \
+                report_utils._collect_data(chip, flow=flow, flowgraph_nodes=[(step, index)])
+
+            status = chip.get('record', 'status', step=step, index=index)
+
+            # Render the general email template
+            text_msg = get_file_template('email/general.j2').render(
+                design=chip.design,
+                job=jobname,
+                step=step,
+                index=index,
+                status=status,
+                records=records,
+                nodes=nodes,
+                errors=errors,
+                metrics=metrics,
+                metrics_unit=metrics_unit,
+                metric_keys=metrics_to_show)
 
     body = MIMEText(text_msg, 'html')
     msg.attach(body)
 
+    # Determine whether to use SSL for the SMTP connection
     if cred['ssl']:
         smtp_use = smtplib.SMTP_SSL
     else:
         smtp_use = smtplib.SMTP
 
+    # Connect to the SMTP server and send the email
     with smtp_use(cred["server"], cred["port"]) as smtp_server:
         do_send = False
         try:
@@ -180,10 +226,12 @@ def send(chip, msg_type, step, index):
 
 
 if __name__ == "__main__":
+    # Example usage for testing the send function
     from siliconcompiler import Chip
     from siliconcompiler.targets import freepdk45_demo
     chip = Chip('test')
     chip.use(freepdk45_demo)
     chip.set('option', 'scheduler', 'msgevent', 'ALL')
-    # chip.set('option', 'scheduler', 'msgcontact', 'fillin')
+    # To test, uncomment the following line and fill in a valid email address
+    # chip.set('option', 'scheduler', 'msgcontact', 'your.email@example.com')
     send(chip, "BEGIN", "import", "0")

siliconcompiler/scheduler/slurm.py

@@ -1,21 +1,37 @@
+import json
 import os
 import shlex
-import subprocess
+import shutil
 import stat
+import subprocess
 import uuid
-import json
-import shutil
 
 import os.path
 
 from siliconcompiler import utils
 from siliconcompiler.package import RemoteResolver
 from siliconcompiler.flowgraph import RuntimeFlowgraph
-from siliconcompiler.scheduler.schedulernode import SchedulerNode
+from siliconcompiler.scheduler import SchedulerNode
 
 
 class SlurmSchedulerNode(SchedulerNode):
+    """A SchedulerNode implementation for running tasks on a Slurm cluster.
+
+    This class extends the base SchedulerNode to handle the specifics of
+    submitting a compilation step as a job to a Slurm workload manager.
+    It prepares a run script, a manifest, and uses the 'srun' command
+    to execute the step on a compute node.
+    """
+
     def __init__(self, chip, step, index, replay=False):
+        """Initializes a SlurmSchedulerNode.
+
+        Args:
+            chip (Chip): The parent Chip object.
+            step (str): The step name in the flowgraph.
+            index (str): The index for the step.
+            replay (bool): If True, sets up the node to replay a previous run.
+        """
         super().__init__(chip, step, index, replay=replay)
 
         # Get the temporary UID associated with this job run.
@@ -26,10 +42,22 @@ class SlurmSchedulerNode(SchedulerNode):
 
     @property
     def jobhash(self):
+        """str: A unique hash identifying the entire job run."""
         return self.__job_hash
 
     @staticmethod
     def init(chip):
+        """
+        A static pre-processing hook for the Slurm scheduler.
+
+        This method checks if the compilation flow starts from an entry node.
+        If so, it calls `chip.collect()` to gather all necessary source files
+        into a central location before any remote jobs are submitted. This
+        ensures that compute nodes have access to all required source files.
+
+        Args:
+            chip (Chip): The Chip object to perform pre-processing on.
+        """
         if os.path.exists(chip._getcollectdir()):
             # nothing to do
             return
@@ -53,26 +81,61 @@ class SlurmSchedulerNode(SchedulerNode):
 
     @property
     def is_local(self):
+        """bool: Returns False, as this node executes on a remote cluster."""
         return False
 
     @staticmethod
     def get_configuration_directory(chip):
-        '''
-        Helper function to get the configuration directory for the scheduler
-        '''
+        """Gets the directory for storing Slurm-related configuration files.
+
+        Args:
+            chip (Chip): The Chip object.
+
+        Returns:
+            str: The path to the configuration directory.
+        """
 
         return os.path.join(chip.getworkdir(), 'sc_configs')
 
     @staticmethod
     def get_job_name(jobhash, step, index):
+        """Generates a unique job name for a Slurm job.
+
+        Args:
+            jobhash (str): The unique hash for the entire run.
+            step (str): The step name of the node.
+            index (str): The index of the node.
+
+        Returns:
+            str: A unique job name string.
+        """
         return f'{jobhash}_{step}_{index}'
 
     @staticmethod
     def get_runtime_file_name(jobhash, step, index, ext):
+        """Generates a standardized filename for runtime files.
+
+        Args:
+            jobhash (str): The unique hash for the entire run.
+            step (str): The step name of the node.
+            index (str): The index of the node.
+            ext (str): The file extension.
+
+        Returns:
+            str: A standardized filename.
+        """
         return f"{SlurmSchedulerNode.get_job_name(jobhash, step, index)}.{ext}"
 
     @staticmethod
     def get_slurm_partition():
+        """Determines a default Slurm partition by querying the cluster.
+
+        Returns:
+            str: The name of the first available Slurm partition.
+
+        Raises:
+            RuntimeError: If the 'sinfo' command fails.
+        """
         partitions = subprocess.run(['sinfo', '--json'],
                                     stdout=subprocess.PIPE,
                                     stderr=subprocess.STDOUT)
@@ -86,12 +149,13 @@ class SlurmSchedulerNode(SchedulerNode):
         return sinfo['nodes'][0]['partitions'][0]
 
     def run(self):
-        '''
-        Helper method to run an individual step on a slurm cluster.
+        """
+        Runs the node's task as a job on a Slurm cluster.
 
-        Blocks until the compute node
-        finishes processing this step, and it sets the active/error bits.
-        '''
+        This method prepares all necessary files (manifest, run script),
+        constructs an 'srun' command, and submits the job. It then blocks
+        until the job completes on the compute node.
+        """
 
         self._init_run_logger()
 

siliconcompiler/scheduler/taskscheduler.py

@@ -14,10 +14,18 @@ from siliconcompiler.flowgraph import RuntimeFlowgraph
 from siliconcompiler.package import Resolver
 from siliconcompiler.schema import Journal
 
-from siliconcompiler.utils.logging import SCBlankLoggerFormatter
+from siliconcompiler.utils.logging import SCBlankLoggerFormatter, SCBlankColorlessLoggerFormatter
+from siliconcompiler.utils.multiprocessing import MPManager
 
 
 class TaskScheduler:
+    """A class for managing the execution of individual tasks in a flowgraph.
+
+    This class is responsible for the fine-grained scheduling of tasks,
+    handling multiprocessing, resource allocation (cores/threads), and
+    dependency checking. It operates on a set of pending tasks defined by the
+    main Scheduler and executes them in a loop until the flow is complete.
+    """
     __callbacks = {
         "pre_run": lambda chip: None,
         "pre_node": lambda chip, step, index: None,
@@ -27,11 +35,30 @@ class TaskScheduler:
 
     @staticmethod
     def register_callback(hook, func):
+        """Registers a callback function to be executed at a specific hook point.
+
+        Valid hooks are 'pre_run', 'pre_node', 'post_node', and 'post_run'.
+
+        Args:
+            hook (str): The name of the hook to register the callback for.
+            func (function): The function to be called. It should accept the
+                chip object and, for node hooks, the step and index as arguments.
+
+        Raises:
+            ValueError: If the specified hook is not valid.
+        """
         if hook not in TaskScheduler.__callbacks:
             raise ValueError(f"{hook} is not a valid callback")
         TaskScheduler.__callbacks[hook] = func
 
     def __init__(self, chip, tasks):
+        """Initializes the TaskScheduler.
+
+        Args:
+            chip (Chip): The Chip object containing the configuration.
+            tasks (dict): A dictionary of SchedulerNode objects keyed by
+                (step, index) tuples.
+        """
         self.__chip = chip
         self.__logger = self.__chip.logger
         self.__logger_console_handler = self.__chip._logger_console
@@ -55,7 +82,7 @@ class TaskScheduler:
             to_steps=self.__chip.get('option', 'to'),
             prune_nodes=self.__chip.get('option', 'prune'))
 
-        self.__log_queue = multiprocessing.Queue(-1)
+        self.__log_queue = MPManager.get_manager().Queue()
 
         self.__nodes = {}
         self.__startTimes = {}
@@ -64,6 +91,16 @@ class TaskScheduler:
         self.__create_nodes(tasks)
 
     def __create_nodes(self, tasks):
+        """
+        Private helper to prepare all pending tasks for execution.
+
+        This method iterates through the tasks identified by the main Scheduler,
+        creates a multiprocessing.Process for each one, and sets up pipes for
+        inter-process communication (primarily for logging and package resolution).
+
+        Args:
+            tasks (dict): A dictionary of SchedulerNode objects.
+        """
         runtime = RuntimeFlowgraph(
             self.__flow,
             from_steps=set([step for step, _ in self.__flow.get_entry_nodes()]),
@@ -75,25 +112,24 @@ class TaskScheduler:
             if self.__record.get('status', step=step, index=index) != NodeStatus.PENDING:
                 continue
 
-            with tasks[(step, index)].runtime():
-                threads = tasks[(step, index)].threads
-            if not threads:
-                threads = self.__max_threads
-            threads = max(1, min(threads, self.__max_threads))
-
             task = {
                 "name": f"{step}/{index}",
                 "inputs": runtime.get_node_inputs(step, index, record=self.__record),
                 "proc": None,
                 "parent_pipe": None,
-                "threads": threads,
+                "threads": None,
                 "running": False,
-                "manifest": os.path.join(self.__chip.getworkdir(step=step, index=index),
-                                         'outputs',
-                                         f'{self.__chip.design}.pkg.json'),
+                "manifest": None,
                 "node": tasks[(step, index)]
             }
 
+            with tasks[(step, index)].runtime():
+                threads = tasks[(step, index)].threads
+                task["manifest"] = tasks[(step, index)].get_manifest()
+            if not threads:
+                threads = self.__max_threads
+            task["threads"] = max(1, min(threads, self.__max_threads))
+
             task["parent_pipe"], pipe = multiprocessing.Pipe()
             task["node"].set_queue(pipe, self.__log_queue)
 
@@ -105,14 +141,29 @@ class TaskScheduler:
         for init_func in init_funcs:
             init_func(self.__chip)
 
-    def run(self):
+    def run(self, job_log_handler):
+        """
+        The main entry point for the task scheduling loop.
+
+        This method sets up a listener to handle logs from child processes,
+        calls the 'pre_run' callback, enters the main execution loop, and
+        handles cleanup and the 'post_run' callback.
+
+        Args:
+            job_log_handler (logging.FileHandler): The handler for the main job log file.
+        """
         # Call this in case this was invoked without __main__
         multiprocessing.freeze_support()
 
         # Handle logs across threads
-        log_listener = QueueListener(self.__log_queue, self.__logger_console_handler)
+        log_listener = QueueListener(self.__log_queue, self.__logger_console_handler,
+                                     job_log_handler)
         console_format = self.__logger_console_handler.formatter
+        file_formatter = job_log_handler.formatter
         self.__logger_console_handler.setFormatter(SCBlankLoggerFormatter())
+        job_log_handler.setFormatter(SCBlankColorlessLoggerFormatter())
+        self.__logger.removeHandler(job_log_handler)
+
         log_listener.start()
 
         # Update dashboard before run begins
@@ -123,18 +174,30 @@ class TaskScheduler:
 
         try:
             self.__run_loop()
+            TaskScheduler.__callbacks["post_run"](self.__chip)
         except KeyboardInterrupt:
             # exit immediately
             log_listener.stop()
             sys.exit(0)
-
-        TaskScheduler.__callbacks["post_run"](self.__chip)
-
-        # Cleanup logger
-        log_listener.stop()
-        self.__logger_console_handler.setFormatter(console_format)
+        finally:
+            # Cleanup logger
+            try:
+                log_listener.stop()
+            except AttributeError:
+                # Logger already stopped
+                pass
+            self.__logger_console_handler.setFormatter(console_format)
+            job_log_handler.setFormatter(file_formatter)
+            self.__logger.addHandler(job_log_handler)
 
     def __run_loop(self):
+        """
+        The core execution loop of the scheduler.
+
+        This loop continues as long as there are nodes running or waiting to
+        run. In each iteration, it processes completed nodes and launches new
+        ones whose dependencies have been met.
+        """
         self.__startTimes = {None: time.time()}
 
         while len(self.get_nodes_waiting_to_run()) > 0 or len(self.get_running_nodes()) > 0:
@@ -164,9 +227,19 @@ class TaskScheduler:
                 self.__nodes[running_nodes[0]]["proc"].join(timeout=self.__dwellTime)
 
     def get_nodes(self):
+        """Gets a sorted list of all nodes managed by this scheduler.
+
+        Returns:
+            list: A list of (step, index) tuples for all nodes.
+        """
         return sorted(self.__nodes.keys())
 
     def get_running_nodes(self):
+        """Gets a sorted list of all nodes that are currently running.
+
+        Returns:
+            list: A list of (step, index) tuples for running nodes.
+        """
         nodes = []
         for node, info in self.__nodes.items():
             if info["running"]:
@@ -174,6 +247,11 @@ class TaskScheduler:
         return sorted(nodes)
 
     def get_nodes_waiting_to_run(self):
+        """Gets a sorted list of all nodes that are pending execution.
+
+        Returns:
+            list: A list of (step, index) tuples for pending nodes.
+        """
         nodes = []
         for node, info in self.__nodes.items():
             if not info["running"] and info["proc"]:
@@ -181,6 +259,17 @@ class TaskScheduler:
         return sorted(nodes)
 
     def __process_completed_nodes(self):
+        """
+        Private helper to check for and process completed nodes.
+
+        This method iterates through running nodes, checks if their process has
+        terminated, and if so, merges their results (manifest and package cache)
+        back into the main chip object. It updates the node's status based on
+        the process exit code.
+
+        Returns:
+            bool: True if any node's status changed, False otherwise.
+        """
         changed = False
         for node in self.get_running_nodes():
             info = self.__nodes[node]
@@ -225,6 +314,18 @@ class TaskScheduler:
         return changed
 
     def __allow_start(self, node):
+        """
+        Private helper to check if a node is allowed to start based on resources.
+
+        This method checks if launching a new node would exceed the configured
+        maximum number of parallel jobs or the total available CPU cores.
+
+        Args:
+            node (tuple): The (step, index) of the node to check.
+
+        Returns:
+            bool: True if the node can be launched, False otherwise.
+        """
         info = self.__nodes[node]
 
         if not info["node"].is_local:
@@ -247,6 +348,16 @@ class TaskScheduler:
         return True
 
     def __lanuch_nodes(self):
+        """
+        Private helper to launch new nodes whose dependencies are met.
+
+        This method iterates through pending nodes, checks if all their input
+        nodes have completed successfully, and if system resources are available.
+        If all conditions are met, it starts the node's process.
+
+        Returns:
+            bool: True if any new node was launched, False otherwise.
+        """
         changed = False
         for node in self.get_nodes_waiting_to_run():
             # TODO: breakpoint logic:
@@ -298,6 +409,15 @@ class TaskScheduler:
         return changed
 
     def check(self):
+        """
+        Checks if the flow completed successfully.
+
+        This method verifies that all nodes designated as exit points in the
+        flowgraph have been successfully completed.
+
+        Raises:
+            RuntimeError: If any final steps in the flow were not reached.
+        """
         exit_steps = set([step for step, _ in self.__runtime_flow.get_exit_nodes()])
         completed_steps = set([step for step, _ in
                                self.__runtime_flow.get_completed_nodes(record=self.__record)])

siliconcompiler/schema/__init__.py

@@ -3,7 +3,6 @@ from .journal import Journal
 from .safeschema import SafeSchema
 from .editableschema import EditableSchema
 from .baseschema import BaseSchema
-from .cmdlineschema import CommandLineSchema
 from .namedschema import NamedSchema
 
 from .schema_cfg import SCHEMA_VERSION
@@ -13,7 +12,6 @@ __all__ = [
     "BaseSchema",
     "SafeSchema",
     "EditableSchema",
-    "CommandLineSchema",
     "NamedSchema",
     "Parameter",
     "Scope",