siliconcompiler 0.35.3__py3-none-any.whl → 0.36.0__py3-none-any.whl

siliconcompiler/scheduler/slurm.py

@@ -5,15 +5,18 @@ import shutil
 import stat
 import subprocess
 import uuid
+import time
 
 import os.path
 
-from siliconcompiler import utils
-from siliconcompiler.utils.curation import collect
-from siliconcompiler.utils.paths import collectiondir, jobdir
+from typing import List, Union, Final
+
+from siliconcompiler import utils, sc_open
+from siliconcompiler.utils.paths import jobdir
 from siliconcompiler.package import RemoteResolver
-from siliconcompiler.flowgraph import RuntimeFlowgraph
 from siliconcompiler.scheduler import SchedulerNode
+from siliconcompiler.utils.logging import SCBlankLoggerFormatter
+from siliconcompiler.utils.multiprocessing import MPManager
 
 
 class SlurmSchedulerNode(SchedulerNode):
@@ -24,6 +27,10 @@ class SlurmSchedulerNode(SchedulerNode):
     It prepares a run script, a manifest, and uses the 'srun' command
     to execute the step on a compute node.
     """
+    __OPTIONS: Final[str] = "scheduler-slurm"
+
+    _MAX_FS_DELAY = 2
+    _FS_DWELL = 0.1
 
     def __init__(self, project, step, index, replay=False):
         """Initializes a SlurmSchedulerNode.
@@ -52,34 +59,31 @@ class SlurmSchedulerNode(SchedulerNode):
         """
         A static pre-processing hook for the Slurm scheduler.
 
-        This method checks if the compilation flow starts from an entry node.
-        If so, it calls :meth:`.collect()` to gather all necessary source files
-        into a central location before any remote jobs are submitted. This
-        ensures that compute nodes have access to all required source files.
+        This method ensures that the Slurm environment is available and loads
+        any existing user configuration for the scheduler.
 
         Args:
             project (Project): The project object to perform pre-processing on.
         """
-        if os.path.exists(collectiondir(project)):
-            # nothing to do
-            return
-
-        do_collect = False
-        flow = project.get('option', 'flow')
-        entry_nodes = project.get("flowgraph", flow, field="schema").get_entry_nodes()
+        SlurmSchedulerNode.assert_slurm()
 
-        runtime = RuntimeFlowgraph(
-            project.get("flowgraph", flow, field='schema'),
-            from_steps=project.get('option', 'from'),
-            to_steps=project.get('option', 'to'),
-            prune_nodes=project.get('option', 'prune'))
+    @staticmethod
+    def _set_user_config(tag: str, value: Union[List[str], str]) -> None:
+        """
+        Sets a specific value in the user configuration map.
 
-        for (step, index) in runtime.get_nodes():
-            if (step, index) in entry_nodes:
-                do_collect = True
+        Args:
+            tag (str): The configuration key to update.
+            value (Union[List[str], str]): The value to assign to the key.
+        """
+        MPManager.get_settings().set(SlurmSchedulerNode.__OPTIONS, tag, value)
 
-        if do_collect:
-            collect(project)
+    @staticmethod
+    def _write_user_config() -> None:
+        """
+        Writes the current system configuration to the user configuration file.
+        """
+        MPManager.get_settings().save()
 
     @property
     def is_local(self):
@@ -150,6 +154,49 @@ class SlurmSchedulerNode(SchedulerNode):
         # Return the first listed partition
         return sinfo['nodes'][0]['partitions'][0]
 
+    @staticmethod
+    def assert_slurm() -> None:
+        """
+        Check if slurm is installed and throw error when not installed.
+        """
+        if shutil.which('sinfo') is None:
+            raise RuntimeError('slurm is not available or installed on this machine')
+
+    def mark_copy(self) -> bool:
+        sharedprefix: List[str] = MPManager.get_settings().get(
+            SlurmSchedulerNode.__OPTIONS, "sharedpaths", default=[])
+
+        if "/" in sharedprefix:
+            # Entire filesystem is shared so no need to check
+            return False
+
+        do_collect = False
+        for key in self.get_required_path_keys():
+            mark_copy = True
+            if sharedprefix:
+                mark_copy = False
+
+                check_step, check_index = self.step, self.index
+                if self.project.get(*key, field='pernode').is_never():
+                    check_step, check_index = None, None
+
+                paths = self.project.find_files(*key, missing_ok=True,
+                                                step=check_step, index=check_index)
+                if not isinstance(paths, list):
+                    paths = [paths]
+                paths = [str(path) for path in paths if path]
+
+                for path in paths:
+                    if not any([path.startswith(shared) for shared in sharedprefix]):
+                        # File exists outside shared paths and needs to be copied
+                        mark_copy = True
+                        break
+
+            if mark_copy:
+                self.project.set(*key, True, field='copy')
+                do_collect = True
+        return do_collect
+
     def run(self):
         """
         Runs the node's task as a job on a Slurm cluster.
@@ -161,12 +208,8 @@ class SlurmSchedulerNode(SchedulerNode):
 
         self._init_run_logger()
 
-        if shutil.which('sinfo') is None:
-            raise RuntimeError('slurm is not available or installed on this machine')
-
         # Determine which cluster parititon to use.
-        partition = self.project.get('option', 'scheduler', 'queue',
-                                     step=self.step, index=self.index)
+        partition = self.project.option.scheduler.get_queue(step=self.step, index=self.index)
         if not partition:
             partition = SlurmSchedulerNode.get_slurm_partition()
 
@@ -191,7 +234,7 @@ class SlurmSchedulerNode(SchedulerNode):
             with open(script_file, 'w') as sf:
                 sf.write(utils.get_file_template('slurm/run.sh').render(
                     cfg_file=shlex.quote(cfg_file),
-                    build_dir=shlex.quote(self.project.get("option", "builddir")),
+                    build_dir=shlex.quote(self.project.option.get_builddir()),
                     step=shlex.quote(self.step),
                     index=shlex.quote(self.index),
                     cachedir=shlex.quote(str(RemoteResolver.determine_cache_dir(self.project)))
@@ -202,7 +245,6 @@ class SlurmSchedulerNode(SchedulerNode):
                  os.stat(script_file).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
 
         schedule_cmd = ['srun',
-                        '--exclusive',
                         '--partition', partition,
                         '--chdir', self.project_cwd,
                         '--job-name', SlurmSchedulerNode.get_job_name(self.__job_hash,
@@ -210,16 +252,22 @@ class SlurmSchedulerNode(SchedulerNode):
                         '--output', log_file]
 
         # Only delay the starting time if the 'defer' Schema option is specified.
-        defer_time = self.project.get('option', 'scheduler', 'defer',
-                                      step=self.step, index=self.index)
+        defer_time = self.project.option.scheduler.get_defer(step=self.step, index=self.index)
         if defer_time:
             schedule_cmd.extend(['--begin', defer_time])
 
+        # Forward additional user options
+        options = self.project.option.scheduler.get_options(step=self.step, index=self.index)
+        if options:
+            schedule_cmd.extend(options)
+
         schedule_cmd.append(script_file)
 
+        self.logger.debug(f"Executing slurm command: {shlex.join(schedule_cmd)}")
+
         # Run the 'srun' command, and track its output.
-        # TODO: output should be fed to log, and stdout if quiet = False
         step_result = subprocess.Popen(schedule_cmd,
+                                       stdin=subprocess.DEVNULL,
                                        stdout=subprocess.PIPE,
                                        stderr=subprocess.STDOUT)
 
@@ -227,3 +275,40 @@ class SlurmSchedulerNode(SchedulerNode):
         # as it has closed its output stream. But if we don't call '.wait()',
         # the '.returncode' value will not be set correctly.
         step_result.wait()
+
+        # Attempt to list dir to trigger network FS to update
+        try:
+            os.listdir(os.path.dirname(log_file))
+        except:  # noqa E722
+            pass
+
+        # Print the log to logger
+        if os.path.exists(log_file):
+            org_formatter = self.project._logger_console.formatter
+            try:
+                self.project._logger_console.setFormatter(SCBlankLoggerFormatter())
+                with sc_open(log_file) as log:
+                    for line in log.readlines():
+                        self.logger.info(line.rstrip())
+            finally:
+                self.project._logger_console.setFormatter(org_formatter)
+
+        if step_result.returncode != 0:
+            self.logger.error(f"Slurm exited with a non-zero code ({step_result.returncode}).")
+            if os.path.exists(log_file):
+                self.logger.error(f"Node log file: {log_file}")
+            self.halt()
+
+        # Wait for manifest to propagate through network filesystem
+        start = time.time()
+        elapsed = 0
+        manifest_path = self.get_manifest()
+        while not os.path.exists(manifest_path) and elapsed <= SlurmSchedulerNode._MAX_FS_DELAY:
+            os.listdir(os.path.dirname(manifest_path))
+            elapsed = time.time() - start
+            time.sleep(SlurmSchedulerNode._FS_DWELL)
+        if not os.path.exists(manifest_path):
+            self.logger.error(f"Manifest was not created on time: {manifest_path}")
+
+    def check_required_paths(self) -> bool:
+        return True

siliconcompiler/scheduler/taskscheduler.py

@@ -5,7 +5,7 @@ import time
 
 import os.path
 
-from typing import List, Dict, Tuple, Optional, Callable, ClassVar, Any, Literal, TYPE_CHECKING
+from typing import List, Dict, Tuple, Optional, Callable, Any, Literal, TYPE_CHECKING
 
 from logging.handlers import QueueListener
 
@@ -35,12 +35,6 @@ class TaskScheduler:
     dependency checking. It operates on a set of pending tasks defined by the
     main Scheduler and executes them in a loop until the flow is complete.
     """
-    __callbacks: ClassVar[Dict[str, Callable[..., None]]] = {
-        "pre_run": lambda project: None,
-        "pre_node": lambda project, step, index: None,
-        "post_node": lambda project, step, index: None,
-        "post_run": lambda project: None,
-    }
 
     @staticmethod
     def register_callback(hook: Literal["pre_run", "pre_node", "post_node", "post_run"],
@@ -57,9 +51,9 @@ class TaskScheduler:
         Raises:
             ValueError: If the specified hook is not valid.
         """
-        if hook not in TaskScheduler.__callbacks:
+        if hook not in ('pre_run', 'pre_node', 'post_node', 'post_run'):
             raise ValueError(f"{hook} is not a valid callback")
-        TaskScheduler.__callbacks[hook] = func
+        MPManager.get_transient_settings().set('TaskScheduler', hook, func)
 
     def __init__(self, project: "Project", tasks: Dict[Tuple[str, str], "SchedulerNode"]):
         """Initializes the TaskScheduler.
@@ -117,8 +111,6 @@ class TaskScheduler:
             from_steps=set([step for step, _ in self.__flow.get_entry_nodes()]),
             prune_nodes=self.__project.option.get_prune())
 
-        init_funcs = set()
-
         for step, index in self.__runtime_flow.get_nodes():
             if self.__record.get('status', step=step, index=index) != NodeStatus.PENDING:
                 continue
@@ -141,11 +133,7 @@ class TaskScheduler:
                 threads = self.__max_threads
             task["threads"] = max(1, min(threads, self.__max_threads))
 
-            task["parent_pipe"], pipe = multiprocessing.Pipe()
-            task["node"].set_queue(pipe, self.__log_queue)
-
             task["proc"] = multiprocessing.Process(target=task["node"].run)
-            init_funcs.add(task["node"].init)
             self.__nodes[(step, index)] = task
 
         # Create ordered list of nodes
@@ -155,10 +143,6 @@ class TaskScheduler:
                 if node in self.__nodes:
                     self.__ordered_nodes.append(node)
 
-        # Call preprocessing for schedulers
-        for init_func in init_funcs:
-            init_func(self.__project)
-
     def run(self, job_log_handler: logging.Handler) -> None:
         """
         The main entry point for the task scheduling loop.
@@ -188,11 +172,13 @@ class TaskScheduler:
         if self.__dashboard:
             self.__dashboard.update_manifest()
 
-        TaskScheduler.__callbacks["pre_run"](self.__project)
+        MPManager.get_transient_settings().get(
+            'TaskScheduler', 'pre_run', lambda project: None)(self.__project)
 
         try:
             self.__run_loop()
-            TaskScheduler.__callbacks["post_run"](self.__project)
+            MPManager.get_transient_settings().get(
+                'TaskScheduler', 'post_run', lambda project: None)(self.__project)
         except KeyboardInterrupt:
             # exit immediately
             log_listener.stop()
@@ -311,6 +297,10 @@ class TaskScheduler:
                     except:  # noqa E722
                         pass
 
+                # Remove pipe
+                info["parent_pipe"] = None
+                info["node"].set_queue(None, None)
+
                 step, index = node
                 if info["proc"].exitcode > 0:
                     status = NodeStatus.ERROR
@@ -326,7 +316,9 @@ class TaskScheduler:
 
                 changed = True
 
-                TaskScheduler.__callbacks['post_node'](self.__project, step, index)
+                MPManager.get_transient_settings().get(
+                    'TaskScheduler', 'post_node',
+                    lambda project, step, index: None)(self.__project, step, index)
 
         return changed
 
@@ -413,7 +405,9 @@ class TaskScheduler:
             if ready and self.__allow_start(node):
                 self.__logger.debug(f'Launching {info["name"]}')
 
-                TaskScheduler.__callbacks['pre_node'](self.__project, step, index)
+                MPManager.get_transient_settings().get(
+                    'TaskScheduler', 'pre_node',
+                    lambda project, step, index: None)(self.__project, step, index)
 
                 self.__record.set('status', NodeStatus.RUNNING, step=step, index=index)
                 self.__startTimes[node] = time.time()
@@ -421,6 +415,8 @@ class TaskScheduler:
 
                 # Start the process
                 info["running"] = True
+                info["parent_pipe"], pipe = multiprocessing.Pipe()
+                info["node"].set_queue(pipe, self.__log_queue)
                 info["proc"].start()
 
         return changed

siliconcompiler/schema/_metadata.py

@@ -1,2 +1,2 @@
 # Version number following semver standard.
-version = '0.52.1'
+version = '0.53.1'

siliconcompiler/schema/editableschema.py

@@ -6,6 +6,7 @@
 
 from .parameter import Parameter
 from .baseschema import BaseSchema
+from .namedschema import NamedSchema
 
 from typing import Union, Tuple
 
@@ -38,6 +39,11 @@ class EditableSchema:
             if isinstance(value, BaseSchema):
                 value._BaseSchema__parent = self.__schema
                 value._BaseSchema__key = key
+                if isinstance(value, NamedSchema):
+                    if key == "default":
+                        value._NamedSchema__name = None
+                    else:
+                        value._NamedSchema__name = key
 
             if key == "default":
                 self.__schema._BaseSchema__default = value
@@ -146,3 +152,26 @@ class EditableSchema:
             raise ValueError("Keypath must only be strings")
 
         return self.__schema._BaseSchema__search(*keypath, require_leaf=False)
+
+    def copy(self) -> BaseSchema:
+        '''
+        Creates a copy of the schema object, disconnected from any parent schema
+        '''
+
+        new_schema = self.__schema.copy()
+        if new_schema._parent() is not new_schema:
+            new_schema._BaseSchema__parent = None
+        return new_schema
+
+    def rename(self, name: str):
+        '''
+        Renames a named schema
+        '''
+
+        if not isinstance(self.__schema, NamedSchema):
+            raise TypeError("schema must be a named schema")
+
+        if self.__schema._parent() is not self.__schema:
+            raise ValueError("object is already in a schema")
+
+        self.__schema._NamedSchema__name = name

siliconcompiler/schema/namedschema.py

@@ -44,12 +44,10 @@ class NamedSchema(BaseSchema):
         """
 
         try:
-            if self.__name is not None:
-                raise RuntimeError("Cannot call set_name more than once.")
+            if self._parent() is not self:
+                raise RuntimeError("Cannot call set_name after it has been inserted into schema.")
         except AttributeError:
             pass
-        if name is not None and "." in name:
-            raise ValueError("Named schema object cannot contains: .")
         self.__name = name
 
     def type(self) -> str:

siliconcompiler/schema/parametervalue.py

@@ -185,7 +185,13 @@ class NodeListValue:
         '''
         Returns a copy of the values stored in the list
         '''
-        return self.__values.copy()
+        if self.__values:
+            return self.__values.copy()
+        else:
+            if self.__base.has_value:
+                return [self.__base]
+            else:
+                return []
 
     def copy(self) -> "NodeListValue":
         """
@@ -385,7 +391,13 @@ class NodeSetValue:
         '''
         Returns a copy of the values stored in the list
         '''
-        return self.__values.copy()
+        if self.__values:
+            return self.__values.copy()
+        else:
+            if self.__base.has_value:
+                return [self.__base]
+            else:
+                return []
 
     def copy(self) -> "NodeSetValue":
         """

siliconcompiler/schema_support/cmdlineschema.py

@@ -172,9 +172,6 @@ class CommandLineSchema(BaseSchema):
 
         # Iterate over all keys from an empty schema to add parser arguments
         for keypath in sorted(keyschema.allkeys()):
-            if keypath == ("option", "cfg"):  # TODO: remove this when cfg is removed from schema
-                continue
-
             param: Parameter = keyschema.get(*keypath, field=None)
 
             dest, switches = param.add_commandline_arguments(

siliconcompiler/schema_support/dependencyschema.py

@@ -210,12 +210,6 @@ class DependencySchema(BaseSchema):
 
         if name:
             if not self.has_dep(name):
-                if "." in name:
-                    name0, *name1 = name.split(".")
-                    subdep = self.get_dep(name0)
-                    if isinstance(subdep, DependencySchema):
-                        return subdep.get_dep(".".join(name1))
-                    raise KeyError(f"{name} does not contain dependency information")
                 raise KeyError(f"{name} is not an imported module")
 
             return self.__deps[name]

siliconcompiler/schema_support/option.py

@@ -1,7 +1,8 @@
-from typing import Union, List, Tuple, Callable, Dict, Optional
+from typing import Union, List, Tuple, Callable, Dict, Optional, Final
 
 from siliconcompiler.schema import BaseSchema, EditableSchema, Parameter, Scope, PerNode
 from siliconcompiler.schema.utils import trim
+from siliconcompiler.utils.multiprocessing import MPManager
 
 
 class SchedulerSchema(BaseSchema):
@@ -427,6 +428,8 @@ class OptionSchema(BaseSchema):
     compiler's behavior, such as flow control, logging, build settings, and
     remote execution. It provides getter and setter methods for each parameter.
     """
+    __OPTIONS: Final[str] = "schema-options"
+
     def __init__(self):
         """Initializes the options schema and defines all its parameters."""
         super().__init__()
@@ -844,6 +847,84 @@ class OptionSchema(BaseSchema):
 
         schema.insert('scheduler', SchedulerSchema())
 
+        self.__load_defaults()
+
+    def __load_defaults(self) -> None:
+        """Loads and applies settings from the default options file.
+
+        This method reads the configuration file specified by the settings
+        manager. It iterates through the list of option
+        objects in the file.
+
+        For each object, it checks for a "key" and a "value". If the key
+        is recognized (exists in `self.allkeys()`), it attempts to apply
+        the value using `self.set()`.
+
+        Errors during value setting (`ValueError`) are silently ignored.
+        """
+        options = MPManager.get_settings().get_category(OptionSchema.__OPTIONS)
+
+        if not options:
+            return
+
+        allkeys = self.allkeys()
+        for key, value in options.items():
+            if key is None:
+                continue
+
+            key = tuple(key.split(","))
+            if key not in allkeys:
+                continue
+
+            try:
+                self.set(*key, value)
+            except ValueError:
+                pass
+
+    def write_defaults(self) -> None:
+        """Saves all non-default settings to the configuration file.
+
+        This method iterates through all parameters known to the system
+        (via `self.allkeys()`). It compares the current value of each
+        parameter against its default value.
+
+        Any parameter whose current value differs from its default is
+        collected. This list of non-default settings is then
+        serialized as a JSON array to the file specified by
+        `default_options_file()`.
+
+        If all parameters are set to their default values, the list
+        will be empty, and no file will be written.
+        """
+        transientkeys = {
+            # Flow information
+            ("flow",),
+            ("from",),
+            ("to",),
+            ("prune",),
+
+            # Design information
+            ("design",),
+            ("alias",),
+            ("fileset",),
+        }
+
+        settings = MPManager.get_settings()
+        settings.delete(OptionSchema.__OPTIONS)
+
+        for key in self.allkeys():
+            if key in transientkeys:
+                continue
+
+            param: Parameter = self.get(*key, field=None)
+
+            value = param.get()
+            if value != param.default.get():
+                settings.set(OptionSchema.__OPTIONS, ",".join(key), value)
+
+        if settings.get_category(OptionSchema.__OPTIONS):
+            settings.save()
+
     # Getters and Setters
     def get_remote(self) -> bool:
         """Gets the remote processing flag.

siliconcompiler/schema_support/pathschema.py

@@ -11,7 +11,7 @@ from siliconcompiler.schema.parameter import Parameter, Scope
 from siliconcompiler.schema.utils import trim
 
 from siliconcompiler.package import Resolver
-from siliconcompiler.utils.paths import collectiondir
+from siliconcompiler.utils.paths import collectiondir, cwdirsafe
 
 
 class PathSchemaBase(BaseSchema):
@@ -51,14 +51,12 @@ class PathSchemaBase(BaseSchema):
             the schema.
         """
         schema_root = self._parent(root=True)
-        cwd = getattr(schema_root, "_Project__cwd", os.getcwd())
-        collection_dir = collectiondir(schema_root)
 
         return super()._find_files(*keypath,
                                    missing_ok=missing_ok,
                                    step=step, index=index,
-                                   collection_dir=collection_dir,
-                                   cwd=cwd)
+                                   collection_dir=collectiondir(schema_root),
+                                   cwd=cwdirsafe(schema_root))
 
     def check_filepaths(self, ignore_keys: Optional[List[Tuple[str, ...]]] = None) -> bool:
         '''
@@ -71,17 +69,15 @@ class PathSchemaBase(BaseSchema):
             True if all file paths are valid, otherwise False.
         '''
         schema_root = self._parent(root=True)
-        cwd = getattr(schema_root, "_Project__cwd", os.getcwd())
         logger = getattr(schema_root,
                          "logger",
                          logging.getLogger("siliconcompiler.check_filepaths"))
-        collection_dir = collectiondir(schema_root)
 
         return super()._check_filepaths(
             ignore_keys=ignore_keys,
             logger=logger,
-            collection_dir=collection_dir,
-            cwd=cwd)
+            collection_dir=collectiondir(schema_root),
+            cwd=cwdirsafe(schema_root))
 
     def hash_files(self, *keypath: str,
                    update: bool = True,
@@ -126,11 +122,9 @@ class PathSchemaBase(BaseSchema):
             Computes, stores, and returns hashes of files in :keypath:`input, rtl, verilog`.
         '''
         schema_root = self._parent(root=True)
-        cwd = getattr(schema_root, "_Project__cwd", os.getcwd())
         logger = getattr(schema_root,
                          "logger",
                          logging.getLogger("siliconcompiler.hash_files"))
-        collection_dir = collectiondir(schema_root)
 
         if verbose:
             logger.info(f"Computing hash value for [{','.join([*self._keypath, *keypath])}]")
@@ -138,8 +132,8 @@ class PathSchemaBase(BaseSchema):
         hashes = super()._hash_files(*keypath,
                                      missing_ok=missing_ok,
                                      step=step, index=index,
-                                     collection_dir=collection_dir,
-                                     cwd=cwd)
+                                     collection_dir=collectiondir(schema_root),
+                                     cwd=cwdirsafe(schema_root))
 
         if check:
             check_hashes = self.get(*keypath, field="filehash", step=step, index=index)

siliconcompiler/schema_support/record.py

@@ -15,6 +15,7 @@ from enum import Enum
 from siliconcompiler.schema import BaseSchema, LazyLoad
 from siliconcompiler.schema import EditableSchema, Parameter, PerNode, Scope
 from siliconcompiler.schema.utils import trim
+from siliconcompiler.utils.multiprocessing import MPManager
 
 from siliconcompiler import _metadata
 
@@ -138,9 +139,9 @@ class RecordSchema(BaseSchema):
             "region": str
             }
         '''
-        # TODO: add logic to figure out if we're running on a remote cluster and
-        # extract the region in a provider-specific way.
-        return {"region": "local"}
+
+        region = MPManager().get_settings().get("record", "region", default="local")
+        return {"region": region}
 
     @staticmethod
     def get_ip_information() -> Dict[str, Optional[str]]: