siliconcompiler 0.35.4__py3-none-any.whl → 0.36.1__py3-none-any.whl

siliconcompiler/scheduler/scheduler.py

@@ -14,7 +14,7 @@ from datetime import datetime
 
 from typing import Union, Dict, Optional, Tuple, List, Set, TYPE_CHECKING
 
-from siliconcompiler import NodeStatus
+from siliconcompiler import NodeStatus, Task
 from siliconcompiler.schema import Journal
 from siliconcompiler.flowgraph import RuntimeFlowgraph
 from siliconcompiler.scheduler import SchedulerNode
@@ -292,6 +292,10 @@ class Scheduler:
             self.__run_setup()
             self.configure_nodes()
 
+            # Verify task classes
+            if not self.__check_task_classes():
+                raise SCRuntimeError("Task classes are missing")
+
             # Verify tool setups
             if not self.__check_tool_versions():
                 raise SCRuntimeError("Tools did not meet version requirements")
@@ -353,6 +357,33 @@ class Scheduler:
 
         return not error
 
+    def __check_task_classes(self) -> bool:
+        """
+        Verifies that all runtime nodes have loaded their specific Task implementation classes.
+
+        Iterates through all nodes in the execution flow and checks if the associated
+        task object is a generic instance of the base `Task` class. If so, it indicates
+        that the specific module for that tool/task was not loaded correctly.
+
+        Returns:
+            bool: `True` if all nodes are using specialized Task subclasses, `False` if any
+            node is using the base `Task` class.
+        """
+        nodes = self.__flow_runtime.get_nodes()
+        error = False
+
+        for (step, index) in nodes:
+            tool = self.__flow.get(step, index, "tool")
+            task = self.__flow.get(step, index, "task")
+
+            task_cls = self.project.get("tool", tool, "task", task, field="schema")
+            if type(task_cls) is Task:
+                self.__logger.error(f"Invalid task: {step}/{index} did not load "
+                                    "the correct class module")
+                error = True
+
+        return not error
+
     def __check_flowgraph_io(self) -> bool:
         """
         Validate that every runtime node will receive its required input files and that no
@@ -409,6 +440,8 @@ class Scheduler:
                         node_inp = task.compute_input_file_node_name(inp, in_step, in_index)
                         if node_inp in requirements:
                             inp = node_inp
+                        if inp not in requirements:
+                            continue
                         if inp in all_inputs:
                             self.__logger.error(f'Invalid flow: {step}/{index} '
                                                 f'receives {inp} from multiple input tasks')
@@ -908,43 +941,39 @@ class Scheduler:
 
         error = False
 
-        cwd = os.getcwd()
-        with tempfile.TemporaryDirectory(prefix="sc_tool_check") as d:
-            try:
-                versions: Dict[str, Optional[str]] = {}
+        with tempfile.TemporaryDirectory(
+                prefix=f"sc_tool_check_{self.project.option.get_jobname()}_") as d:
+            versions: Dict[str, Optional[str]] = {}
 
-                self.__logger.debug(f"Executing tool checks in: {d}")
-                os.chdir(d)
-                for (step, index) in self.__flow_runtime.get_nodes():
-                    if self.__project.option.scheduler.get_name(step=step, index=index) is not None:
-                        continue
+            self.__logger.debug(f"Executing tool checks in: {d}")
+            for (step, index) in self.__flow_runtime.get_nodes():
+                if self.__project.option.scheduler.get_name(step=step, index=index) is not None:
+                    continue
 
-                    node = self.__tasks[(step, index)]
-                    with node.runtime():
-                        try:
-                            exe = node.get_exe_path()
-                        except TaskExecutableNotReceived:
-                            continue
-                        except TaskExecutableNotFound:
-                            exe = node.task.get("exe")
-                            self.__logger.error(f"Executable for {step}/{index} could not "
-                                                f"be found: {exe}")
-                            error = True
-                            continue
+                node = self.__tasks[(step, index)]
+                with node.runtime():
+                    try:
+                        exe = node.get_exe_path()
+                    except TaskExecutableNotReceived:
+                        continue
+                    except TaskExecutableNotFound:
+                        exe = node.task.get("exe")
+                        self.__logger.error(f"Executable for {step}/{index} could not "
+                                            f"be found: {exe}")
+                        error = True
+                        continue
 
-                        try:
-                            if exe:
-                                version: Optional[str] = versions.get(exe, None)
-                                version, check = node.check_version(version)
-                                versions[exe] = version
-                                if not check:
-                                    self.__logger.error(f"Executable for {step}/{index} did not "
-                                                        "meet version checks")
-                                    error = True
-                        except NotImplementedError:
-                            self.__logger.error(f"Unable to process version for {step}/{index}")
-            finally:
-                os.chdir(cwd)
+                    try:
+                        if exe:
+                            version: Optional[str] = versions.get(exe, None)
+                            version, check = node.check_version(version, workdir=d)
+                            versions[exe] = version
+                            if not check:
+                                self.__logger.error(f"Executable for {step}/{index} did not "
+                                                    "meet version checks")
+                                error = True
+                    except NotImplementedError:
+                        self.__logger.error(f"Unable to process version for {step}/{index}")
 
         return not error
 

siliconcompiler/scheduler/schedulernode.py

@@ -895,7 +895,8 @@ class SchedulerNode:
         with self.__set_env():
             return self.__task.get_exe()
 
-    def check_version(self, version: Optional[str] = None) -> Tuple[Optional[str], bool]:
+    def check_version(self, version: Optional[str] = None,
+                      workdir: Optional[str] = None) -> Tuple[Optional[str], bool]:
         """Checks the version of the tool for this task.
 
         Compares a version string against the tool's requirements. This check
@@ -908,6 +909,8 @@ class SchedulerNode:
         Args:
             version: The version string to check. If None, the task's
                 configured version is fetched and used.
+            workdir: The working directory to use for the version check. If None,
+                the current working directory is used.
 
         Returns:
             A tuple (version_str, check_passed):
@@ -921,7 +924,7 @@ class SchedulerNode:
 
         with self.__set_env():
             if version is None:
-                version = self.__task.get_exe_version()
+                version = self.__task.get_exe_version(workdir=workdir)
 
             check = self.__task.check_exe_version(version)
 

siliconcompiler/scheduler/slurm.py

@@ -209,8 +209,7 @@ class SlurmSchedulerNode(SchedulerNode):
         self._init_run_logger()
 
         # Determine which cluster parititon to use.
-        partition = self.project.get('option', 'scheduler', 'queue',
-                                     step=self.step, index=self.index)
+        partition = self.project.option.scheduler.get_queue(step=self.step, index=self.index)
         if not partition:
             partition = SlurmSchedulerNode.get_slurm_partition()
 
@@ -246,7 +245,6 @@ class SlurmSchedulerNode(SchedulerNode):
                  os.stat(script_file).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
 
         schedule_cmd = ['srun',
-                        '--exclusive',
                         '--partition', partition,
                         '--chdir', self.project_cwd,
                         '--job-name', SlurmSchedulerNode.get_job_name(self.__job_hash,
@@ -254,17 +252,20 @@ class SlurmSchedulerNode(SchedulerNode):
                         '--output', log_file]
 
         # Only delay the starting time if the 'defer' Schema option is specified.
-        defer_time = self.project.get('option', 'scheduler', 'defer',
-                                      step=self.step, index=self.index)
+        defer_time = self.project.option.scheduler.get_defer(step=self.step, index=self.index)
         if defer_time:
             schedule_cmd.extend(['--begin', defer_time])
 
+        # Forward additional user options
+        options = self.project.option.scheduler.get_options(step=self.step, index=self.index)
+        if options:
+            schedule_cmd.extend(options)
+
         schedule_cmd.append(script_file)
 
         self.logger.debug(f"Executing slurm command: {shlex.join(schedule_cmd)}")
 
         # Run the 'srun' command, and track its output.
-        # TODO: output should be fed to log, and stdout if quiet = False
         step_result = subprocess.Popen(schedule_cmd,
                                        stdin=subprocess.DEVNULL,
                                        stdout=subprocess.PIPE,

siliconcompiler/scheduler/taskscheduler.py

@@ -5,7 +5,7 @@ import time
 
 import os.path
 
-from typing import List, Dict, Tuple, Optional, Callable, ClassVar, Any, Literal, TYPE_CHECKING
+from typing import List, Dict, Tuple, Optional, Callable, Any, Literal, TYPE_CHECKING
 
 from logging.handlers import QueueListener
 
@@ -35,12 +35,6 @@ class TaskScheduler:
     dependency checking. It operates on a set of pending tasks defined by the
     main Scheduler and executes them in a loop until the flow is complete.
     """
-    __callbacks: ClassVar[Dict[str, Callable[..., None]]] = {
-        "pre_run": lambda project: None,
-        "pre_node": lambda project, step, index: None,
-        "post_node": lambda project, step, index: None,
-        "post_run": lambda project: None,
-    }
 
     @staticmethod
     def register_callback(hook: Literal["pre_run", "pre_node", "post_node", "post_run"],
@@ -57,9 +51,9 @@ class TaskScheduler:
         Raises:
             ValueError: If the specified hook is not valid.
         """
-        if hook not in TaskScheduler.__callbacks:
+        if hook not in ('pre_run', 'pre_node', 'post_node', 'post_run'):
             raise ValueError(f"{hook} is not a valid callback")
-        TaskScheduler.__callbacks[hook] = func
+        MPManager.get_transient_settings().set('TaskScheduler', hook, func)
 
     def __init__(self, project: "Project", tasks: Dict[Tuple[str, str], "SchedulerNode"]):
         """Initializes the TaskScheduler.
@@ -139,9 +133,6 @@ class TaskScheduler:
                 threads = self.__max_threads
             task["threads"] = max(1, min(threads, self.__max_threads))
 
-            task["parent_pipe"], pipe = multiprocessing.Pipe()
-            task["node"].set_queue(pipe, self.__log_queue)
-
             task["proc"] = multiprocessing.Process(target=task["node"].run)
             self.__nodes[(step, index)] = task
 
@@ -181,11 +172,13 @@ class TaskScheduler:
         if self.__dashboard:
             self.__dashboard.update_manifest()
 
-        TaskScheduler.__callbacks["pre_run"](self.__project)
+        MPManager.get_transient_settings().get(
+            'TaskScheduler', 'pre_run', lambda project: None)(self.__project)
 
         try:
             self.__run_loop()
-            TaskScheduler.__callbacks["post_run"](self.__project)
+            MPManager.get_transient_settings().get(
+                'TaskScheduler', 'post_run', lambda project: None)(self.__project)
         except KeyboardInterrupt:
             # exit immediately
             log_listener.stop()
@@ -304,6 +297,10 @@ class TaskScheduler:
                     except:  # noqa E722
                         pass
 
+                # Remove pipe
+                info["parent_pipe"] = None
+                info["node"].set_queue(None, None)
+
                 step, index = node
                 if info["proc"].exitcode > 0:
                     status = NodeStatus.ERROR
@@ -319,7 +316,9 @@ class TaskScheduler:
 
                 changed = True
 
-                TaskScheduler.__callbacks['post_node'](self.__project, step, index)
+                MPManager.get_transient_settings().get(
+                    'TaskScheduler', 'post_node',
+                    lambda project, step, index: None)(self.__project, step, index)
 
         return changed
 
@@ -406,7 +405,9 @@ class TaskScheduler:
             if ready and self.__allow_start(node):
                 self.__logger.debug(f'Launching {info["name"]}')
 
-                TaskScheduler.__callbacks['pre_node'](self.__project, step, index)
+                MPManager.get_transient_settings().get(
+                    'TaskScheduler', 'pre_node',
+                    lambda project, step, index: None)(self.__project, step, index)
 
                 self.__record.set('status', NodeStatus.RUNNING, step=step, index=index)
                 self.__startTimes[node] = time.time()
@@ -414,6 +415,8 @@ class TaskScheduler:
 
                 # Start the process
                 info["running"] = True
+                info["parent_pipe"], pipe = multiprocessing.Pipe()
+                info["node"].set_queue(pipe, self.__log_queue)
                 info["proc"].start()
 
         return changed

siliconcompiler/schema/_metadata.py

@@ -1,2 +1,2 @@
 # Version number following semver standard.
-version = '0.52.1'
+version = '0.53.1'

siliconcompiler/schema/namedschema.py

@@ -44,12 +44,10 @@ class NamedSchema(BaseSchema):
         """
 
         try:
-            if self.__name is not None:
-                raise RuntimeError("Cannot call set_name more than once.")
+            if self._parent() is not self:
+                raise RuntimeError("Cannot call set_name after it has been inserted into schema.")
         except AttributeError:
             pass
-        if name is not None and "." in name:
-            raise ValueError("Named schema object cannot contains: .")
         self.__name = name
 
     def type(self) -> str:

siliconcompiler/schema_support/cmdlineschema.py

@@ -172,9 +172,6 @@ class CommandLineSchema(BaseSchema):
 
         # Iterate over all keys from an empty schema to add parser arguments
         for keypath in sorted(keyschema.allkeys()):
-            if keypath == ("option", "cfg"):  # TODO: remove this when cfg is removed from schema
-                continue
-
             param: Parameter = keyschema.get(*keypath, field=None)
 
             dest, switches = param.add_commandline_arguments(

siliconcompiler/schema_support/dependencyschema.py

@@ -210,12 +210,6 @@ class DependencySchema(BaseSchema):
 
         if name:
             if not self.has_dep(name):
-                if "." in name:
-                    name0, *name1 = name.split(".")
-                    subdep = self.get_dep(name0)
-                    if isinstance(subdep, DependencySchema):
-                        return subdep.get_dep(".".join(name1))
-                    raise KeyError(f"{name} does not contain dependency information")
                 raise KeyError(f"{name} is not an imported module")
 
             return self.__deps[name]

siliconcompiler/schema_support/record.py

@@ -15,6 +15,7 @@ from enum import Enum
 from siliconcompiler.schema import BaseSchema, LazyLoad
 from siliconcompiler.schema import EditableSchema, Parameter, PerNode, Scope
 from siliconcompiler.schema.utils import trim
+from siliconcompiler.utils.multiprocessing import MPManager
 
 from siliconcompiler import _metadata
 
@@ -138,9 +139,9 @@ class RecordSchema(BaseSchema):
             "region": str
             }
         '''
-        # TODO: add logic to figure out if we're running on a remote cluster and
-        # extract the region in a provider-specific way.
-        return {"region": "local"}
+
+        region = MPManager().get_settings().get("record", "region", default="local")
+        return {"region": region}
 
     @staticmethod
     def get_ip_information() -> Dict[str, Optional[str]]:

siliconcompiler/tool.py

@@ -11,7 +11,6 @@ import shlex
 import shutil
 import subprocess
 import sys
-import threading
 import time
 import yaml
 
@@ -35,7 +34,7 @@ from packaging.specifiers import SpecifierSet, InvalidSpecifier
 from typing import List, Dict, Tuple, Union, Optional, Set, TextIO, Type, TypeVar, TYPE_CHECKING
 from pathlib import Path
 
-from siliconcompiler.schema import BaseSchema, NamedSchema, Journal, DocsSchema, LazyLoad
+from siliconcompiler.schema import BaseSchema, NamedSchema, DocsSchema, LazyLoad
 from siliconcompiler.schema import EditableSchema, Parameter, PerNode, Scope
 from siliconcompiler.schema.parametertype import NodeType
 from siliconcompiler.schema.utils import trim
@@ -43,6 +42,7 @@ from siliconcompiler.schema.utils import trim
 from siliconcompiler import utils, NodeStatus, Flowgraph
 from siliconcompiler import sc_open
 from siliconcompiler.utils import paths
+from siliconcompiler.utils.multiprocessing import MPManager
 
 from siliconcompiler.schema_support.pathschema import PathSchema
 from siliconcompiler.schema_support.record import RecordTool, RecordSchema
@@ -390,7 +390,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
         if self.tool() in tools:
             self.logger.info(f"Missing tool can be installed via: \"sc-install {self.tool()}\"")
 
-    def get_exe_version(self) -> Optional[str]:
+    def get_exe_version(self, workdir: Optional[str] = None) -> Optional[str]:
         """
         Gets the version of the task's executable by running it with a version switch.
 
@@ -398,6 +398,10 @@ class Task(NamedSchema, PathSchema, DocsSchema):
             TaskExecutableNotFound: If the executable is not found.
             NotImplementedError: If the `parse_version` method is not implemented.
 
+        Args:
+            workdir (str): The working directory to use for the version check. If None,
+                the current working directory is used.
+
         Returns:
             str: The parsed version string.
         """
@@ -422,7 +426,8 @@ class Task(NamedSchema, PathSchema, DocsSchema):
                               stdin=subprocess.DEVNULL,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.STDOUT,
-                              universal_newlines=True)
+                              universal_newlines=True,
+                              cwd=workdir)
 
         if proc.returncode != 0:
             self.logger.warning(f"Version check on '{exe_base}' ended with "
@@ -736,9 +741,7 @@ class Task(NamedSchema, PathSchema, DocsSchema):
             fout.write(template.render(manifest_dict='\n'.join(tcl_set_cmds),
                                        scroot=os.path.abspath(
                                            os.path.join(os.path.dirname(__file__))),
-                                       toolvars=self.get_tcl_variables(manifest),
-                                       record_access="get" in Journal.access(self).get_types(),
-                                       record_access_id="TODO"))
+                                       toolvars=self.get_tcl_variables(manifest)))
         else:
             for cmd in tcl_set_cmds:
                 fout.write(cmd + '\n')
@@ -2084,9 +2087,6 @@ class ShowTask(Task):
     Subclasses should implement `get_supported_show_extentions` to declare
     which file types they can handle.
     """
-    __TASKS_LOCK = threading.Lock()
-    __TASKS = {}
-
     def __init__(self):
         """Initializes a ShowTask, adding specific parameters for show tasks."""
         super().__init__()
@@ -2133,8 +2133,10 @@ class ShowTask(Task):
         if not cls.__check_task(task):
             raise TypeError(f"task must be a subclass of {cls.__name__}")
 
-        with cls.__TASKS_LOCK:
-            cls.__TASKS.setdefault(cls, set()).add(task)
+        MPManager.get_transient_settings().set(
+            cls.__name__,
+            f"{task.__module__}/{task.__name__}",
+            task)
 
     @classmethod
     def __populate_tasks(cls) -> None:
@@ -2146,6 +2148,9 @@ class ShowTask(Task):
         """
         cls.__check_task(None)
 
+        if MPManager.get_transient_settings().get_category(cls.__name__):
+            return  # Already populated
+
         def recurse(searchcls: Type["ShowTask"]):
             subclss = set()
             if not cls.__check_task(searchcls):
@@ -2160,20 +2165,25 @@ class ShowTask(Task):
         classes = recurse(cls)
 
         # Support non-SC defined tasks from plugins
-        for plugin in utils.get_plugins('showtask'):  # TODO rename
+        for plugin in utils.get_plugins('showtask'):
             plugin()
 
         if not classes:
             return
 
-        with ShowTask.__TASKS_LOCK:
-            ShowTask.__TASKS.setdefault(cls, set()).update(classes)
+        for c in classes:
+            cls.register_task(c)
 
     @classmethod
     def get_task(cls, ext: Optional[str]) -> Union[Optional["ShowTask"], Set[Type["ShowTask"]]]:
         """
         Retrieves a suitable show task instance for a given file extension.
 
+        The method first checks the user's settings file (~/.sc/settings.json)
+        under the 'showtask' category for a preferred tool. If no preference
+        is found or the preferred tool is not available, it falls back to
+        automatic discovery.
+
         Args:
             ext (str): The file extension to find a viewer for.
 
@@ -2182,24 +2192,45 @@ class ShowTask(Task):
             no suitable task is found.
         """
         cls.__check_task(None)
+        cls.__populate_tasks()
 
-        if cls not in ShowTask.__TASKS:
-            cls.__populate_tasks()
-
-        with ShowTask.__TASKS_LOCK:
-            if cls not in ShowTask.__TASKS:
-                return None
-            tasks = ShowTask.__TASKS[cls].copy()
-
-        # TODO: add user preference lookup (ext -> task)
+        settings = MPManager.get_transient_settings()
+        tasks = set(settings.get_category(cls.__name__).values())
+        if not tasks:
+            return None
 
         if ext is None:
             return tasks
 
-        for task in tasks:
+        # 1. Check User Settings for Preference
+        preference = MPManager.get_settings().get("showtask", ext)
+
+        if preference:
+            # Preference format: "tool" or "tool/task"
+            pref_parts = preference.split('/')
+            pref_tool = pref_parts[0]
+            pref_task = pref_parts[1] if len(pref_parts) > 1 else None
+
+            for task_cls in tasks:
+                try:
+                    task_inst = task_cls()
+                    # Check if this task matches the preference
+                    if task_inst.tool() == pref_tool:
+                        if pref_task and task_inst.task() != pref_task:
+                            continue
+
+                        # Verify the preferred tool actually supports the extension
+                        if ext in task_inst.get_supported_show_extentions():
+                            return task_inst
+                except NotImplementedError:
+                    continue
+
+        # 2. Fallback to Automatic Discovery
+        for task_cls in tasks:
             try:
-                if ext in task().get_supported_show_extentions():
-                    return task()
+                task_inst = task_cls()
+                if ext in task_inst.get_supported_show_extentions():
+                    return task_inst
             except NotImplementedError:
                 pass
 

siliconcompiler/tools/_common/tcl/sc_schema_access.tcl

@@ -1,11 +1,5 @@
-proc _sc_cfg_get_debug { args } {
-
-}
-
 # Shortcut to get values from configuration
 proc sc_cfg_get { args } {
-    _sc_cfg_get_debug $args
-
     # Refer to global sc_cfg dictionary
     global sc_cfg
 

siliconcompiler/tools/chisel/convert.py

@@ -4,6 +4,8 @@ import glob
 
 import os.path
 
+from typing import Optional, List, Union
+
 from siliconcompiler import sc_open
 
 from siliconcompiler import Task
@@ -18,6 +20,48 @@ class ConvertTask(Task):
         self.add_parameter("targetdir", "str", "Output target directory name",
                            defvalue="chisel-output")
 
+    def set_chisel_application(self, application: str,
+                               step: Optional[str] = None, index: Optional[str] = None) -> None:
+        """
+        Sets the application name of the chisel program.
+
+        Args:
+            application (str): The application name.
+            step (str, optional): The specific step to apply this configuration to.
+            index (str, optional): The specific index to apply this configuration to.
+        """
+        self.set("var", "application", application, step=step, index=index)
+
+    def add_chisel_argument(self, argument: Union[str, List[str]],
+                            step: Optional[str] = None, index: Optional[str] = None,
+                            clobber: bool = False) -> None:
+        """
+        Adds arguments for the chisel build.
+
+        Args:
+            argument (Union[str, List[str]]): The argument(s) to add.
+            step (str, optional): The specific step to apply this configuration to.
+            index (str, optional): The specific index to apply this configuration to.
+            clobber (bool, optional): If True, overwrites the existing list. Defaults to False.
+        """
+        if clobber:
+            self.set("var", "argument", argument, step=step, index=index)
+        else:
+            self.add("var", "argument", argument, step=step, index=index)
+
+    def set_chisel_targetdir(self, targetdir: str,
+                             step: Optional[str] = None,
+                             index: Optional[str] = None) -> None:
+        """
+        Sets the output target directory name.
+
+        Args:
+            targetdir (str): The target directory name.
+            step (str, optional): The specific step to apply this configuration to.
+            index (str, optional): The specific index to apply this configuration to.
+        """
+        self.set("var", "targetdir", targetdir, step=step, index=index)
+
     def tool(self):
         return "chisel"