siliconcompiler 0.35.2__py3-none-any.whl → 0.35.4__py3-none-any.whl

siliconcompiler/scheduler/slurm.py

@@ -5,15 +5,18 @@ import shutil
 import stat
 import subprocess
 import uuid
+import time
 
 import os.path
 
-from siliconcompiler import utils
-from siliconcompiler.utils.curation import collect
-from siliconcompiler.utils.paths import collectiondir, jobdir
+from typing import List, Union, Final
+
+from siliconcompiler import utils, sc_open
+from siliconcompiler.utils.paths import jobdir
 from siliconcompiler.package import RemoteResolver
-from siliconcompiler.flowgraph import RuntimeFlowgraph
 from siliconcompiler.scheduler import SchedulerNode
+from siliconcompiler.utils.logging import SCBlankLoggerFormatter
+from siliconcompiler.utils.multiprocessing import MPManager
 
 
 class SlurmSchedulerNode(SchedulerNode):
@@ -24,6 +27,10 @@ class SlurmSchedulerNode(SchedulerNode):
     It prepares a run script, a manifest, and uses the 'srun' command
     to execute the step on a compute node.
     """
+    __OPTIONS: Final[str] = "scheduler-slurm"
+
+    _MAX_FS_DELAY = 2
+    _FS_DWELL = 0.1
 
     def __init__(self, project, step, index, replay=False):
         """Initializes a SlurmSchedulerNode.
@@ -52,34 +59,31 @@ class SlurmSchedulerNode(SchedulerNode):
         """
         A static pre-processing hook for the Slurm scheduler.
 
-        This method checks if the compilation flow starts from an entry node.
-        If so, it calls :meth:`.collect()` to gather all necessary source files
-        into a central location before any remote jobs are submitted. This
-        ensures that compute nodes have access to all required source files.
+        This method ensures that the Slurm environment is available and loads
+        any existing user configuration for the scheduler.
 
         Args:
             project (Project): The project object to perform pre-processing on.
         """
-        if os.path.exists(collectiondir(project)):
-            # nothing to do
-            return
-
-        do_collect = False
-        flow = project.get('option', 'flow')
-        entry_nodes = project.get("flowgraph", flow, field="schema").get_entry_nodes()
+        SlurmSchedulerNode.assert_slurm()
 
-        runtime = RuntimeFlowgraph(
-            project.get("flowgraph", flow, field='schema'),
-            from_steps=project.get('option', 'from'),
-            to_steps=project.get('option', 'to'),
-            prune_nodes=project.get('option', 'prune'))
+    @staticmethod
+    def _set_user_config(tag: str, value: Union[List[str], str]) -> None:
+        """
+        Sets a specific value in the user configuration map.
 
-        for (step, index) in runtime.get_nodes():
-            if (step, index) in entry_nodes:
-                do_collect = True
+        Args:
+            tag (str): The configuration key to update.
+            value (Union[List[str], str]): The value to assign to the key.
+        """
+        MPManager.get_settings().set(SlurmSchedulerNode.__OPTIONS, tag, value)
 
-        if do_collect:
-            collect(project)
+    @staticmethod
+    def _write_user_config() -> None:
+        """
+        Writes the current system configuration to the user configuration file.
+        """
+        MPManager.get_settings().save()
 
     @property
     def is_local(self):
@@ -150,6 +154,49 @@ class SlurmSchedulerNode(SchedulerNode):
         # Return the first listed partition
         return sinfo['nodes'][0]['partitions'][0]
 
+    @staticmethod
+    def assert_slurm() -> None:
+        """
+        Check if slurm is installed and throw error when not installed.
+        """
+        if shutil.which('sinfo') is None:
+            raise RuntimeError('slurm is not available or installed on this machine')
+
+    def mark_copy(self) -> bool:
+        sharedprefix: List[str] = MPManager.get_settings().get(
+            SlurmSchedulerNode.__OPTIONS, "sharedpaths", default=[])
+
+        if "/" in sharedprefix:
+            # Entire filesystem is shared so no need to check
+            return False
+
+        do_collect = False
+        for key in self.get_required_path_keys():
+            mark_copy = True
+            if sharedprefix:
+                mark_copy = False
+
+                check_step, check_index = self.step, self.index
+                if self.project.get(*key, field='pernode').is_never():
+                    check_step, check_index = None, None
+
+                paths = self.project.find_files(*key, missing_ok=True,
+                                                step=check_step, index=check_index)
+                if not isinstance(paths, list):
+                    paths = [paths]
+                paths = [str(path) for path in paths if path]
+
+                for path in paths:
+                    if not any([path.startswith(shared) for shared in sharedprefix]):
+                        # File exists outside shared paths and needs to be copied
+                        mark_copy = True
+                        break
+
+            if mark_copy:
+                self.project.set(*key, True, field='copy')
+                do_collect = True
+        return do_collect
+
     def run(self):
         """
         Runs the node's task as a job on a Slurm cluster.
@@ -161,9 +208,6 @@ class SlurmSchedulerNode(SchedulerNode):
 
         self._init_run_logger()
 
-        if shutil.which('sinfo') is None:
-            raise RuntimeError('slurm is not available or installed on this machine')
-
         # Determine which cluster parititon to use.
         partition = self.project.get('option', 'scheduler', 'queue',
                                      step=self.step, index=self.index)
@@ -191,7 +235,7 @@ class SlurmSchedulerNode(SchedulerNode):
             with open(script_file, 'w') as sf:
                 sf.write(utils.get_file_template('slurm/run.sh').render(
                     cfg_file=shlex.quote(cfg_file),
-                    build_dir=shlex.quote(self.project.get("option", "builddir")),
+                    build_dir=shlex.quote(self.project.option.get_builddir()),
                     step=shlex.quote(self.step),
                     index=shlex.quote(self.index),
                     cachedir=shlex.quote(str(RemoteResolver.determine_cache_dir(self.project)))
@@ -217,9 +261,12 @@ class SlurmSchedulerNode(SchedulerNode):
 
         schedule_cmd.append(script_file)
 
+        self.logger.debug(f"Executing slurm command: {shlex.join(schedule_cmd)}")
+
         # Run the 'srun' command, and track its output.
         # TODO: output should be fed to log, and stdout if quiet = False
         step_result = subprocess.Popen(schedule_cmd,
+                                       stdin=subprocess.DEVNULL,
                                        stdout=subprocess.PIPE,
                                        stderr=subprocess.STDOUT)
 
@@ -227,3 +274,40 @@ class SlurmSchedulerNode(SchedulerNode):
         # as it has closed its output stream. But if we don't call '.wait()',
         # the '.returncode' value will not be set correctly.
         step_result.wait()
+
+        # Attempt to list dir to trigger network FS to update
+        try:
+            os.listdir(os.path.dirname(log_file))
+        except:  # noqa E722
+            pass
+
+        # Print the log to logger
+        if os.path.exists(log_file):
+            org_formatter = self.project._logger_console.formatter
+            try:
+                self.project._logger_console.setFormatter(SCBlankLoggerFormatter())
+                with sc_open(log_file) as log:
+                    for line in log.readlines():
+                        self.logger.info(line.rstrip())
+            finally:
+                self.project._logger_console.setFormatter(org_formatter)
+
+        if step_result.returncode != 0:
+            self.logger.error(f"Slurm exited with a non-zero code ({step_result.returncode}).")
+            if os.path.exists(log_file):
+                self.logger.error(f"Node log file: {log_file}")
+            self.halt()
+
+        # Wait for manifest to propagate through network filesystem
+        start = time.time()
+        elapsed = 0
+        manifest_path = self.get_manifest()
+        while not os.path.exists(manifest_path) and elapsed <= SlurmSchedulerNode._MAX_FS_DELAY:
+            os.listdir(os.path.dirname(manifest_path))
+            elapsed = time.time() - start
+            time.sleep(SlurmSchedulerNode._FS_DWELL)
+        if not os.path.exists(manifest_path):
+            self.logger.error(f"Manifest was not created on time: {manifest_path}")
+
+    def check_required_paths(self) -> bool:
+        return True

siliconcompiler/scheduler/taskscheduler.py

@@ -117,8 +117,6 @@ class TaskScheduler:
             from_steps=set([step for step, _ in self.__flow.get_entry_nodes()]),
             prune_nodes=self.__project.option.get_prune())
 
-        init_funcs = set()
-
         for step, index in self.__runtime_flow.get_nodes():
             if self.__record.get('status', step=step, index=index) != NodeStatus.PENDING:
                 continue
@@ -145,7 +143,6 @@ class TaskScheduler:
             task["node"].set_queue(pipe, self.__log_queue)
 
             task["proc"] = multiprocessing.Process(target=task["node"].run)
-            init_funcs.add(task["node"].init)
             self.__nodes[(step, index)] = task
 
         # Create ordered list of nodes
@@ -155,10 +152,6 @@ class TaskScheduler:
                 if node in self.__nodes:
                     self.__ordered_nodes.append(node)
 
-        # Call preprocessing for schedulers
-        for init_func in init_funcs:
-            init_func(self.__project)
-
     def run(self, job_log_handler: logging.Handler) -> None:
         """
         The main entry point for the task scheduling loop.

siliconcompiler/schema/__init__.py

@@ -4,7 +4,7 @@ from .parameter import Parameter, Scope, PerNode
 from .journal import Journal
 from .safeschema import SafeSchema
 from .editableschema import EditableSchema
-from .baseschema import BaseSchema
+from .baseschema import BaseSchema, LazyLoad
 from .namedschema import NamedSchema
 from .docschema import DocsSchema
 
@@ -17,7 +17,8 @@ __all__ = [
     "Scope",
     "PerNode",
     "Journal",
-    "DocsSchema"
+    "DocsSchema",
+    "LazyLoad"
 ]
 
 SCHEMA_VERSION = __version__

siliconcompiler/schema/_metadata.py

@@ -1,2 +1,2 @@
 # Version number following semver standard.
-version = '0.52.0'
+version = '0.52.1'