siliconcompiler 0.33.2__py3-none-any.whl → 0.34.1__py3-none-any.whl

siliconcompiler/scheduler/send_messages.py

@@ -66,7 +66,7 @@ def send(chip, msg_type, step, index):
     msg = MIMEMultipart()
 
     if step and index:
-        subject = f'SiliconCompiler : {chip.design} | {jobname} | {step}{index} | {msg_type}'
+        subject = f'SiliconCompiler : {chip.design} | {jobname} | {step} | {index} | {msg_type}'
     else:
         subject = f'SiliconCompiler : {chip.design} | {jobname} | {msg_type}'
 
@@ -92,7 +92,7 @@ def send(chip, msg_type, step, index):
                     msg.attach(img_attach)
 
             runtime = RuntimeFlowgraph(
-                chip.schema.get("flowgraph", flow, field='schema'),
+                chip.get("flowgraph", flow, field='schema'),
                 from_steps=chip.get('option', 'from'),
                 to_steps=chip.get('option', 'to'),
                 prune_nodes=chip.get('option', 'prune'))
@@ -110,7 +110,7 @@ def send(chip, msg_type, step, index):
                 metric_keys=metrics_to_show)
         else:
             # Attach logs
-            for log in (f'sc_{step}{index}.log', f'{step}.log'):
+            for log in (f'sc_{step}_{index}.log', f'{step}.log'):
                 log_file = f'{chip.getworkdir(step=step, index=index)}/{log}'
                 if os.path.exists(log_file):
                     with sc_open(log_file) as f:

siliconcompiler/scheduler/slurm.py

@@ -5,171 +5,157 @@ import stat
 import uuid
 import json
 import shutil
-from siliconcompiler import utils, SiliconCompilerError
-from siliconcompiler.package import get_cache_path
-from siliconcompiler.flowgraph import RuntimeFlowgraph
-
-# Full list of Slurm states, split into 'active' and 'inactive' categories.
-# Many of these do not apply to a minimal configuration, but we'll track them all.
-# https://slurm.schedmd.com/squeue.html#SECTION_JOB-STATE-CODES
-SLURM_ACTIVE_STATES = [
-    'RUNNING',
-    'PENDING',
-    'CONFIGURING',
-    'COMPLETING',
-    'SIGNALING',
-    'STAGE_OUT',
-    'RESIZING',
-    'REQUEUED',
-]
-SLURM_INACTIVE_STATES = [
-    'BOOT_FAIL',
-    'CANCELLED',
-    'COMPLETED',
-    'DEADLINE',
-    'FAILED',
-    'NODE_FAIL',
-    'OUT_OF_MEMORY',
-    'PREEMPTED',
-    'RESV_DEL_HOLD',
-    'REQUEUE_FED',
-    'REQUEUE_HOLD',
-    'REVOKED',
-    'SPECIAL_EXIT',
-    'STOPPED',
-    'SUSPENDED',
-    'TIMEOUT',
-]
-
-
-###########################################################################
-def get_configuration_directory(chip):
-    '''
-    Helper function to get the configuration directory for the scheduler
-    '''
-
-    return f'{chip.getworkdir()}/configs'
-
-
-def init(chip):
-    if os.path.exists(chip._getcollectdir()):
-        # nothing to do
-        return
-
-    collect = False
-    flow = chip.get('option', 'flow')
-    entry_nodes = chip.schema.get("flowgraph", flow, field="schema").get_entry_nodes()
-
-    runtime = RuntimeFlowgraph(
-        chip.schema.get("flowgraph", flow, field='schema'),
-        from_steps=chip.get('option', 'from'),
-        to_steps=chip.get('option', 'to'),
-        prune_nodes=chip.get('option', 'prune'))
-
-    for (step, index) in runtime.get_nodes():
-        if (step, index) in entry_nodes:
-            collect = True
-
-    if collect:
-        chip.collect()
-
-
-###########################################################################
-def _defernode(chip, step, index, replay):
-    '''
-    Helper method to run an individual step on a slurm cluster.
-
-    Blocks until the compute node
-    finishes processing this step, and it sets the active/error bits.
-    '''
-
-    # Determine which HPC job scheduler being used.
-    scheduler_type = chip.get('option', 'scheduler', 'name', step=step, index=index)
-
-    if scheduler_type != 'slurm':
-        raise ValueError(f'{scheduler_type} is not a supported scheduler')
-
-    if not check_slurm():
-        raise SiliconCompilerError('slurm is not available or installed on this machine', chip=chip)
-
-    # Determine which cluster parititon to use. (Default value can be overridden on per-step basis)
-    partition = chip.get('option', 'scheduler', 'queue', step=step, index=index)
-    if not partition:
-        partition = _get_slurm_partition()
-
-    # Get the temporary UID associated with this job run.
-    job_hash = chip.get('record', 'remoteid')
-    if not job_hash:
-        # Generate a new uuid since it was not set
-        job_hash = uuid.uuid4().hex
-
-    job_name = f'{job_hash}_{step}{index}'
-
-    # Write out the current schema for the compute node to pick up.
-    cfg_dir = get_configuration_directory(chip)
-    cfg_file = f'{cfg_dir}/{step}{index}.json'
-    log_file = f'{cfg_dir}/{step}{index}.log'
-    script_file = f'{cfg_dir}/{step}{index}.sh'
-    os.makedirs(cfg_dir, exist_ok=True)
-
-    chip.set('option', 'scheduler', 'name', None, step=step, index=index)
-    chip.write_manifest(cfg_file)
-
-    # Allow user-defined compute node execution script if it already exists on the filesystem.
-    # Otherwise, create a minimal script to run the task using the SiliconCompiler CLI.
-    if not os.path.isfile(script_file):
-        with open(script_file, 'w') as sf:
-            sf.write(utils.get_file_template('slurm/run.sh').render(
-                cfg_file=shlex.quote(cfg_file),
-                build_dir=shlex.quote(chip.get("option", "builddir")),
-                step=shlex.quote(step),
-                index=shlex.quote(index),
-                cachedir=shlex.quote(get_cache_path(chip))
-            ))
-
-    # This is Python for: `chmod +x [script_path]`
-    os.chmod(script_file,
-             os.stat(script_file).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
-
-    schedule_cmd = ['srun',
-                    '--exclusive',
-                    '--partition', partition,
-                    '--chdir', chip.cwd,
-                    '--job-name', job_name,
-                    '--output', log_file]
-
-    # Only delay the starting time if the 'defer' Schema option is specified.
-    defer_time = chip.get('option', 'scheduler', 'defer', step=step, index=index)
-    if defer_time:
-        schedule_cmd.extend(['--begin', defer_time])
-
-    schedule_cmd.append(script_file)
 
-    # Run the 'srun' command, and track its output.
-    # TODO: output should be fed to log, and stdout if quiet = False
-    step_result = subprocess.Popen(schedule_cmd,
-                                   stdout=subprocess.PIPE,
-                                   stderr=subprocess.STDOUT)
+import os.path
 
-    # Wait for the subprocess call to complete. It should already be done,
-    # as it has closed its output stream. But if we don't call '.wait()',
-    # the '.returncode' value will not be set correctly.
-    step_result.wait()
-
-
-def _get_slurm_partition():
-    partitions = subprocess.run(['sinfo', '--json'],
-                                stdout=subprocess.PIPE,
-                                stderr=subprocess.STDOUT)
-
-    if partitions.returncode != 0:
-        raise RuntimeError('Unable to determine partitions in slurm')
-
-    sinfo = json.loads(partitions.stdout.decode())
+from siliconcompiler import utils
+from siliconcompiler.package import RemoteResolver
+from siliconcompiler.flowgraph import RuntimeFlowgraph
+from siliconcompiler.scheduler.schedulernode import SchedulerNode
 
-    # Return the first listed partition
-    return sinfo['nodes'][0]['partitions'][0]
 
+class SlurmSchedulerNode(SchedulerNode):
+    def __init__(self, chip, step, index, replay=False):
+        super().__init__(chip, step, index, replay=replay)
 
-def check_slurm():
-    return shutil.which('sinfo') is not None
+        # Get the temporary UID associated with this job run.
+        self.__job_hash = chip.get('record', 'remoteid')
+        if not self.__job_hash:
+            # Generate a new uuid since it was not set
+            self.__job_hash = uuid.uuid4().hex
+
+    @property
+    def jobhash(self):
+        return self.__job_hash
+
+    @staticmethod
+    def init(chip):
+        if os.path.exists(chip._getcollectdir()):
+            # nothing to do
+            return
+
+        collect = False
+        flow = chip.get('option', 'flow')
+        entry_nodes = chip.get("flowgraph", flow, field="schema").get_entry_nodes()
+
+        runtime = RuntimeFlowgraph(
+            chip.get("flowgraph", flow, field='schema'),
+            from_steps=chip.get('option', 'from'),
+            to_steps=chip.get('option', 'to'),
+            prune_nodes=chip.get('option', 'prune'))
+
+        for (step, index) in runtime.get_nodes():
+            if (step, index) in entry_nodes:
+                collect = True
+
+        if collect:
+            chip.collect()
+
+    @property
+    def is_local(self):
+        return False
+
+    @staticmethod
+    def get_configuration_directory(chip):
+        '''
+        Helper function to get the configuration directory for the scheduler
+        '''
+
+        return os.path.join(chip.getworkdir(), 'sc_configs')
+
+    @staticmethod
+    def get_job_name(jobhash, step, index):
+        return f'{jobhash}_{step}_{index}'
+
+    @staticmethod
+    def get_runtime_file_name(jobhash, step, index, ext):
+        return f"{SlurmSchedulerNode.get_job_name(jobhash, step, index)}.{ext}"
+
+    @staticmethod
+    def get_slurm_partition():
+        partitions = subprocess.run(['sinfo', '--json'],
+                                    stdout=subprocess.PIPE,
+                                    stderr=subprocess.STDOUT)
+
+        if partitions.returncode != 0:
+            raise RuntimeError('Unable to determine partitions in slurm')
+
+        sinfo = json.loads(partitions.stdout.decode())
+
+        # Return the first listed partition
+        return sinfo['nodes'][0]['partitions'][0]
+
+    def run(self):
+        '''
+        Helper method to run an individual step on a slurm cluster.
+
+        Blocks until the compute node
+        finishes processing this step, and it sets the active/error bits.
+        '''
+
+        self._init_run_logger()
+
+        if shutil.which('sinfo') is None:
+            raise RuntimeError('slurm is not available or installed on this machine')
+
+        # Determine which cluster parititon to use.
+        partition = self.chip.get('option', 'scheduler', 'queue', step=self.step, index=self.index)
+        if not partition:
+            partition = SlurmSchedulerNode.get_slurm_partition()
+
+        # Write out the current schema for the compute node to pick up.
+        cfg_dir = SlurmSchedulerNode.get_configuration_directory(self.chip)
+        os.makedirs(cfg_dir, exist_ok=True)
+
+        cfg_file = os.path.join(cfg_dir, SlurmSchedulerNode.get_runtime_file_name(
+            self.__job_hash, self.step, self.index, "pkg.json"))
+        log_file = os.path.join(cfg_dir, SlurmSchedulerNode.get_runtime_file_name(
+            self.__job_hash, self.step, self.index, "log"))
+        script_file = os.path.join(cfg_dir, SlurmSchedulerNode.get_runtime_file_name(
+            self.__job_hash, self.step, self.index, "sh"))
+
+        # Remove scheduler as this is now a local run
+        self.chip.set('option', 'scheduler', 'name', None, step=self.step, index=self.index)
+        self.chip.write_manifest(cfg_file)
+
+        # Allow user-defined compute node execution script if it already exists on the filesystem.
+        # Otherwise, create a minimal script to run the task using the SiliconCompiler CLI.
+        if not os.path.isfile(script_file):
+            with open(script_file, 'w') as sf:
+                sf.write(utils.get_file_template('slurm/run.sh').render(
+                    cfg_file=shlex.quote(cfg_file),
+                    build_dir=shlex.quote(self.chip.get("option", "builddir")),
+                    step=shlex.quote(self.step),
+                    index=shlex.quote(self.index),
+                    cachedir=shlex.quote(str(RemoteResolver.determine_cache_dir(self.chip)))
+                ))
+
+        # This is Python for: `chmod +x [script_path]`
+        os.chmod(script_file,
+                 os.stat(script_file).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+
+        schedule_cmd = ['srun',
+                        '--exclusive',
+                        '--partition', partition,
+                        '--chdir', self.chip.cwd,
+                        '--job-name', SlurmSchedulerNode.get_job_name(self.__job_hash,
+                                                                      self.step, self.index),
+                        '--output', log_file]
+
+        # Only delay the starting time if the 'defer' Schema option is specified.
+        defer_time = self.chip.get('option', 'scheduler', 'defer', step=self.step, index=self.index)
+        if defer_time:
+            schedule_cmd.extend(['--begin', defer_time])
+
+        schedule_cmd.append(script_file)
+
+        # Run the 'srun' command, and track its output.
+        # TODO: output should be fed to log, and stdout if quiet = False
+        step_result = subprocess.Popen(schedule_cmd,
+                                       stdout=subprocess.PIPE,
+                                       stderr=subprocess.STDOUT)
+
+        # Wait for the subprocess call to complete. It should already be done,
+        # as it has closed its output stream. But if we don't call '.wait()',
+        # the '.returncode' value will not be set correctly.
+        step_result.wait()

siliconcompiler/scheduler/taskscheduler.py

@@ -11,11 +11,8 @@ from siliconcompiler import SiliconCompilerError
 from siliconcompiler import utils
 from siliconcompiler.flowgraph import RuntimeFlowgraph
 
-from siliconcompiler.schema import JournalingSchema
+from siliconcompiler.schema import Journal
 
-from siliconcompiler.scheduler import slurm
-from siliconcompiler.scheduler import docker_runner
-from siliconcompiler.tools._common import get_tool_task
 from siliconcompiler.utils.logging import SCBlankLoggerFormatter
 
 
@@ -33,9 +30,10 @@ class TaskScheduler:
             raise ValueError(f"{hook} is not a valid callback")
         TaskScheduler.__callbacks[hook] = func
 
-    def __init__(self, chip):
+    def __init__(self, chip, tasks):
         self.__chip = chip
         self.__logger = self.__chip.logger
+        self.__logger_console_handler = self.__chip._logger_console
         self.__schema = self.__chip.schema
         self.__flow = self.__schema.get("flowgraph", self.__chip.get('option', 'flow'),
                                         field="schema")
@@ -50,17 +48,21 @@ class TaskScheduler:
         # clip max parallel jobs to 1 <= jobs <= max_cores
         self.__max_parallel_run = max(1, min(self.__max_parallel_run, self.__max_cores))
 
+        self.__runtime_flow = RuntimeFlowgraph(
+            self.__flow,
+            from_steps=self.__chip.get('option', 'from'),
+            to_steps=self.__chip.get('option', 'to'),
+            prune_nodes=self.__chip.get('option', 'prune'))
+
         self.__log_queue = multiprocessing.Queue(-1)
 
         self.__nodes = {}
         self.__startTimes = {}
         self.__dwellTime = 0.1
 
-        self.__create_nodes()
-
-    def __create_nodes(self):
-        from siliconcompiler.scheduler import _executenode, _runtask
+        self.__create_nodes(tasks)
 
+    def __create_nodes(self, tasks):
         runtime = RuntimeFlowgraph(
             self.__flow,
             from_steps=set([step for step, _ in self.__flow.get_entry_nodes()]),
@@ -68,63 +70,34 @@ class TaskScheduler:
 
         init_funcs = set()
 
-        runtime_flow = RuntimeFlowgraph(
-            self.__flow,
-            from_steps=self.__chip.get('option', 'from'),
-            to_steps=self.__chip.get('option', 'to'),
-            prune_nodes=self.__chip.get('option', 'prune'))
-
-        for step, index in runtime_flow.get_nodes():
+        for step, index in self.__runtime_flow.get_nodes():
             if self.__record.get('status', step=step, index=index) != NodeStatus.PENDING:
                 continue
 
-            tool_name, task_name = get_tool_task(self.__chip, step, index)
-            threads = self.__chip.get('tool', tool_name, 'task', task_name, 'threads',
-                                      step=step, index=index)
+            with tasks[(step, index)].runtime():
+                threads = tasks[(step, index)].threads
             if not threads:
                 threads = self.__max_threads
             threads = max(1, min(threads, self.__max_threads))
 
             task = {
-                "name": f"{step}{index}",
+                "name": f"{step}/{index}",
                 "inputs": runtime.get_node_inputs(step, index, record=self.__record),
                 "proc": None,
-                "child_pipe": None,
                 "parent_pipe": None,
-                "local": False,
-                "tool": tool_name,
-                "task": task_name,
                 "threads": threads,
                 "running": False,
                 "manifest": os.path.join(self.__chip.getworkdir(step=step, index=index),
                                          'outputs',
-                                         f'{self.__chip.design}.pkg.json')
+                                         f'{self.__chip.design}.pkg.json'),
+                "node": tasks[(step, index)]
             }
 
-            exec_func = _executenode
-
-            node_scheduler = self.__chip.get('option', 'scheduler', 'name', step=step, index=index)
-            if node_scheduler == 'slurm':
-                # Defer job to compute node
-                # If the job is configured to run on a cluster, collect the schema
-                # and send it to a compute node for deferred execution.
-                init_funcs.add(slurm.init)
-                exec_func = slurm._defernode
-            elif node_scheduler == 'docker':
-                # Run job in docker
-                init_funcs.add(docker_runner.init)
-                exec_func = docker_runner.run
-                task["local"] = True
-            else:
-                task["local"] = True
-
-            task["parent_pipe"], task["child_pipe"] = multiprocessing.Pipe()
-            task["proc"] = multiprocessing.Process(
-                target=_runtask,
-                args=(self.__chip, self.__flow.name(), step, index, exec_func),
-                kwargs={"pipe": task["child_pipe"],
-                        "queue": self.__log_queue})
+            task["parent_pipe"], pipe = multiprocessing.Pipe()
+            task["node"].set_queue(pipe, self.__log_queue)
 
+            task["proc"] = multiprocessing.Process(target=task["node"].run)
+            init_funcs.add(task["node"].init)
             self.__nodes[(step, index)] = task
 
         # Call preprocessing for schedulers
@@ -136,9 +109,9 @@ class TaskScheduler:
         multiprocessing.freeze_support()
 
         # Handle logs across threads
-        log_listener = QueueListener(self.__log_queue, self.__logger._console)
-        console_format = self.__logger._console.formatter
-        self.__logger._console.setFormatter(SCBlankLoggerFormatter())
+        log_listener = QueueListener(self.__log_queue, self.__logger_console_handler)
+        console_format = self.__logger_console_handler.formatter
+        self.__logger_console_handler.setFormatter(SCBlankLoggerFormatter())
         log_listener.start()
 
         # Update dashboard before run begins
@@ -158,7 +131,7 @@ class TaskScheduler:
 
         # Cleanup logger
         log_listener.stop()
-        self.__logger._console.setFormatter(console_format)
+        self.__logger_console_handler.setFormatter(console_format)
 
     def __run_loop(self):
         self.__startTimes = {None: time.time()}
@@ -217,13 +190,17 @@ class TaskScheduler:
                 self.__logger.debug(f'{info["name"]} is complete merging: {manifest}')
 
                 if os.path.exists(manifest):
-                    JournalingSchema(self.__schema).read_journal(manifest)
+                    Journal.replay_file(self.__schema, manifest)
+                    # TODO: once tool is fixed this can go away
+                    self.__schema.unset("arg", "step")
+                    self.__schema.unset("arg", "index")
 
                 if info["parent_pipe"] and info["parent_pipe"].poll(1):
                     try:
                         packages = info["parent_pipe"].recv()
                         if isinstance(packages, dict):
-                            self.__chip._packages.update(packages)
+                            for package, path in packages.items():
+                                self.__chip.get("package", field="schema")._set_cache(package, path)
                     except:  # noqa E722
                         pass
 
@@ -249,7 +226,7 @@ class TaskScheduler:
     def __allow_start(self, node):
         info = self.__nodes[node]
 
-        if not info["local"]:
+        if not info["node"].is_local:
             # using a different scheduler, so allow
             return True
 
@@ -286,7 +263,7 @@ class TaskScheduler:
                 if not NodeStatus.is_done(in_status):
                     ready = False
                     break
-                if NodeStatus.is_error(in_status) and info["tool"] != "builtin":
+                if NodeStatus.is_error(in_status) and not info["node"].is_builtin:
                     # Fail if any dependency failed for non-builtin task
                     self.__record.set("status", NodeStatus.ERROR, step=step, index=index)
 
@@ -295,7 +272,7 @@ class TaskScheduler:
                 any_success = any([status == NodeStatus.SUCCESS for status in inputs])
             else:
                 any_success = True
-            if ready and info["tool"] == "builtin" and not any_success:
+            if ready and info["node"].is_builtin and not any_success:
                 self.__record.set("status", NodeStatus.ERROR, step=step, index=index)
 
             if self.__record.get('status', step=step, index=index) == NodeStatus.ERROR:
@@ -318,3 +295,14 @@ class TaskScheduler:
                 info["proc"].start()
 
         return changed
+
+    def check(self):
+        exit_steps = set([step for step, _ in self.__runtime_flow.get_exit_nodes()])
+        completed_steps = set([step for step, _ in
+                               self.__runtime_flow.get_completed_nodes(record=self.__record)])
+
+        unreached = set(exit_steps).difference(completed_steps)
+
+        if unreached:
+            raise RuntimeError(
+                f'These final steps could not be reached: {", ".join(sorted(unreached))}')

siliconcompiler/schema/__init__.py

@@ -1,9 +1,9 @@
 from .parameter import Parameter, Scope, PerNode
+from .journal import Journal
 from .safeschema import SafeSchema
 from .editableschema import EditableSchema
 from .baseschema import BaseSchema
 from .cmdlineschema import CommandLineSchema
-from .journalingschema import JournalingSchema
 from .namedschema import NamedSchema
 from .packageschema import PackageSchema
 
@@ -15,10 +15,10 @@ __all__ = [
     "SafeSchema",
     "EditableSchema",
     "CommandLineSchema",
-    "JournalingSchema",
     "NamedSchema",
     "PackageSchema",
     "Parameter",
     "Scope",
-    "PerNode"
+    "PerNode",
+    "Journal"
 ]