siliconcompiler 0.33.0__py3-none-any.whl → 0.33.2__py3-none-any.whl

siliconcompiler/remote/server.py

@@ -1,6 +1,7 @@
 # Copyright 2020 Silicon Compiler Authors. All Rights Reserved.
 
 from aiohttp import web
+import copy
 import threading
 import json
 import logging as log
@@ -12,7 +13,8 @@ import sys
 import fastjsonschema
 from pathlib import Path
 from fastjsonschema import JsonSchemaException
-import io
+
+import os.path
 
 from siliconcompiler import Chip, Schema
 from siliconcompiler.schema import utils as schema_utils
@@ -20,8 +22,10 @@ from siliconcompiler._metadata import version as sc_version
 from siliconcompiler.schema import SCHEMA_VERSION as sc_schema_version
 from siliconcompiler.remote.schema import ServerSchema
 from siliconcompiler.remote import banner, JobStatus
-from siliconcompiler.scheduler.slurm import get_configuration_directory
-from siliconcompiler.utils.flowgraph import nodes_to_execute
+from siliconcompiler import NodeStatus as SCNodeStatus
+from siliconcompiler.remote import NodeStatus
+from siliconcompiler.flowgraph import RuntimeFlowgraph
+from siliconcompiler.scheduler.taskscheduler import TaskScheduler
 
 
 # Compile validation code for API request bodies.
@@ -81,7 +85,61 @@ class Server:
         self.schema = ServerSchema()
 
         # Set up a dictionary to track running jobs.
+        self.sc_jobs_lock = threading.Lock()
         self.sc_jobs = {}
+        self.sc_chip_lookup = {}
+
+        # Register callbacks
+        TaskScheduler.register_callback("pre_run", self.__run_start)
+        TaskScheduler.register_callback("pre_node", self.__node_start)
+        TaskScheduler.register_callback("post_node", self.__node_end)
+
+    def __run_start(self, chip):
+        flow = chip.get("option", "flow")
+        nodes = chip.schema.get("flowgraph", flow, field="schema").get_nodes()
+
+        with self.sc_jobs_lock:
+            job_hash = self.sc_chip_lookup[chip]["jobhash"]
+
+        start_tar = os.path.join(self.nfs_mount, job_hash, f'{job_hash}_None.tar.gz')
+        start_status = NodeStatus.SUCCESS
+        with tarfile.open(start_tar, "w:gz") as tf:
+            start_manifest = os.path.join(chip.getworkdir(), f"{chip.design}.pkg.json")
+            tf.add(start_manifest, arcname=os.path.relpath(start_manifest, self.nfs_mount))
+
+        with self.sc_jobs_lock:
+            job_name = self.sc_chip_lookup[chip]["name"]
+
+            self.sc_jobs[job_name][None]["status"] = start_status
+
+            for step, index in nodes:
+                name = f"{step}{index}"
+                if name not in self.sc_jobs[job_name]:
+                    continue
+                self.sc_jobs[job_name][name]["status"] = \
+                    chip.get('record', 'status', step=step, index=index)
+
+    def __node_start(self, chip, step, index):
+        with self.sc_jobs_lock:
+            job_name = self.sc_chip_lookup[chip]["name"]
+            self.sc_jobs[job_name][f"{step}{index}"]["status"] = NodeStatus.RUNNING
+
+    def __node_end(self, chip, step, index):
+        with self.sc_jobs_lock:
+            job_hash = self.sc_chip_lookup[chip]["jobhash"]
+            job_name = self.sc_chip_lookup[chip]["name"]
+
+        chip = copy.deepcopy(chip)
+        chip.cwd = os.path.join(chip.get('option', 'builddir'), '..')
+        with tarfile.open(os.path.join(self.nfs_mount,
+                                       job_hash,
+                                       f'{job_hash}_{step}{index}.tar.gz'),
+                          mode='w:gz') as tf:
+            chip._archive_node(tf, step=step, index=index, include="*")
+
+        with self.sc_jobs_lock:
+            self.sc_jobs[job_name][f"{step}{index}"]["status"] = \
+                chip.get('record', 'status', step=step, index=index)
 
     def run(self):
         if not os.path.exists(self.nfs_mount):
@@ -226,9 +284,6 @@ class Server:
         # Remove 'remote' JSON config value to run locally on compute node.
         chip.set('option', 'remote', False)
 
-        # Write JSON config to shared compute storage.
-        os.makedirs(os.path.join(job_root, 'configs'), exist_ok=True)
-
         # Run the job with the configured clustering option. (Non-blocking)
         job_proc = threading.Thread(target=self.remote_sc,
                                     args=[
@@ -258,31 +313,13 @@ class Server:
         job_hash = job_params['job_hash']
         node = job_params['node'] if 'node' in job_params else None
 
-        resp = web.StreamResponse(
-            status=200,
-            reason='OK',
-            headers={
-                'Content-Type': 'application/x-tar',
-                'Content-Disposition': f'attachment; filename="{job_hash}_{node}.tar.gz"'
-            },
-        )
-        await resp.prepare(request)
-
         zipfn = os.path.join(self.nfs_mount, job_hash, f'{job_hash}_{node}.tar.gz')
-        if not node:
-            with tarfile.open(zipfn, 'w:gz') as tar:
-                text = "Done"
-                metadata_file = io.BytesIO(text.encode('ascii'))
-                tarinfo = tarfile.TarInfo(f'{job_hash}/done')
-                tarinfo.size = metadata_file.getbuffer().nbytes
-                tar.addfile(tarinfo=tarinfo, fileobj=metadata_file)
-
-        with open(zipfn, 'rb') as zipf:
-            await resp.write(zipf.read())
+        if not os.path.exists(zipfn):
+            return web.json_response(
+                {'message': 'Could not find results for the requested job/node.'},
+                status=404)
 
-        await resp.write_eof()
-
-        return resp
+        return web.FileResponse(zipfn)
 
     ####################
     async def handle_delete_job(self, request):
@@ -300,9 +337,10 @@ class Server:
         job_hash = job_params['job_hash']
 
         # Determine if the job is running.
-        for job in self.sc_jobs:
-            if job_hash in job:
-                return self.__response("Error: job is still running.", status=400)
+        with self.sc_jobs_lock:
+            for job in self.sc_jobs:
+                if job_hash in job:
+                    return self.__response("Error: job is still running.", status=400)
 
         # Delete job hash directory, only if it exists.
         # TODO: This assumes no malicious input.
@@ -342,16 +380,17 @@ class Server:
 
         # Determine if the job is running.
         # TODO: Return information about individual flowgraph nodes.
-        if jobname in self.sc_jobs:
-            resp = {
-                'status': JobStatus.RUNNING,
-                'message': 'Job is currently running on the server.',
-            }
-        else:
-            resp = {
-                'status': JobStatus.COMPLETED,
-                'message': 'Job has no running steps.',
-            }
+        with self.sc_jobs_lock:
+            if jobname in self.sc_jobs:
+                resp = {
+                    'status': JobStatus.RUNNING,
+                    'message': self.sc_jobs[jobname],
+                }
+            else:
+                resp = {
+                    'status': JobStatus.COMPLETED,
+                    'message': 'Job has no running steps.',
+                }
         return web.json_response(resp)
 
     ####################
@@ -402,18 +441,39 @@ class Server:
         # Assemble core job parameters.
         job_hash = chip.get('record', 'remoteid')
 
+        runtime = RuntimeFlowgraph(
+            chip.schema.get("flowgraph", chip.get('option', 'flow'), field='schema'),
+            from_steps=chip.get('option', 'from'),
+            to_steps=chip.get('option', 'to'),
+            prune_nodes=chip.get('option', 'prune'))
+
+        nodes = {}
+        nodes[None] = {
+            "status": SCNodeStatus.PENDING
+        }
+        for step, index in runtime.get_nodes():
+            status = chip.get('record', 'status', step=step, index=index)
+            if not status:
+                status = SCNodeStatus.PENDING
+            if SCNodeStatus.is_done(status):
+                status = NodeStatus.UPLOADED
+            nodes[f"{step}{index}"] = {
+                "status": status
+            }
+
         # Mark the job run as busy.
         sc_job_name = self.job_name(username, job_hash)
-        self.sc_jobs[sc_job_name] = 'busy'
+        with self.sc_jobs_lock:
+            self.sc_chip_lookup[chip] = {
+                "name": sc_job_name,
+                "jobhash": job_hash
+            }
+            self.sc_jobs[sc_job_name] = nodes
 
         build_dir = os.path.join(self.nfs_mount, job_hash)
         chip.set('option', 'builddir', build_dir)
         chip.set('option', 'remote', False)
 
-        job_cfg_dir = get_configuration_directory(chip)
-        os.makedirs(job_cfg_dir, exist_ok=True)
-        chip.write_manifest(f"{job_cfg_dir}/chip{chip.get('option', 'jobname')}.json")
-
         if self.get('option', 'cluster') == 'slurm':
             # Run the job with slurm clustering.
             chip.set('option', 'scheduler', 'name', 'slurm')
@@ -421,20 +481,10 @@ class Server:
         # Run the job.
         chip.run()
 
-        # Archive each task.
-        for (step, index) in nodes_to_execute(chip):
-            chip.cwd = os.path.join(chip.get('option', 'builddir'), '..')
-            tf = tarfile.open(os.path.join(self.nfs_mount,
-                                           job_hash,
-                                           f'{job_hash}_{step}{index}.tar.gz'),
-                              mode='w:gz')
-            chip._archive_node(tf, step=step, index=index)
-            tf.close()
-
-        # (Email notifications can be sent here using your preferred API)
-
         # Mark the job hash as being done.
-        self.sc_jobs.pop(sc_job_name)
+        with self.sc_jobs_lock:
+            self.sc_jobs.pop(sc_job_name)
+            self.sc_chip_lookup.pop(chip)
 
     ####################
     def __auth_password(self, username, password):

siliconcompiler/report/dashboard/cli/__init__.py

@@ -66,6 +66,8 @@ class CliDashboard(AbstractDashboard):
         """
         self._dashboard.end_of_run(self._chip)
 
+        self._dashboard.stop()
+
         # Restore logger
         if self.__logger_console:
             self._logger.removeHandler(self._dashboard._log_handler)

siliconcompiler/report/dashboard/cli/board.py

@@ -22,6 +22,8 @@ from siliconcompiler import SiliconCompilerError, NodeStatus
 from siliconcompiler.utils.logging import SCColorLoggerFormatter
 from siliconcompiler.flowgraph import RuntimeFlowgraph
 
+import atexit
+
 
 class LogBufferHandler(logging.Handler):
     def __init__(self, sync_queue, n=50, event=None):
@@ -246,6 +248,15 @@ class Board(metaclass=BoardSingleton):
 
     def _init_singleton(self):
         self._console = Console(theme=Board.__theme)
+
+        self.live = Live(
+            console=self._console,
+            screen=False,
+            auto_refresh=True,
+        )
+
+        atexit.register(self._stop_on_exit)
+
         self._active = self._console.is_terminal
         if not self._active:
             self._console = None
@@ -279,6 +290,9 @@ class Board(metaclass=BoardSingleton):
 
         self._metrics = ("warnings", "errors")
 
+    def _stop_on_exit(self):
+        self.stop()
+
     def open_dashboard(self):
         """Starts the dashboard rendering thread if it is not already running."""
 
@@ -289,7 +303,7 @@ class Board(metaclass=BoardSingleton):
             self._update_render_data(None)
 
             with self._job_data_lock:
-                if not self._render_thread:
+                if not self._render_thread or not self._render_thread.is_alive():
                     self._render_thread = threading.Thread(target=self._render, daemon=True)
                     self._render_event.clear()
                     self._render_stop_event.clear()
@@ -328,7 +342,6 @@ class Board(metaclass=BoardSingleton):
             return
 
         self._update_render_data(chip, complete=True)
-        self.stop()
 
     def stop(self):
         """
@@ -337,18 +350,20 @@ class Board(metaclass=BoardSingleton):
         if not self.is_running():
             return
 
-        # check for running jobs
-        with self._job_data_lock:
-            if self._job_data:
-                if any([not job.complete for job in self._job_data.values()]):
-                    return
+        if self._job_data:
+            if any([not job.complete for job in self._job_data.values()]):
+                return
 
-            self._render_stop_event.set()
-            self._render_event.set()
+        self._render_stop_event.set()
+        self._render_event.set()
 
         # Wait for rendering to finish
         self.wait()
 
+        # Restore terminal
+        self.live.stop()
+        self._console.show_cursor()
+
     def wait(self):
         """Waits for the dashboard rendering thread to finish."""
         if not self.is_running():
@@ -595,18 +610,11 @@ class Board(metaclass=BoardSingleton):
                 # Catch any multiprocessing errors
                 return True
 
-        live = None
         try:
             update_data()
-            live = Live(
-                self._get_rendable(),
-                console=self._console,
-                screen=False,
-                # transient=True,
-                auto_refresh=True,
-                # refresh_per_second=60,
-            )
-            live.start()
+
+            if not self.live.is_started:
+                self.live.start(refresh=True)
 
             while not check_stop_event():
                 try:
@@ -620,18 +628,14 @@ class Board(metaclass=BoardSingleton):
                     break
 
                 update_data()
-                live.update(self._get_rendable(), refresh=True)
+                self.live.update(self._get_rendable(), refresh=True)
+
         finally:
-            try:
-                update_data()
-                if live:
-                    live.update(self._get_rendable(), refresh=True)
-                    live.stop()
-                else:
-                    self._console.print(self._get_rendable())
-            finally:
-                # Restore the prompt
-                print("\033[?25h", end="")
+            update_data()
+            if self.live:
+                self.live.update(self._get_rendable(), refresh=True)
+            else:
+                self._console.print(self._get_rendable())
 
     def _update_layout(self):
         with self._render_data_lock:
@@ -747,7 +751,6 @@ class Board(metaclass=BoardSingleton):
 
             runtime_flow = RuntimeFlowgraph(
                 chip.schema.get("flowgraph", flow, field='schema'),
-                args=(chip.get('arg', 'step'), chip.get('arg', 'index')),
                 to_steps=chip.get('option', 'to'),
                 prune_nodes=chip.get('option', 'prune'))
             record = chip.schema.get("record", field='schema')

siliconcompiler/report/report.py

@@ -3,8 +3,8 @@ import os
 from siliconcompiler import Schema
 from siliconcompiler.schema import PerNode
 from siliconcompiler.report import utils
-from siliconcompiler.utils.flowgraph import nodes_to_execute
 from siliconcompiler.tools._common import get_tool_task
+from siliconcompiler.flowgraph import RuntimeFlowgraph
 
 
 def make_metric_dataframe(chip):
@@ -199,7 +199,12 @@ def get_flowgraph_path(chip):
         Returns the "winning" path for that job.
     '''
     flow = chip.get('option', 'flow')
-    return utils._get_flowgraph_path(chip, flow, nodes_to_execute(chip))
+    runtime = RuntimeFlowgraph(
+        chip.schema.get("flowgraph", flow, field='schema'),
+        from_steps=chip.get('option', 'from'),
+        to_steps=chip.get('option', 'to'),
+        prune_nodes=chip.get('option', 'prune'))
+    return utils._get_flowgraph_path(chip, flow, runtime.get_nodes())
 
 
 def search_manifest_keys(manifest, key):
@@ -289,9 +294,9 @@ def get_total_manifest_key_count(manifest):
         acc (int) : An accumulator of the current number of folders and files.
     '''
     acc = len(manifest)
-    for dictKeys in manifest:
-        if isinstance(manifest[dictKeys], dict):
-            acc += get_total_manifest_key_count(manifest[dictKeys])
+    for key in manifest:
+        if isinstance(manifest[key], dict):
+            acc += get_total_manifest_key_count(manifest[key])
     return acc
 
 

siliconcompiler/report/utils.py

@@ -1,14 +1,13 @@
 from siliconcompiler import NodeStatus
 from siliconcompiler.utils import units
-from siliconcompiler.utils.flowgraph import _get_flowgraph_execution_order, \
-    nodes_to_execute
 from siliconcompiler.tools._common import get_tool_task
 
 from siliconcompiler.flowgraph import RuntimeFlowgraph
 
 
 def _find_summary_image(chip, ext='png'):
-    for nodes in reversed(_get_flowgraph_execution_order(chip, chip.get('option', 'flow'))):
+    for nodes in reversed(chip.schema.get(
+            "flowgraph", chip.get('option', 'flow'), field="schema").get_execution_order()):
         for step, index in nodes:
             layout_img = chip.find_result(ext, step=step, index=index)
             if layout_img:
@@ -18,7 +17,8 @@ def _find_summary_image(chip, ext='png'):
 
 def _find_summary_metrics(chip, metrics_map):
     metrics = {}
-    for nodes in reversed(_get_flowgraph_execution_order(chip, chip.get('option', 'flow'))):
+    for nodes in reversed(chip.schema.get(
+            "flowgraph", chip.get('option', 'flow'), field="schema").get_execution_order()):
         for step, index in nodes:
             for name, metric_info in metrics_map.items():
                 if name in metrics:
@@ -44,7 +44,13 @@ def _collect_data(chip, flow=None, flowgraph_nodes=None, format_as_string=True):
         return [], {}, {}, {}, [], {}
 
     if not flowgraph_nodes:
-        flowgraph_nodes = list(nodes_to_execute(chip))
+        runtime = RuntimeFlowgraph(
+            chip.schema.get("flowgraph", flow, field='schema'),
+            from_steps=chip.get('option', 'from'),
+            to_steps=chip.get('option', 'to'),
+            prune_nodes=chip.get('option', 'prune'))
+
+        flowgraph_nodes = list(runtime.get_nodes())
         # only report tool based steps functions
         for (step, index) in list(flowgraph_nodes):
             tool, task = get_tool_task(chip, step, '0', flow=flow)
@@ -64,7 +70,7 @@ def _collect_data(chip, flow=None, flowgraph_nodes=None, format_as_string=True):
     reports = {}
 
     # Build ordered list of nodes in flowgraph
-    for level_nodes in _get_flowgraph_execution_order(chip, flow):
+    for level_nodes in chip.schema.get("flowgraph", flow, field="schema").get_execution_order():
         nodes.extend(sorted(level_nodes))
     nodes = [node for node in nodes if node in flowgraph_nodes]
     for (step, index) in nodes: