siliconcompiler 0.28.9__py3-none-any.whl → 0.29.1__py3-none-any.whl

siliconcompiler/report/report.py

@@ -60,6 +60,8 @@ def get_flowgraph_nodes(chip, step, index):
         if value is not None:
             if key == 'inputnode':
                 value = ", ".join([f'{step}{index}' for step, index in value])
+            if key == 'pythonpackage':
+                value = ", ".join(value)
             nodes[key] = str(value)
     return nodes
 

siliconcompiler/report/summary_table.py

@@ -14,7 +14,7 @@ def _show_summary_table(chip, flow, flowgraph_nodes, show_all_indices):
     # Display data
     column_width = 15
 
-    max_line_width = max(2 * column_width, int(0.95*shutil.get_terminal_size().columns))
+    max_line_width = max(4 * column_width, int(0.95*shutil.get_terminal_size().columns))
 
     nodes, _, metrics, metrics_unit, metrics_to_show, _ = \
         _collect_data(chip, flow, flowgraph_nodes)

siliconcompiler/scheduler/__init__.py

@@ -1,3 +1,4 @@
+import contextlib
 import copy
 import distro
 import getpass
@@ -20,7 +21,7 @@ from datetime import datetime
 from siliconcompiler import sc_open
 from siliconcompiler import utils
 from siliconcompiler import _metadata
-from siliconcompiler.remote import client
+from siliconcompiler.remote import Client
 from siliconcompiler.schema import Schema
 from siliconcompiler.scheduler import slurm
 from siliconcompiler.scheduler import docker_runner
@@ -92,6 +93,7 @@ def run(chip):
     copy_old_run_dir(chip, org_jobname)
     clean_build_dir(chip)
     _reset_flow_nodes(chip, flow, nodes_to_execute(chip, flow))
+    __record_packages(chip)
 
     # Save current environment
     environment = copy.deepcopy(os.environ)
@@ -101,7 +103,8 @@ def run(chip):
         os.environ[envvar] = val
 
     if chip.get('option', 'remote'):
-        client.remote_process(chip)
+        client = Client(chip)
+        client.run()
     else:
         _local_process(chip, flow)
 
@@ -427,7 +430,7 @@ def _check_version(chip, reported_version, tool, step, index):
 
 
 ###########################################################################
-def _runtask(chip, flow, step, index, exec_func, replay=False):
+def _runtask(chip, flow, step, index, exec_func, pipe=None, replay=False):
     '''
     Private per node run method called by run().
 
@@ -477,6 +480,9 @@ def _runtask(chip, flow, step, index, exec_func, replay=False):
     os.chdir(cwd)
     chip.schema._stop_journal()
 
+    if pipe:
+        pipe.send(chip._packages)
+
 
 ###########################################################################
 def _haltstep(chip, flow, step, index, log=True):
@@ -612,17 +618,19 @@ def _copy_previous_steps_output_data(chip, step, index, replay):
                     os.rename(f'inputs/{outfile.name}', f'inputs/{new_name}')
 
 
-def __read_std_streams(chip, quiet, is_stdout_log, stdout_reader, is_stderr_log, stderr_reader):
+def __read_std_streams(chip, quiet,
+                       is_stdout_log, stdout_reader, stdout_print,
+                       is_stderr_log, stderr_reader, stderr_print):
     '''
     Handle directing tool outputs to logger
     '''
     if not quiet:
         if is_stdout_log:
             for line in stdout_reader.readlines():
-                chip.logger.info(line.rstrip())
+                stdout_print(line.rstrip())
         if is_stderr_log:
             for line in stderr_reader.readlines():
-                chip.logger.error(line.rstrip())
+                stderr_print(line.rstrip())
 
 
 ############################################################################
@@ -770,36 +778,89 @@ def _makecmd(chip, tool, task, step, index, script_name='replay.sh', include_pat
     return cmdlist, print_cmd, cmd, cmd_args
 
 
+def __get_stdio(chip, tool, task, flow, step, index):
+    def get_file(io_type):
+        suffix = chip.get('tool', tool, 'task', task, io_type, 'suffix',
+                          step=step, index=index)
+        destination = chip.get('tool', tool, 'task', task, io_type, 'destination',
+                               step=step, index=index)
+
+        io_file = None
+        if destination == 'log':
+            io_file = step + "." + suffix
+        elif destination == 'output':
+            io_file = os.path.join('outputs', chip.top() + "." + suffix)
+        elif destination == 'none':
+            io_file = os.devnull
+        else:
+            # This should not happen
+            chip.logger.error(f'{io_type}/destination has no support for {destination}.')
+            _haltstep(chip, flow, step, index)
+
+        return io_file
+
+    stdout_file = get_file('stdout')
+    stderr_file = get_file('stderr')
+
+    return stdout_file, stderr_file
+
+
 def _run_executable_or_builtin(chip, step, index, version, toolpath, workdir, run_func=None):
     '''
     Run executable (or copy inputs to outputs for builtin functions)
     '''
 
     flow = chip.get('option', 'flow')
-    top = chip.top()
     tool, task = get_tool_task(chip, step, index, flow)
 
     quiet = (
-        chip.get('option', 'quiet', step=step, index=index) and not
-        chip.get('option', 'breakpoint', step=step, index=index)
+        chip.get('option', 'quiet', step=step, index=index) and
+        not chip.get('option', 'breakpoint', step=step, index=index)
     )
 
+    stdout_print = chip.logger.info
+    stderr_print = chip.logger.error
+    if chip.get('option', 'loglevel', step=step, index=index) == "quiet":
+        stdout_print = chip.logger.error
+        stderr_print = chip.logger.error
+
     # TODO: Currently no memory usage tracking in breakpoints, builtins, or unexpected errors.
     max_mem_bytes = 0
 
+    stdout_file, stderr_file = __get_stdio(chip, tool, task, flow, step, index)
+    is_stdout_log = chip.get('tool', tool, 'task', task, 'stdout', 'destination',
+                             step=step, index=index) == 'log'
+    is_stderr_log = chip.get('tool', tool, 'task', task, 'stderr', 'destination',
+                             step=step, index=index) == 'log' and stderr_file != stdout_file
+
     retcode = 0
     cmdlist = []
     cmd_args = []
     if run_func:
         logfile = None
         try:
-            retcode = run_func(chip)
+            with open(stdout_file, 'w') as stdout_writer, \
+                    open(stderr_file, 'w') as stderr_writer:
+                if stderr_file == stdout_file:
+                    stderr_writer.close()
+                    stderr_writer = sys.stdout
+
+                with contextlib.redirect_stderr(stderr_writer), \
+                        contextlib.redirect_stdout(stdout_writer):
+                    retcode = run_func(chip)
         except Exception as e:
             chip.logger.error(f'Failed in run() for {tool}/{task}: {e}')
             retcode = 1  # default to non-zero
             print_traceback(chip, e)
             chip._error = True
         finally:
+            with sc_open(stdout_file) as stdout_reader, \
+                    sc_open(stderr_file) as stderr_reader:
+                __read_std_streams(chip,
+                                   quiet,
+                                   is_stdout_log, stdout_reader, stdout_print,
+                                   is_stderr_log, stderr_reader, stderr_print)
+
             try:
                 if resource:
                     # Since memory collection is not possible, collect the current process
@@ -838,48 +899,10 @@ def _run_executable_or_builtin(chip, step, index, version, toolpath, workdir, ru
                 import pty  # Note: this import throws exception on Windows
                 retcode = pty.spawn(cmdlist, read)
         else:
-            stdout_file = ''
-            stdout_suffix = chip.get('tool', tool, 'task', task, 'stdout', 'suffix',
-                                     step=step, index=index)
-            stdout_destination = chip.get('tool', tool, 'task', task, 'stdout', 'destination',
-                                          step=step, index=index)
-            if stdout_destination == 'log':
-                stdout_file = step + "." + stdout_suffix
-            elif stdout_destination == 'output':
-                stdout_file = os.path.join('outputs', top + "." + stdout_suffix)
-            elif stdout_destination == 'none':
-                stdout_file = os.devnull
-            else:
-                chip.logger.error(f'stdout/destination has no support for {stdout_destination}. '
-                                  'Use [log|output|none].')
-                _haltstep(chip, flow, step, index)
-
-            stderr_file = ''
-            stderr_suffix = chip.get('tool', tool, 'task', task, 'stderr', 'suffix',
-                                     step=step, index=index)
-            stderr_destination = chip.get('tool', tool, 'task', task, 'stderr', 'destination',
-                                          step=step, index=index)
-            if stderr_destination == 'log':
-                stderr_file = step + "." + stderr_suffix
-            elif stderr_destination == 'output':
-                stderr_file = os.path.join('outputs', top + "." + stderr_suffix)
-            elif stderr_destination == 'none':
-                stderr_file = os.devnull
-            else:
-                chip.logger.error(f'stderr/destination has no support for {stderr_destination}. '
-                                  'Use [log|output|none].')
-                _haltstep(chip, flow, step, index)
-
             with open(stdout_file, 'w') as stdout_writer, \
                     open(stdout_file, 'r', errors='replace_with_warning') as stdout_reader, \
                     open(stderr_file, 'w') as stderr_writer, \
                     open(stderr_file, 'r', errors='replace_with_warning') as stderr_reader:
-                # Use separate reader/writer file objects as hack to display
-                # live output in non-blocking way, so we can monitor the
-                # timeout. Based on https://stackoverflow.com/a/18422264.
-                is_stdout_log = chip.get('tool', tool, 'task', task, 'stdout', 'destination',
-                                         step=step, index=index) == 'log'
-                is_stderr_log = stderr_destination == 'log' and stderr_file != stdout_file
                 # if STDOUT and STDERR are to be redirected to the same file,
                 # use a single writer
                 if stderr_file == stdout_file:
@@ -937,8 +960,8 @@ def _run_executable_or_builtin(chip, step, index, version, toolpath, workdir, ru
                         # Loop until process terminates
                         __read_std_streams(chip,
                                            quiet,
-                                           is_stdout_log, stdout_reader,
-                                           is_stderr_log, stderr_reader)
+                                           is_stdout_log, stdout_reader, stdout_print,
+                                           is_stderr_log, stderr_reader, stderr_print)
 
                         if timeout is not None and time.time() - cmd_start_time > timeout:
                             chip.logger.error(f'Step timed out after {timeout} seconds')
@@ -956,8 +979,8 @@ def _run_executable_or_builtin(chip, step, index, version, toolpath, workdir, ru
                 # Read the remaining
                 __read_std_streams(chip,
                                    quiet,
-                                   is_stdout_log, stdout_reader,
-                                   is_stderr_log, stderr_reader)
+                                   is_stdout_log, stdout_reader, stdout_print,
+                                   is_stderr_log, stderr_reader, stderr_print)
                 retcode = proc.returncode
 
     chip.set('record', 'toolexitcode', retcode, step=step, index=index)
@@ -1389,7 +1412,10 @@ def _reset_flow_nodes(chip, flow, nodes_to_execute):
         for metric in chip.getkeys('metric'):
             _clear_metric(chip, step, index, metric)
         for record in chip.getkeys('record'):
-            _clear_record(chip, step, index, record, preserve=['remoteid', 'status'])
+            _clear_record(chip, step, index, record, preserve=[
+                'remoteid',
+                'status',
+                'pythonpackage'])
 
     # Mark all nodes as pending
     for step, index in _get_flowgraph_nodes(chip, flow):
@@ -1464,8 +1490,18 @@ def _prepare_nodes(chip, nodes_to_run, processes, local_processes, flow):
         else:
             local_processes.append((step, index))
 
-        processes[node] = multiprocessing.Process(target=_runtask,
-                                                  args=(chip, flow, step, index, exec_func))
+        process = {
+            "child_pipe": None,
+            "parent_pipe": None,
+            "proc": None
+        }
+        process["parent_pipe"], process["child_pipe"] = multiprocessing.Pipe()
+        process["proc"] = multiprocessing.Process(
+            target=_runtask,
+            args=(chip, flow, step, index, exec_func),
+            kwargs={"pipe": process["child_pipe"]})
+
+        processes[node] = process
 
     for init_func in init_funcs:
         init_func(chip)
@@ -1563,7 +1599,7 @@ def _launch_nodes(chip, nodes_to_run, processes, local_processes):
                     chip.set('record', 'status', NodeStatus.RUNNING, step=node[0], index=node[1])
                     changed = True
 
-                    processes[node].start()
+                    processes[node]["proc"].start()
                     del nodes_to_run[node]
                     running_nodes[node] = requested_threads
 
@@ -1586,7 +1622,7 @@ def _launch_nodes(chip, nodes_to_run, processes, local_processes):
 def _process_completed_nodes(chip, processes, running_nodes):
     changed = False
     for node in list(running_nodes.keys()):
-        if not processes[node].is_alive():
+        if not processes[node]["proc"].is_alive():
             step, index = node
             manifest = os.path.join(chip.getworkdir(step=step, index=index),
                                     'outputs',
@@ -1595,8 +1631,16 @@ def _process_completed_nodes(chip, processes, running_nodes):
             if os.path.exists(manifest):
                 chip.schema.read_journal(manifest)
 
+            if processes[node]["parent_pipe"] and processes[node]["parent_pipe"].poll(1):
+                try:
+                    packages = processes[node]["parent_pipe"].recv()
+                    if isinstance(packages, dict):
+                        chip._packages.update(packages)
+                except:  # noqa E722
+                    pass
+
             del running_nodes[node]
-            if processes[node].exitcode > 0:
+            if processes[node]["proc"].exitcode > 0:
                 status = NodeStatus.ERROR
             else:
                 status = chip.get('record', 'status', step=step, index=index)
@@ -1627,6 +1671,22 @@ def _check_nodes_status(chip, flow):
 #######################################
 def __record_version(chip, step, index):
     chip.set('record', 'scversion', _metadata.version, step=step, index=index)
+    chip.set('record', 'pythonversion', platform.python_version(), step=step, index=index)
+
+
+#######################################
+def __record_packages(chip):
+    try:
+        from pip._internal.operations.freeze import freeze
+    except:  # noqa E722
+        freeze = None
+
+    if freeze:
+        # clear record
+        chip.set('record', 'pythonpackage', [])
+
+        for pkg in freeze():
+            chip.add('record', 'pythonpackage', pkg)
 
 
 #######################################

siliconcompiler/scheduler/send_messages.py

@@ -12,7 +12,6 @@ import fastjsonschema
 from pathlib import Path
 from siliconcompiler.flowgraph import get_executed_nodes
 import uuid
-from siliconcompiler.targets import freepdk45_demo
 
 
 # Compile validation code for API request bodies.
@@ -176,6 +175,7 @@ def send(chip, msg_type, step, index):
 
 if __name__ == "__main__":
     from siliconcompiler import Chip
+    from siliconcompiler.targets import freepdk45_demo
     chip = Chip('test')
     chip.use(freepdk45_demo)
     chip.set('option', 'scheduler', 'msgevent', 'ALL')

siliconcompiler/schema/schema_cfg.py

@@ -10,7 +10,7 @@ try:
 except ImportError:
     from siliconcompiler.schema.utils import trim
 
-SCHEMA_VERSION = '0.48.4'
+SCHEMA_VERSION = '0.48.6'
 
 #############################################################################
 # PARAM DEFINITION
@@ -2482,6 +2482,9 @@ def schema_record(cfg, step='default', index='default'):
                'toolargs': ['tool CLI arguments',
                             '-I include/ foo.v',
                             'Arguments passed to tool via CLI.'],
+               'pythonversion': ['Python version',
+                                 '3.12.3',
+                                 """Version of python used to run this task."""],
                'osversion': ['O/S version',
                              '20.04.1-Ubuntu',
                              """Since there is not standard version system for operating
@@ -2515,16 +2518,25 @@ def schema_record(cfg, step='default', index='default'):
             pernode='required',
             schelp='Record tracking the tool exit code per step and index basis.')
 
-    # Unlike most other 'record' fields, job ID is not set per-node.
+    # Non-per-node records.
     scparam(cfg, ['record', 'remoteid'],
             sctype='str',
             shorthelp="Record: remote job ID",
-            switch="-record_remoteid 'step index <str>'",
+            switch="-record_remoteid '<str>'",
             example=[
                 "cli: -record_remoteid '0123456789abcdeffedcba9876543210'",
                 "api: chip.set('record', 'remoteid', '0123456789abcdeffedcba9876543210')"],
             schelp='Record tracking the job ID for a remote run.')
 
+    scparam(cfg, ['record', 'pythonpackage'],
+            sctype='[str]',
+            shorthelp="Record: python packages",
+            switch="-record_pythonpackage '<str>'",
+            example=[
+                "cli: -record_pythonpackage 'siliconcompiler==0.28.0'",
+                "api: chip.set('record', 'pythonpackage', 'siliconcompiler==0.28.0')"],
+            schelp='Record tracking for the python packages installed.')
+
     # flowgraph status
     scparam(cfg, ['record', 'status'],
             sctype='enum',
@@ -2760,7 +2772,7 @@ def schema_option(cfg):
 
     scparam(cfg, ['option', 'loglevel'],
             sctype='enum',
-            enum=["info", "warning", "error", "critical", "debug"],
+            enum=["info", "warning", "error", "critical", "debug", "quiet"],
             pernode='optional',
             scope='job',
             defvalue='info',

siliconcompiler/schema/schema_obj.py

@@ -506,6 +506,22 @@ class Schema:
             else:
                 self.set(*key, values, step=step, index=index)
 
+    ###########################################################################
+    def copy_key(self, src, dst):
+        '''
+        Copy a parameters information from the source keypath to the destination
+        keypath.
+
+        Args:
+            src (list): Key to use as source.
+            dst (list): Key to use as destination
+        '''
+
+        data = self.getdict(*src)
+
+        cfg = self.__search(*dst[0:-1], insert_defaults=True)
+        cfg[dst[-1]] = data
+
     ###########################################################################
     def remove(self, *keypath):
         '''
@@ -1495,7 +1511,7 @@ class Schema:
             switchstrs, metavar = self.__get_switches(schema, *keypath)
 
             # Three switch types (bool, list, scalar)
-            if not switchlist or any(switch in switchlist for switch in switchstrs):
+            if switchlist is None or any(switch in switchlist for switch in switchstrs):
                 used_switches.update(switchstrs)
                 if typestr == 'bool':
                     # Boolean type arguments
@@ -1590,7 +1606,18 @@ class Schema:
         # Grab argument from pre-process sysargs
         cmdargs = vars(parser.parse_args(scargs))
 
-        if print_banner:
+        # Set loglevel if set at command line
+        do_print_banner = True
+        if 'option_loglevel' in cmdargs.keys():
+            log_level = cmdargs['option_loglevel']
+            if isinstance(log_level, list):
+                # if multiple found, pick the first one
+                log_level = log_level[0]
+            if log_level == 'quiet':
+                do_print_banner = False
+            logger.setLevel(translate_loglevel(log_level).split()[-1])
+
+        if print_banner and do_print_banner:
             print_banner()
 
         extra_params = None
@@ -1607,14 +1634,6 @@ class Schema:
                     # Remove from cmdargs
                     del cmdargs[arg]
 
-        # Set loglevel if set at command line
-        if 'option_loglevel' in cmdargs.keys():
-            log_level = cmdargs['option_loglevel']
-            if isinstance(log_level, list):
-                # if multiple found, pick the first one
-                log_level = log_level[0]
-            logger.setLevel(translate_loglevel(log_level).split()[-1])
-
         # Read in all cfg files
         if 'option_cfg' in cmdargs.keys():
             for item in cmdargs['option_cfg']:

siliconcompiler/schema/utils.py

@@ -90,4 +90,6 @@ def trim(docstring):
 
 
 def translate_loglevel(level):
+    if level == "quiet":
+        level = "error"
     return level.upper()

siliconcompiler/sphinx_ext/__init__.py

@@ -0,0 +1,85 @@
+import os.path
+
+import siliconcompiler
+from siliconcompiler import __version__ as sc_version
+
+from siliconcompiler.targets import get_targets
+from siliconcompiler.flows import get_flows
+from siliconcompiler.checklists import get_checklists
+from siliconcompiler.libs import get_libs
+from siliconcompiler.pdks import get_pdks
+from siliconcompiler.apps import get_apps
+from siliconcompiler.tools import get_tools
+
+
+sc_root = os.path.dirname(os.path.dirname(os.path.abspath(siliconcompiler.__file__)))
+
+
+def relpath(file):
+    file = os.path.abspath(file)
+    if file.startswith(sc_root):
+        return os.path.relpath(file, sc_root)
+    return None
+
+
+def get_codeurl(file=None):
+    base_url = f"https://github.com/siliconcompiler/siliconcompiler/blob/v{sc_version}"
+
+    if not file:
+        return base_url
+
+    if os.path.isabs(file):
+        file = relpath(file)
+        if not file:
+            return None
+
+    return f"{base_url}/{file}"
+
+
+def targets():
+    modules = []
+    for name, mod in get_targets().items():
+        modules.append((mod, name))
+    return modules
+
+
+def flows():
+    modules = []
+    for name, mod in get_flows().items():
+        modules.append((mod, name))
+    return modules
+
+
+def libraries():
+    modules = []
+    for name, mod in get_libs().items():
+        modules.append((mod, name))
+    return modules
+
+
+def pdks():
+    modules = []
+    for name, mod in get_pdks().items():
+        modules.append((mod, name))
+    return modules
+
+
+def tools():
+    modules = []
+    for name, mod in get_tools().items():
+        modules.append((mod, name))
+    return modules
+
+
+def apps():
+    modules = []
+    for name, mod in get_apps().items():
+        modules.append((mod, name))
+    return modules
+
+
+def checklists():
+    modules = []
+    for name, mod in get_checklists().items():
+        modules.append((mod, name))
+    return modules