siliconcompiler 0.32.3__py3-none-any.whl → 0.33.0__py3-none-any.whl

siliconcompiler/schema/cmdlineschema.py

@@ -0,0 +1,250 @@
+import argparse
+import re
+import sys
+
+
+class CommandLineSchema:
+    '''
+    Class to provide the :meth:`create_cmdline` option to a schema object.
+
+    This class should not be instantiated by itself.
+
+    Examples:
+        class NewSchema(BaseSchema, CommandLineSchema):
+            creates a new class with the commandline options available
+    '''
+
+    ###########################################################################
+    def create_cmdline(self,
+                       progname,
+                       description=None,
+                       switchlist=None,
+                       additional_args=None,
+                       version=None,
+                       print_banner=None,
+                       input_map_handler=None,
+                       preprocess_keys=None,
+                       post_process=None,
+                       logger=None):
+        """Creates a Schema command line interface.
+
+        Exposes parameters in the SC schema as command line switches,
+        simplifying creation of SC apps with a restricted set of schema
+        parameters exposed at the command line. The order of command
+        line switch settings parsed from the command line is as follows:
+
+         1. loglevel, if available in schema
+         2. read_manifest([cfg]), if available in schema
+         3. read inputs with input_map_handler
+         4. all other switches
+         5. Run post_process
+
+        The cmdline interface is implemented using the Python argparse package
+        and the following use restrictions apply.
+
+        * Help is accessed with the '-h' switch.
+        * Arguments that include spaces must be enclosed with double quotes.
+        * List parameters are entered individually. (ie. -y libdir1 -y libdir2)
+        * For parameters with Boolean types, the switch implies "true".
+        * Special characters (such as '-') must be enclosed in double quotes.
+        * Compiler compatible switches include: -D, -I, -O{0,1,2,3}
+        * Legacy switch formats are supported: +libext+, +incdir+
+
+        Args:
+            progname (str): Name of program to be executed.
+            description (str): Short program description.
+            switchlist (list of str): List of SC parameter switches to expose
+                at the command line. By default all SC schema switches are
+                available. Parameter switches should be entered based on the
+                parameter 'switch' field in the schema. For parameters with
+                multiple switches, both will be accepted if any one is included
+                in this list.
+            input_map (dict of str): Dictionary mapping file extensions to input
+                filetypes. This is used to automatically assign positional
+                source arguments to ['input', 'fileset', ...] keypaths based on their file
+                extension. If None, the CLI will not accept positional source
+                arguments.
+            additional_args (dict of dict): Dictionary of extra arguments to add
+                to the command line parser, with the arguments matching the
+                argparse.add_argument() call.
+            version (str): Version to report when calling with -version
+            print_banner (function): Function callback to print command line banner
+            input_map_handler (function): Function callback handle inputs to the input map
+            preprocess_keys (function): Function callback to preprocess keys that need to be
+                corrected
+            post_process (function): Function callback to process arguments before returning
+
+        Returns:
+            None if additional_args is not provided, otherwise a dictionary with the
+                command line options detected from the additional_args
+
+        Examples:
+            >>> schema.create_cmdline(progname='sc-show',switchlist=['-input','-cfg'])
+            Creates a command line interface for 'sc-show' app.
+            >>> schema.create_cmdline(progname='sc', input_map={'v': ('rtl', 'verilog')})
+            All sources ending in .v will be stored in ['input', 'rtl', 'verilog']
+            >>> extra = schema.create_cmdline(progname='sc',
+                                              additional_args={'-demo': {'action': 'store_true'}})
+            Returns extra = {'demo': False/True}
+        """
+
+        # Argparse
+        parser = argparse.ArgumentParser(prog=progname,
+                                         prefix_chars='-+',
+                                         formatter_class=argparse.RawDescriptionHelpFormatter,
+                                         description=description,
+                                         allow_abbrev=False)
+
+        # Create empty copy of schema
+        schema = type(self)()
+
+        # Track arguments
+        argument_map = {}
+        arguments = set()
+
+        # Iterate over all keys from an empty schema to add parser arguments
+        for keypath in sorted(schema.allkeys()):
+            param = schema.get(*keypath, field=None)
+
+            dest, switches = param.add_commandline_arguments(
+                parser,
+                *keypath,
+                switchlist=switchlist)
+
+            if switches:
+                argument_map[dest] = (keypath, param)
+                arguments.update(switches)
+
+        print_additional_arg_value = {}
+        if additional_args:
+            # Add additional user specified arguments
+            arg_dests = []
+            for arg, arg_detail in additional_args.items():
+                do_print = True
+                if "sc_print" in arg_detail:
+                    do_print = arg_detail["sc_print"]
+                    del arg_detail["sc_print"]
+                argument = parser.add_argument(arg, **arg_detail)
+                print_additional_arg_value[argument.dest] = do_print
+
+                arg_dests.append(argument.dest)
+                arguments.add(arg)
+            # rewrite additional_args with new dest information
+            additional_args = arg_dests
+
+        if version:
+            parser.add_argument('-version', action='version', version=version)
+
+        # Check if there are invalid switches
+        if switchlist:
+            for switch in switchlist:
+                if switch not in arguments:
+                    raise ValueError(f'{switch} is not a valid commandline argument')
+
+        if input_map_handler:
+            parser.add_argument('source',
+                                nargs='*',
+                                help='Input files with filetype inferred by extension')
+
+        # Preprocess sys.argv to enable linux commandline switch formats
+        # (gcc, verilator, etc)
+        scargs = []
+
+        # Iterate from index 1, otherwise we end up with script name as a
+        # 'source' positional argument
+        for argument in sys.argv[1:]:
+            # Split switches with one character and a number after (O0,O1,O2)
+            opt = re.match(r'(\-\w)(\d+)', argument)
+            # Split assign switches (-DCFG_ASIC=1)
+            assign = re.search(r'(\-\w)(\w+\=\w+)', argument)
+            # Split plusargs (+incdir+/path)
+            plusarg = re.search(r'(\+\w+\+)(.*)', argument)
+            if opt:
+                scargs.append(opt.group(1))
+                scargs.append(opt.group(2))
+            elif plusarg:
+                scargs.append(plusarg.group(1))
+                scargs.append(plusarg.group(2))
+            elif assign:
+                scargs.append(assign.group(1))
+                scargs.append(assign.group(2))
+            else:
+                scargs.append(argument)
+
+        # Grab argument from pre-process sysargs
+        cmdargs = vars(parser.parse_args(scargs))
+
+        # Set loglevel if set at command line
+        do_print_banner = True
+        if 'option_loglevel' in cmdargs:
+            log_level = cmdargs['option_loglevel']
+            if isinstance(log_level, list):
+                # if multiple found, pick the first one
+                log_level = log_level[0]
+            if log_level == 'quiet':
+                do_print_banner = False
+            if logger:
+                logger.setLevel(log_level.split()[-1].upper())
+
+        if print_banner and do_print_banner:
+            print_banner()
+
+        extra_params = None
+        if additional_args:
+            # Grab user specified arguments
+            extra_params = {}
+            for arg in additional_args:
+                if arg in cmdargs:
+                    val = cmdargs[arg]
+                    if print_additional_arg_value[arg] and val and logger:
+                        logger.info(
+                            f'Command line argument entered: "{arg}" Value: {val}')
+                    extra_params[arg] = val
+                    # Remove from cmdargs
+                    del cmdargs[arg]
+
+        # Read in all cfg files
+        if 'option_cfg' in cmdargs.keys():
+            for item in cmdargs['option_cfg']:
+                self.read_manifest(item)
+
+        if input_map_handler:
+            # Map sources to ['input'] keypath.
+            if 'source' in cmdargs:
+                input_map_handler(cmdargs['source'])
+                # we don't want to handle this in the next loop
+                del cmdargs['source']
+
+        # Cycle through all command args and write to manifest
+        for dest, vals in sorted(cmdargs.items(), key=lambda d: d[0]):
+            keypath, param = argument_map[dest]
+
+            # Turn everything into a list for uniformity
+            if not isinstance(vals, list):
+                vals = [vals]
+
+            # Cycle through all items
+            for item in vals:
+                if preprocess_keys:
+                    item = preprocess_keys(keypath, item)
+
+                valkeypath, step, index, item = param.parse_commandline_arguments(item, *keypath)
+
+                if logger:
+                    msg = f'Command line argument entered: [{",".join(valkeypath)}] Value: {item}'
+                    if step is not None:
+                        msg += f' step: {step}'
+                    if index is not None:
+                        msg += f' index: {index}'
+                    logger.info(msg)
+
+                # Storing in manifest
+                if param.is_list():
+                    self.add(*valkeypath, item, step=step, index=index)
+                else:
+                    self.set(*valkeypath, item, step=step, index=index)
+
+        if post_process:
+            extra_params = post_process(cmdargs, extra_params)
+
+        return extra_params

siliconcompiler/{sphinx_ext → schema/docs}/__init__.py

@@ -1,5 +1,7 @@
 import os.path
 
+from pathlib import PureWindowsPath
+
 import siliconcompiler
 from siliconcompiler import __version__ as sc_version
 
@@ -18,7 +20,7 @@ sc_root = os.path.dirname(os.path.dirname(os.path.abspath(siliconcompiler.__file
 def relpath(file):
     file = os.path.abspath(file)
     if file.startswith(sc_root):
-        return os.path.relpath(file, sc_root)
+        return PureWindowsPath(os.path.relpath(file, sc_root)).as_posix()
     return None
 
 

siliconcompiler/{sphinx_ext → schema/docs}/dynamicgen.py

@@ -17,9 +17,9 @@ import os
 import subprocess
 
 import siliconcompiler
-from siliconcompiler.sphinx_ext import sc_root as SC_ROOT
-from siliconcompiler.schema import Schema, utils
-from siliconcompiler.sphinx_ext.utils import (
+from siliconcompiler.schema import utils
+from siliconcompiler.schema.docs import sc_root as SC_ROOT
+from siliconcompiler.schema.docs.utils import (
     strong,
     code,
     para,
@@ -39,15 +39,12 @@ from siliconcompiler.sphinx_ext.utils import (
 #############
 
 
-def build_schema_value_table(cfg, refdoc, keypath_prefix=None, skip_zero_weight=False):
+def build_schema_value_table(params, refdoc, keypath_prefix=None):
     '''Helper function for displaying values set in schema as a docutils table.'''
     table = [[strong('Keypath'), strong('Value')]]
 
-    # Nest received dictionary under keypath_prefix
-    rooted_cfg = cfg
-    if keypath_prefix:
-        for key in reversed(keypath_prefix):
-            rooted_cfg = {key: rooted_cfg}
+    if not keypath_prefix:
+        keypath_prefix = []
 
     def format_value(is_list, value):
         if is_list:
@@ -81,33 +78,27 @@ def build_schema_value_table(cfg, refdoc, keypath_prefix=None, skip_zero_weight=
             val_node = format_single_value_file(value, package)
         return val_node
 
-    schema = Schema(rooted_cfg)
-    for kp in schema.allkeys():
-        if skip_zero_weight and \
-           len(kp) == 6 and kp[0] == 'flowgraph' and kp[-2] == 'weight' and \
-           schema.get(*kp) == 0:
-            continue
-
-        values = schema._getvals(*kp, return_defvalue=False)
+    for key, param in sorted(params.items(), key=lambda d: d[0]):
+        values = param.getvalues(return_defvalue=False)
         if values:
             # take first of multiple possible values
             value, step, index = values[0]
-            val_type = schema.get(*kp, field='type')
+            val_type = param.get(field='type')
             is_filedir = 'file' in val_type or 'dir' in val_type
             # Don't display false booleans
             if val_type == 'bool' and value is False:
                 continue
             if is_filedir:
                 val_node = format_value_file(val_type.startswith('['), value,
-                                             schema.get(*kp, field='package',
-                                                        step=step, index=index))
+                                             param.get(field='package',
+                                                       step=step, index=index))
             else:
                 val_node = format_value(val_type.startswith('['), value)
 
             # HTML builder fails if we don't make a text node the parent of the
             # reference node returned by keypath()
             p = nodes.paragraph()
-            p += keypath(kp, refdoc)
+            p += keypath([*keypath_prefix, *key], refdoc)
             table.append([p, val_node])
 
     if len(table) > 1:
@@ -121,21 +112,21 @@ def build_schema_value_table(cfg, refdoc, keypath_prefix=None, skip_zero_weight=
 
 
 def build_package_table(schema):
-    def collect_packages(cfg):
+    def collect_packages():
         packages = []
-        if Schema._is_leaf(cfg):
-            if 'dir' in cfg['type'] or 'file' in cfg['type']:
-                for _, index_data in cfg['node'].items():
+        for key in schema.allkeys(include_default=True):
+            param = schema.get(*key, field=None)
+            if 'dir' in param.get(field='type') or 'file' in param.get(field='type'):
+                for _, index_data in param.getdict()["node"].items():
                     for _, data in index_data.items():
+                        if not data['package']:
+                            continue
+                        if isinstance(data['package'], str):
+                            data['package'] = [data['package']]
                         packages.extend(data['package'])
-        else:
-            for key in cfg:
-                packages.extend(collect_packages(cfg[key]))
-        packages = [p for p in packages if p]
-        return list(set(packages))
+        return list(set([p for p in packages if p]))
 
-    schema = Schema(cfg=schema)
-    packages = collect_packages(schema.cfg)
+    packages = collect_packages()
 
     if not packages:
         return None
@@ -281,7 +272,7 @@ class DynamicGen(SphinxDirective):
         nested_parse_with_titles(self.state, rst, s)
 
     def package_information(self, chip, modname):
-        packages = build_package_table(chip.schema.cfg)
+        packages = build_package_table(chip.schema)
         if packages:
             sec = build_section('Data sources', self.get_data_source_ref_key(modname, chip.design))
             sec += packages
@@ -435,12 +426,11 @@ class DynamicGen(SphinxDirective):
         if sec_key_prefix is None:
             sec_key_prefix = []
 
-        leaves = {}
+        params = {}
         child_sections = []
         for key in schema.getkeys(*keypath):
-            if Schema._is_leaf(schema.getdict(*keypath, key)):
-                val = schema.getdict(*keypath, key)
-                leaves.update({key: val})
+            if schema.valid(*keypath, key, check_complete=True):
+                params[(key,)] = schema.get(*keypath, key, field=None)
             else:
                 children = self.build_config_recursive(
                     schema,
@@ -450,9 +440,9 @@ class DynamicGen(SphinxDirective):
                 child_sections.extend(children)
 
         schema_table = None
-        if len(leaves) > 0:
+        if len(params) > 0:
             # Might return None is none of the leaves are displayable
-            schema_table = build_schema_value_table(leaves, refdoc, keypath_prefix=keypath)
+            schema_table = build_schema_value_table(params, refdoc, keypath_prefix=keypath)
 
         if schema_table is not None:
             # If we've found leaves, create a new section where we'll display a
@@ -493,27 +483,22 @@ class FlowGen(DynamicGen):
         settings = build_section('Configuration', self.get_configuration_ref_key(name))
 
         steps = chip.getkeys('flowgraph', chip.design)
+        schema = chip.schema
         # TODO: should try to order?
 
         # Build section + table for each step (combining entries under flowgraph
         # and metric)
         for step in steps:
             section = build_section(step, self.get_ref(name, 'step', step))
-            step_cfg = {}
-            cfg = chip.getdict('flowgraph', chip.design, step)
-            if cfg is None:
-                continue
-            schema = Schema(cfg=cfg)
-            schema.prune()
-            pruned = schema.cfg
-            if chip.design not in step_cfg:
-                step_cfg[chip.design] = {}
-            step_cfg[chip.design][step] = pruned
-
-            section += build_schema_value_table(step_cfg,
+
+            params = {}
+            for item in schema.allkeys('flowgraph', chip.design, step):
+                if item[1] == 'weight' and not schema.get('flowgraph', chip.design, step, *item):
+                    continue
+                params[item] = schema.get('flowgraph', chip.design, step, *item, field=None)
+            section += build_schema_value_table(params,
                                                 self.env.docname,
-                                                keypath_prefix=['flowgraph'],
-                                                skip_zero_weight=True)
+                                                keypath_prefix=['flowgraph', chip.design, step])
             settings += section
 
         return settings
@@ -636,15 +621,12 @@ class ToolGen(DynamicGen):
 
     def display_config(self, chip, modname):
         '''Display config under `eda, <modname>` in a single table.'''
-        cfg = chip.getdict('tool', modname)
-        schema = Schema(cfg=cfg)
-        schema.prune()
-        pruned = schema.cfg
-        if 'task' in pruned:
-            # Remove task specific items since they will be documented
-            # by the task documentation
-            del pruned['task']
-        table = build_schema_value_table(pruned, self.env.docname, keypath_prefix=['tool', modname])
+        params = {}
+        for key in chip.schema.allkeys('tool', modname):
+            if key == 'task':
+                continue
+            params[key] = chip.schema.get('tool', modname, *key, field=None)
+        table = build_schema_value_table(params, self.env.docname, keypath_prefix=['tool', modname])
         if table is not None:
             return table
         else:
@@ -652,14 +634,13 @@ class ToolGen(DynamicGen):
 
     def task_display_config(self, chip, toolname, taskname):
         '''Display config under `eda, <modname>` in a single table.'''
-        cfg = chip.getdict('tool', toolname, 'task', taskname)
-        schema = Schema(cfg=cfg)
-        for vals, step, index in schema._getvals('require'):
-            schema.set('require', sorted(set(vals)),
-                       step=step, index=index)
-        schema.prune()
-        pruned = schema.cfg
-        table = build_schema_value_table(pruned, self.env.docname,
+        params = {}
+        for key in chip.schema.allkeys('tool', toolname, 'task', taskname):
+            params[key] = chip.schema.get('tool', toolname, 'task', taskname, *key, field=None)
+            if key[0] == "require":
+                for vals, step, index in params[key].getvalues():
+                    params[key].set(sorted(set(vals)), step=step, index=index)
+        table = build_schema_value_table(params, self.env.docname,
                                          keypath_prefix=['tool', toolname, 'task', taskname])
         if table is not None:
             return table
@@ -731,7 +712,7 @@ class ToolGen(DynamicGen):
                                       s)
         except Exception as e:
             print(f'Failed to document task, Chip object probably not configured correctly: {e}')
-            return None
+            raise e
 
         return s
 
@@ -803,7 +784,7 @@ class TargetGen(DynamicGen):
         if len(modules) > 0:
             section = build_section(header, self.get_ref(targetname, modtype))
             modlist = nodes.bullet_list()
-            for module in modules:
+            for module in sorted(modules):
                 list_item = nodes.list_item()
                 # TODO: replace with proper docutils nodes: sphinx.addnodes.pending_xref
                 modkey = get_ref_id(DynamicGen.get_ref_key(*refprefix, module))
@@ -839,14 +820,12 @@ class TargetGen(DynamicGen):
 
         filtered_cfg = {}
         for key in ('asic', 'constraint', 'option'):
-            filtered_cfg[key] = chip.getdict(key)
-        schema = Schema(cfg=filtered_cfg)
-        schema.prune()
-        pruned_cfg = schema.cfg
+            for subkey in chip.schema.allkeys(key):
+                filtered_cfg[(key, *subkey)] = chip.schema.get(key, *subkey, field=None)
 
-        if len(pruned_cfg) > 0:
+        if filtered_cfg:
             schema_section = build_section('Configuration', self.get_configuration_ref_key(modname))
-            schema_section += build_schema_value_table(pruned_cfg, self.env.docname)
+            schema_section += build_schema_value_table(filtered_cfg, self.env.docname)
             sections.append(schema_section)
 
         return sections
@@ -892,15 +871,18 @@ class ChecklistGen(DynamicGen):
         name = chip.design
 
         keypath_prefix = ['checklist', name]
-        cfg = chip.getdict(*keypath_prefix)
+        schema = chip.schema
 
         settings = build_section('Configuration', self.get_configuration_ref_key(name))
 
-        for key in cfg.keys():
+        for key in schema.getkeys(*keypath_prefix):
             if key == 'default':
                 continue
             settings += build_section(key, self.get_ref(name, 'key', key))
-            settings += build_schema_value_table(cfg[key], self.env.docname,
+            params = {}
+            for item in schema.allkeys(*keypath_prefix, key):
+                params[item] = schema.get(*keypath_prefix, key, *item, field=None)
+            settings += build_schema_value_table(params, self.env.docname,
                                                  keypath_prefix=[*keypath_prefix, key])
 
         sections.append(settings)