siliconcompiler 0.29.0__py3-none-any.whl → 0.29.2__py3-none-any.whl

siliconcompiler/optimizer/vizier.py

@@ -0,0 +1,259 @@
+import logging
+import uuid
+import math
+from siliconcompiler import Chip
+from siliconcompiler.optimizer import Optimizer
+from siliconcompiler.flowgraph import _get_flowgraph_nodes
+
+try:
+    from vizier.service import clients as vz_clients
+    from vizier.service import pyvizier as vz
+
+    from jax import config
+    config.update("jax_enable_x64", True)
+    _has_vizier = True
+
+    logging.getLogger('absl').setLevel(logging.CRITICAL)
+    logging.getLogger('jax').setLevel(logging.CRITICAL)
+except ModuleNotFoundError:
+    _has_vizier = False
+
+
+class VizierOptimizier(Optimizer):
+    def __init__(self, chip):
+        if not _has_vizier:
+            raise RuntimeError("vizier is not available")
+
+        super().__init__(chip)
+
+        self.__problem = None
+        self.__study = None
+
+        self.__owner = chip.design
+        self.__experiment_rounds = None
+        self.__parallel_experiment = None
+
+    def __init_parameters(self):
+        search_space = self.__problem.search_space.root
+        for name, info in self._parameters.items():
+            values = info["values"]
+            if info["type"] == 'float':
+                search_space.add_float_param(name, values[0], values[1])
+            elif info["type"] == 'int':
+                search_space.add_int_param(name, values[0], values[1])
+            elif info["type"] == 'bool':
+                search_space.add_discrete_param(name, values)
+            elif info["type"] == 'enum':
+                if any([isinstance(v, str) for v in values]):
+                    search_space.add_categorical_param(name, values)
+                else:
+                    search_space.add_discrete_param(name, values)
+
+    def __init_goals(self):
+        metric_information = self.__problem.metric_information
+
+        for name, info in self._goals.items():
+
+            vz_goal = None
+            if info["goal"] == 'max':
+                vz_goal = vz.ObjectiveMetricGoal.MAXIMIZE
+            elif info["goal"] == 'min':
+                vz_goal = vz.ObjectiveMetricGoal.MINIMIZE
+            else:
+                raise ValueError(f'{info["goal"]} is not a supported goal')
+
+            metric_information.append(vz.MetricInformation(name, goal=vz_goal))
+
+    def __init_study(self):
+        # Setup client and begin optimization
+        # Vizier Service will be implicitly created
+        study_config = vz.StudyConfig.from_problem(self.__problem)
+        study_config.algorithm = 'DEFAULT'
+        if len(self._goals) > 1:
+            if self.__parallel_experiment == 1:
+                study_config.algorithm = 'GAUSSIAN_PROCESS_BANDIT'
+            else:
+                study_config.algorithm = 'QUASI_RANDOM_SEARCH'
+
+        self.__study = vz_clients.Study.from_study_config(
+            study_config,
+            owner=self.__owner,
+            study_id=uuid.uuid4().hex)
+
+    def run(self, experiments=None, parallel=None):
+        if not experiments:
+            experiments = 10 * len(self._parameters)
+            self._chip.logger.debug(f'Setting number of optimizer experiments to {experiments}')
+
+        if not parallel:
+            parallel = 1
+
+        self.__parallel_experiment = parallel
+
+        # Algorithm, search space, and metrics.
+        self.__problem = vz.ProblemStatement()
+
+        self._clear_results()
+
+        self.__init_parameters()
+        self.__init_goals()
+
+        self.__init_study()
+
+        self.__experiment_rounds = int(math.ceil(float(experiments) / parallel))
+        accept = True
+        try:
+            for n in range(self.__experiment_rounds):
+                if self.__run_round(n):
+                    break
+        except KeyboardInterrupt:
+            pass
+        except Exception as e:
+            self._chip.logger.error(f"{e}")
+            accept = False
+        finally:
+            if accept:
+                self.__record_optimal()
+
+            self.__study.delete()
+            self.__study = None
+
+    def __run_round(self, experiment_round):
+        # create a new chip with a copy of its schema
+        chip = Chip(self._chip.design)
+        chip.schema = self._chip.schema.copy()
+
+        suggestions = self.__setup_round(experiment_round, chip)
+
+        # Start run
+        try:
+            chip.logger.info(
+                f"Starting optimizer run ({experiment_round+1} / {self.__experiment_rounds})")
+            chip.run()
+        except KeyboardInterrupt:
+            raise
+        except Exception as e:
+            chip.logger.error(f"{e}")
+
+        return self.__record_round(chip, suggestions)
+
+    def __setup_round(self, experiment_round, chip):
+        org_flow = self._chip.get("option", "flow")
+        org_jobname = self._chip.get("option", "jobname")
+
+        jobname = f"{org_jobname}-{org_flow}-{experiment_round+1}"
+
+        flow_map = {}
+
+        if self.__parallel_experiment > 1:
+            flow = f'optimize_{org_flow}'
+            # Create new graph
+            for m in range(self.__parallel_experiment):
+                graph_name = f'opt{m+1}'
+                flow_map[m] = {
+                    "name": f'{jobname}/{graph_name}',
+                    "prefix": f"{graph_name}.",
+                    "suggestion": None
+                }
+                chip.graph(flow, org_flow, name=graph_name)
+
+            # Complete nodes
+            nodes = _get_flowgraph_nodes(chip, org_flow)
+            for step, _ in list(nodes):
+                nodes.append((step, None))
+            nodes = set(nodes)
+
+            # Forward node specific values
+            for key in chip.schema.allkeys():
+                if key[0] == 'history':
+                    continue
+
+                for value, step, index in chip.schema._getvals(*key):
+                    node = (step, index)
+
+                    if node in nodes:
+                        for info in flow_map.values():
+                            chip.set(
+                                *key,
+                                value,
+                                step=f'{info["prefix"]}{step}',
+                                index=index)
+        else:
+            flow = org_flow
+            flow_map[0] = {
+                "name": jobname,
+                "prefix": "",
+                "suggestion": None
+            }
+
+        # Setup each experiment
+        for m, suggestion in enumerate(self.__study.suggest(count=self.__parallel_experiment)):
+            self._chip.logger.info(f'Setting parameters for {flow_map[m]["name"]}')
+            flow_map[m]["suggestion"] = suggestion
+
+            for param_name, param_value in suggestion.parameters.items():
+                self._set_parameter(
+                    param_name,
+                    param_value,
+                    chip,
+                    flow_prefix=flow_map[m]["prefix"])
+
+        chip.set('option', 'jobname', jobname)
+        chip.set('option', 'flow', flow)
+        chip.set('option', 'quiet', True)
+
+        steps = set()
+        for info in list(self._goals.values()) + list(self._assertions.values()):
+            for flow in flow_map.values():
+                steps.add(f'{flow["prefix"]}{info["step"]}')
+        chip.set('option', 'to', steps)
+
+        return flow_map
+
+    def __record_round(self, chip, suggestions):
+        jobname = chip.get('option', 'jobname')
+
+        # Record history
+        self._chip.schema.cfg['history'][jobname] = chip.schema.history(jobname).cfg
+
+        stop = False
+
+        for trial_entry in suggestions.values():
+            trial_suggestion = trial_entry['suggestion']
+
+            measurement = {}
+            self._chip.logger.info(f'Measuring {trial_entry["name"]}')
+            for meas_name, meas_entry in self._goals.items():
+                measurement[meas_name] = chip.get(
+                    *meas_entry["key"],
+                    step=f'{trial_entry["prefix"]}{meas_entry["step"]}',
+                    index=meas_entry["index"])
+
+                self._chip.logger.info(f'  Measured {meas_entry["print"]} = '
+                                       f'{measurement[meas_name]}')
+
+            failed = None
+            if any([value is None for value in measurement.values()]):
+                failed = "Did not record measurement goal"
+            elif not self._check_assertions(chip, trial_entry["prefix"]):
+                failed = "Failed to meet assertions"
+
+            if failed:
+                self._chip.logger.error(f'{trial_entry["name"]} failed: {failed}')
+                trial_suggestion.complete(vz.Measurement(),
+                                          infeasible_reason=failed)
+            else:
+                trial_suggestion.complete(vz.Measurement(measurement))
+                stop |= self._check_stop_goal(measurement)
+
+        return stop
+
+    def __record_optimal(self):
+        optimal_trials = list(self.__study.optimal_trials())
+        for n, optimal_trial in enumerate(optimal_trials):
+            optimal_trial = optimal_trial.materialize()
+
+            self._add_result(
+                optimal_trial.parameters,
+                optimal_trial.final_measurement
+            )

siliconcompiler/pdks/__init__.py

@@ -0,0 +1,5 @@
+def get_pdks():
+    '''
+    Returns a dict of builtin pdks
+    '''
+    return {}

siliconcompiler/scheduler/__init__.py

@@ -1,3 +1,4 @@
+import contextlib
 import copy
 import distro
 import getpass
@@ -617,17 +618,19 @@ def _copy_previous_steps_output_data(chip, step, index, replay):
                     os.rename(f'inputs/{outfile.name}', f'inputs/{new_name}')
 
 
-def __read_std_streams(chip, quiet, is_stdout_log, stdout_reader, is_stderr_log, stderr_reader):
+def __read_std_streams(chip, quiet,
+                       is_stdout_log, stdout_reader, stdout_print,
+                       is_stderr_log, stderr_reader, stderr_print):
     '''
     Handle directing tool outputs to logger
     '''
     if not quiet:
         if is_stdout_log:
             for line in stdout_reader.readlines():
-                chip.logger.info(line.rstrip())
+                stdout_print(line.rstrip())
         if is_stderr_log:
             for line in stderr_reader.readlines():
-                chip.logger.error(line.rstrip())
+                stderr_print(line.rstrip())
 
 
 ############################################################################
@@ -775,36 +778,89 @@ def _makecmd(chip, tool, task, step, index, script_name='replay.sh', include_pat
     return cmdlist, print_cmd, cmd, cmd_args
 
 
+def __get_stdio(chip, tool, task, flow, step, index):
+    def get_file(io_type):
+        suffix = chip.get('tool', tool, 'task', task, io_type, 'suffix',
+                          step=step, index=index)
+        destination = chip.get('tool', tool, 'task', task, io_type, 'destination',
+                               step=step, index=index)
+
+        io_file = None
+        if destination == 'log':
+            io_file = step + "." + suffix
+        elif destination == 'output':
+            io_file = os.path.join('outputs', chip.top() + "." + suffix)
+        elif destination == 'none':
+            io_file = os.devnull
+        else:
+            # This should not happen
+            chip.logger.error(f'{io_type}/destination has no support for {destination}.')
+            _haltstep(chip, flow, step, index)
+
+        return io_file
+
+    stdout_file = get_file('stdout')
+    stderr_file = get_file('stderr')
+
+    return stdout_file, stderr_file
+
+
 def _run_executable_or_builtin(chip, step, index, version, toolpath, workdir, run_func=None):
     '''
     Run executable (or copy inputs to outputs for builtin functions)
     '''
 
     flow = chip.get('option', 'flow')
-    top = chip.top()
     tool, task = get_tool_task(chip, step, index, flow)
 
     quiet = (
-        chip.get('option', 'quiet', step=step, index=index) and not
-        chip.get('option', 'breakpoint', step=step, index=index)
+        chip.get('option', 'quiet', step=step, index=index) and
+        not chip.get('option', 'breakpoint', step=step, index=index)
     )
 
+    stdout_print = chip.logger.info
+    stderr_print = chip.logger.error
+    if chip.get('option', 'loglevel', step=step, index=index) == "quiet":
+        stdout_print = chip.logger.error
+        stderr_print = chip.logger.error
+
     # TODO: Currently no memory usage tracking in breakpoints, builtins, or unexpected errors.
     max_mem_bytes = 0
 
+    stdout_file, stderr_file = __get_stdio(chip, tool, task, flow, step, index)
+    is_stdout_log = chip.get('tool', tool, 'task', task, 'stdout', 'destination',
+                             step=step, index=index) == 'log'
+    is_stderr_log = chip.get('tool', tool, 'task', task, 'stderr', 'destination',
+                             step=step, index=index) == 'log' and stderr_file != stdout_file
+
     retcode = 0
     cmdlist = []
     cmd_args = []
     if run_func:
         logfile = None
         try:
-            retcode = run_func(chip)
+            with open(stdout_file, 'w') as stdout_writer, \
+                    open(stderr_file, 'w') as stderr_writer:
+                if stderr_file == stdout_file:
+                    stderr_writer.close()
+                    stderr_writer = sys.stdout
+
+                with contextlib.redirect_stderr(stderr_writer), \
+                        contextlib.redirect_stdout(stdout_writer):
+                    retcode = run_func(chip)
         except Exception as e:
             chip.logger.error(f'Failed in run() for {tool}/{task}: {e}')
             retcode = 1  # default to non-zero
             print_traceback(chip, e)
             chip._error = True
         finally:
+            with sc_open(stdout_file) as stdout_reader, \
+                    sc_open(stderr_file) as stderr_reader:
+                __read_std_streams(chip,
+                                   quiet,
+                                   is_stdout_log, stdout_reader, stdout_print,
+                                   is_stderr_log, stderr_reader, stderr_print)
+
             try:
                 if resource:
                     # Since memory collection is not possible, collect the current process
@@ -843,48 +899,10 @@ def _run_executable_or_builtin(chip, step, index, version, toolpath, workdir, ru
                 import pty  # Note: this import throws exception on Windows
                 retcode = pty.spawn(cmdlist, read)
         else:
-            stdout_file = ''
-            stdout_suffix = chip.get('tool', tool, 'task', task, 'stdout', 'suffix',
-                                     step=step, index=index)
-            stdout_destination = chip.get('tool', tool, 'task', task, 'stdout', 'destination',
-                                          step=step, index=index)
-            if stdout_destination == 'log':
-                stdout_file = step + "." + stdout_suffix
-            elif stdout_destination == 'output':
-                stdout_file = os.path.join('outputs', top + "." + stdout_suffix)
-            elif stdout_destination == 'none':
-                stdout_file = os.devnull
-            else:
-                chip.logger.error(f'stdout/destination has no support for {stdout_destination}. '
-                                  'Use [log|output|none].')
-                _haltstep(chip, flow, step, index)
-
-            stderr_file = ''
-            stderr_suffix = chip.get('tool', tool, 'task', task, 'stderr', 'suffix',
-                                     step=step, index=index)
-            stderr_destination = chip.get('tool', tool, 'task', task, 'stderr', 'destination',
-                                          step=step, index=index)
-            if stderr_destination == 'log':
-                stderr_file = step + "." + stderr_suffix
-            elif stderr_destination == 'output':
-                stderr_file = os.path.join('outputs', top + "." + stderr_suffix)
-            elif stderr_destination == 'none':
-                stderr_file = os.devnull
-            else:
-                chip.logger.error(f'stderr/destination has no support for {stderr_destination}. '
-                                  'Use [log|output|none].')
-                _haltstep(chip, flow, step, index)
-
             with open(stdout_file, 'w') as stdout_writer, \
                     open(stdout_file, 'r', errors='replace_with_warning') as stdout_reader, \
                     open(stderr_file, 'w') as stderr_writer, \
                     open(stderr_file, 'r', errors='replace_with_warning') as stderr_reader:
-                # Use separate reader/writer file objects as hack to display
-                # live output in non-blocking way, so we can monitor the
-                # timeout. Based on https://stackoverflow.com/a/18422264.
-                is_stdout_log = chip.get('tool', tool, 'task', task, 'stdout', 'destination',
-                                         step=step, index=index) == 'log'
-                is_stderr_log = stderr_destination == 'log' and stderr_file != stdout_file
                 # if STDOUT and STDERR are to be redirected to the same file,
                 # use a single writer
                 if stderr_file == stdout_file:
@@ -942,8 +960,8 @@ def _run_executable_or_builtin(chip, step, index, version, toolpath, workdir, ru
                         # Loop until process terminates
                         __read_std_streams(chip,
                                            quiet,
-                                           is_stdout_log, stdout_reader,
-                                           is_stderr_log, stderr_reader)
+                                           is_stdout_log, stdout_reader, stdout_print,
+                                           is_stderr_log, stderr_reader, stderr_print)
 
                         if timeout is not None and time.time() - cmd_start_time > timeout:
                             chip.logger.error(f'Step timed out after {timeout} seconds')
@@ -961,8 +979,8 @@ def _run_executable_or_builtin(chip, step, index, version, toolpath, workdir, ru
                 # Read the remaining
                 __read_std_streams(chip,
                                    quiet,
-                                   is_stdout_log, stdout_reader,
-                                   is_stderr_log, stderr_reader)
+                                   is_stdout_log, stdout_reader, stdout_print,
+                                   is_stderr_log, stderr_reader, stderr_print)
                 retcode = proc.returncode
 
     chip.set('record', 'toolexitcode', retcode, step=step, index=index)

siliconcompiler/scheduler/send_messages.py

@@ -12,7 +12,6 @@ import fastjsonschema
 from pathlib import Path
 from siliconcompiler.flowgraph import get_executed_nodes
 import uuid
-from siliconcompiler.targets import freepdk45_demo
 
 
 # Compile validation code for API request bodies.
@@ -176,6 +175,7 @@ def send(chip, msg_type, step, index):
 
 if __name__ == "__main__":
     from siliconcompiler import Chip
+    from siliconcompiler.targets import freepdk45_demo
     chip = Chip('test')
     chip.use(freepdk45_demo)
     chip.set('option', 'scheduler', 'msgevent', 'ALL')

siliconcompiler/schema/schema_cfg.py

@@ -10,7 +10,7 @@ try:
 except ImportError:
     from siliconcompiler.schema.utils import trim
 
-SCHEMA_VERSION = '0.48.5'
+SCHEMA_VERSION = '0.48.6'
 
 #############################################################################
 # PARAM DEFINITION
@@ -2772,7 +2772,7 @@ def schema_option(cfg):
 
     scparam(cfg, ['option', 'loglevel'],
             sctype='enum',
-            enum=["info", "warning", "error", "critical", "debug"],
+            enum=["info", "warning", "error", "critical", "debug", "quiet"],
             pernode='optional',
             scope='job',
             defvalue='info',

siliconcompiler/schema/schema_obj.py

@@ -1511,7 +1511,7 @@ class Schema:
             switchstrs, metavar = self.__get_switches(schema, *keypath)
 
             # Three switch types (bool, list, scalar)
-            if not switchlist or any(switch in switchlist for switch in switchstrs):
+            if switchlist is None or any(switch in switchlist for switch in switchstrs):
                 used_switches.update(switchstrs)
                 if typestr == 'bool':
                     # Boolean type arguments
@@ -1606,7 +1606,18 @@ class Schema:
         # Grab argument from pre-process sysargs
         cmdargs = vars(parser.parse_args(scargs))
 
-        if print_banner:
+        # Set loglevel if set at command line
+        do_print_banner = True
+        if 'option_loglevel' in cmdargs.keys():
+            log_level = cmdargs['option_loglevel']
+            if isinstance(log_level, list):
+                # if multiple found, pick the first one
+                log_level = log_level[0]
+            if log_level == 'quiet':
+                do_print_banner = False
+            logger.setLevel(translate_loglevel(log_level).split()[-1])
+
+        if print_banner and do_print_banner:
             print_banner()
 
         extra_params = None
@@ -1623,14 +1634,6 @@ class Schema:
                     # Remove from cmdargs
                     del cmdargs[arg]
 
-        # Set loglevel if set at command line
-        if 'option_loglevel' in cmdargs.keys():
-            log_level = cmdargs['option_loglevel']
-            if isinstance(log_level, list):
-                # if multiple found, pick the first one
-                log_level = log_level[0]
-            logger.setLevel(translate_loglevel(log_level).split()[-1])
-
         # Read in all cfg files
         if 'option_cfg' in cmdargs.keys():
             for item in cmdargs['option_cfg']:

siliconcompiler/schema/utils.py

@@ -90,4 +90,6 @@ def trim(docstring):
 
 
 def translate_loglevel(level):
+    if level == "quiet":
+        level = "error"
     return level.upper()

siliconcompiler/sphinx_ext/__init__.py

@@ -0,0 +1,85 @@
+import os.path
+
+import siliconcompiler
+from siliconcompiler import __version__ as sc_version
+
+from siliconcompiler.targets import get_targets
+from siliconcompiler.flows import get_flows
+from siliconcompiler.checklists import get_checklists
+from siliconcompiler.libs import get_libs
+from siliconcompiler.pdks import get_pdks
+from siliconcompiler.apps import get_apps
+from siliconcompiler.tools import get_tools
+
+
+sc_root = os.path.dirname(os.path.dirname(os.path.abspath(siliconcompiler.__file__)))
+
+
+def relpath(file):
+    file = os.path.abspath(file)
+    if file.startswith(sc_root):
+        return os.path.relpath(file, sc_root)
+    return None
+
+
+def get_codeurl(file=None):
+    base_url = f"https://github.com/siliconcompiler/siliconcompiler/blob/v{sc_version}"
+
+    if not file:
+        return base_url
+
+    if os.path.isabs(file):
+        file = relpath(file)
+        if not file:
+            return None
+
+    return f"{base_url}/{file}"
+
+
+def targets():
+    modules = []
+    for name, mod in get_targets().items():
+        modules.append((mod, name))
+    return modules
+
+
+def flows():
+    modules = []
+    for name, mod in get_flows().items():
+        modules.append((mod, name))
+    return modules
+
+
+def libraries():
+    modules = []
+    for name, mod in get_libs().items():
+        modules.append((mod, name))
+    return modules
+
+
+def pdks():
+    modules = []
+    for name, mod in get_pdks().items():
+        modules.append((mod, name))
+    return modules
+
+
+def tools():
+    modules = []
+    for name, mod in get_tools().items():
+        modules.append((mod, name))
+    return modules
+
+
+def apps():
+    modules = []
+    for name, mod in get_apps().items():
+        modules.append((mod, name))
+    return modules
+
+
+def checklists():
+    modules = []
+    for name, mod in get_checklists().items():
+        modules.append((mod, name))
+    return modules