siliconcompiler 0.33.2__py3-none-any.whl → 0.34.1__py3-none-any.whl

siliconcompiler/core.py

@@ -1,5 +1,6 @@
 # Copyright 2020 Silicon Compiler Authors. All Rights Reserved.
 
+import copy
 import tarfile
 import os
 import pathlib
@@ -13,20 +14,16 @@ import shutil
 import importlib
 import textwrap
 import graphviz
-import codecs
 import csv
 import yaml
 from inspect import getfullargspec
 from siliconcompiler import Schema
-from siliconcompiler.schema import SCHEMA_VERSION, PerNode, JournalingSchema, EditableSchema
+from siliconcompiler.schema import SCHEMA_VERSION, PerNode, Journal, EditableSchema
 from siliconcompiler.schema.parametertype import NodeType
-from siliconcompiler.schema.parametervalue import FileNodeValue, PathNodeValue
+from siliconcompiler.schema.parametervalue import FileNodeValue
 from siliconcompiler.schema import utils as schema_utils
 from siliconcompiler import utils
-from siliconcompiler.utils.logging import SCColorLoggerFormatter, \
-    SCLoggerFormatter, SCInRunLoggerFormatter, \
-    SCDebugLoggerFormatter, SCDebugInRunLoggerFormatter, \
-    SCBlankLoggerFormatter
+from siliconcompiler.utils.logging import get_console_formatter, SCLoggerFormatter
 from siliconcompiler import _metadata
 from siliconcompiler import NodeStatus, SiliconCompilerError
 from siliconcompiler.report import _show_summary_table
@@ -34,9 +31,8 @@ from siliconcompiler.report import _generate_summary_image, _open_summary_image
 from siliconcompiler.report.dashboard.web import WebDashboard
 from siliconcompiler.report.dashboard.cli import CliDashboard
 from siliconcompiler.report.dashboard import DashboardType
-from siliconcompiler import package as sc_package
 import glob
-from siliconcompiler.scheduler import run as sc_runner
+from siliconcompiler.scheduler.scheduler import Scheduler
 from siliconcompiler.utils.flowgraph import _check_flowgraph_io, _get_flowgraph_information
 from siliconcompiler.tools._common import get_tool_task
 from types import FunctionType, ModuleType
@@ -73,14 +69,13 @@ class Chip:
                 "SiliconCompiler must be run from a directory that exists. "
                 "If you are sure that your working directory is valid, try running `cd $(pwd)`.")
 
-        # Initialize custom error handling for codecs. This has to be called
-        # by each spawned (as opposed to forked) subprocess
-        self._init_codecs()
-
-        self._init_logger()
+        self.__init_logger()
 
         self.schema = Schema(logger=self.logger)
 
+        # Setup console formatting
+        self._logger_console.setFormatter(get_console_formatter(self, False, None, None))
+
         self.register_source('siliconcompiler',
                              'python://siliconcompiler')
 
@@ -206,96 +201,21 @@ class Chip:
         else:
             return module
 
-    def _add_file_logger(self, filename):
-        # Add a file handler for logging
-        file_handler = logging.FileHandler(filename)
-        self.logger.addHandler(file_handler)
-
-        self._init_logger_formats()
-
-        return file_handler
-
     ###########################################################################
-    def _init_logger(self, step=None, index=None, in_run=False):
-
+    def __init_logger(self):
         # Check if the logger exists and create
         if not hasattr(self, 'logger') or not self.logger:
             self.logger = logging.getLogger(f'sc_{id(self)}')
 
         self.logger.propagate = False
 
-        loglevel = 'info'
-        if hasattr(self, 'schema'):
-            loglevel = self.schema.get('option', 'loglevel', step=step, index=index)
-        else:
-            in_run = False
-
-        # Save in run flag
-        self.logger._in_run = in_run
-        self.logger._in_step = step
-        self.logger._in_index = index
-
-        self.logger.setLevel(schema_utils.translate_loglevel(loglevel))
-
-        if not self.logger.hasHandlers():
-            stream_handler = logging.StreamHandler(stream=sys.stdout)
-            # Save console handler
-            self.logger._console = stream_handler
-            self.logger.addHandler(stream_handler)
-
-            self.logger._support_color = SCColorLoggerFormatter.supports_color(stream_handler)
-
-        self._init_logger_formats(loglevel=loglevel)
-
-    def _init_logger_formats(self, loglevel=None):
-        if not loglevel:
-            loglevel = self.schema.get('option', 'loglevel',
-                                       step=self.logger._in_step, index=self.logger._in_index)
-
-        if loglevel == 'quiet':
-            base_format = SCBlankLoggerFormatter()
-        elif self.logger._in_run:
-            if loglevel == 'debug':
-                base_format = SCDebugInRunLoggerFormatter(
-                    self,
-                    self.get('option', 'jobname'),
-                    self.logger._in_step, self.logger._in_index)
-            else:
-                base_format = SCInRunLoggerFormatter(
-                    self,
-                    self.get('option', 'jobname'),
-                    self.logger._in_step, self.logger._in_index)
-        else:
-            if loglevel == 'debug':
-                base_format = SCDebugLoggerFormatter()
-            else:
-                base_format = SCLoggerFormatter()
-
-        for handler in self.logger.handlers.copy():
-            if handler == self.logger._console and self.logger._support_color:
-                formatter = SCColorLoggerFormatter(base_format)
-            else:
-                formatter = base_format
-            handler.setFormatter(formatter)
+        self.logger.setLevel(logging.INFO)
 
-    ###########################################################################
-    def _init_codecs(self):
-        # Custom error handlers used to provide warnings when invalid characters
-        # are encountered in a file for a given encoding. The names
-        # 'replace_with_warning' and 'ignore_with_warning' are supplied to
-        # open() via the 'errors' kwarg.
-
-        # Warning message/behavior for invalid characters while running tool
-        def display_error_handler(e):
-            self.logger.warning('Invalid character in tool output, displaying as �')
-            return codecs.replace_errors(e)
-        codecs.register_error('replace_with_warning', display_error_handler)
-
-        # Warning message/behavior for invalid characters while processing log
-        def log_error_handler(e):
-            self.logger.warning('Ignoring invalid character found while reading log')
-            return codecs.ignore_errors(e)
-        codecs.register_error('ignore_with_warning', log_error_handler)
+        stream_handler = logging.StreamHandler(stream=sys.stdout)
+        stream_handler.setFormatter(SCLoggerFormatter())
+        # Save console handler
+        self._logger_console = stream_handler
+        self.logger.addHandler(stream_handler)
 
     ###########################################################################
     def create_cmdline(self,
@@ -1102,7 +1022,8 @@ class Chip:
         '''
 
         if package:
-            filename = os.path.join(sc_package.path(self, package), filename)
+            resolvers = self.get("package", field="schema").get_resolvers()
+            filename = os.path.join(resolvers[package](), filename)
 
         if not os.path.isfile(filename):
             raise FileNotFoundError(filename)
@@ -1110,10 +1031,16 @@ class Chip:
         package_name = f'flist-{os.path.basename(filename)}'
         package_dir = os.path.dirname(os.path.abspath(filename))
 
-        env_vars = utils.get_env_vars(self, None, None)
-
         def __make_path(rel, path):
-            path = PathNodeValue.resolve_env_vars(path, envvars=env_vars)
+            env_save = os.environ.copy()
+            schema_env = {}
+            for env in self.getkeys('option', 'env'):
+                schema_env[env] = self.get('option', 'env', env)
+            os.environ.update(schema_env)
+            path = os.path.expandvars(path)
+            path = os.path.expanduser(path)
+            os.environ.clear()
+            os.environ.update(env_save)
             if os.path.isabs(path):
                 if path.startswith(rel):
                     return os.path.relpath(path, rel), package_name
@@ -1380,18 +1307,17 @@ class Chip:
         else:
             search_paths = [self.cwd]
 
-        env_vars = utils.get_env_vars(self, step, index)
+        resolvers = self.get("package", field="schema").get_resolvers()
         for (dependency, path) in zip(dependencies, paths):
             faux_param = FileNodeValue()
             faux_param.set(path)
             try:
                 if dependency:
                     faux_param.set(dependency, field='package')
-                    faux_search = [os.path.abspath(os.path.join(sc_package.path(self, dependency)))]
+                    faux_search = [resolvers[dependency]()]
                 else:
                     faux_search = search_paths
                 resolved = faux_param.resolve_path(
-                    envvars=env_vars,
                     search=faux_search,
                     collection_dir=collection_dir)
             except FileNotFoundError:
@@ -1547,39 +1473,19 @@ class Chip:
             True if all file paths are valid, otherwise False.
         '''
 
-        allkeys = self.allkeys()
-        error = False
-        for keypath in allkeys:
-            paramtype = self.get(*keypath, field='type')
-            is_file = 'file' in paramtype
-            is_dir = 'dir' in paramtype
-            is_list = paramtype.startswith('[')
-
-            if is_file or is_dir:
-                if keypath[-2:] == ('option', 'builddir'):
-                    # Skip ['option', 'builddir'] since it will get created by run() if it doesn't
-                    # exist
-                    continue
-
-                for check_files, step, index in self.schema.get(*keypath, field=None).getvalues():
-                    if not check_files:
-                        continue
+        ignore_keys = []
+        for keypath in self.allkeys():
+            if keypath[-2:] == ('option', 'builddir'):
+                ignore_keys.append(keypath)
 
-                    if not is_list:
-                        check_files = [check_files]
-
-                    for idx, check_file in enumerate(check_files):
-                        found_file = self.__find_files(*keypath,
-                                                       missing_ok=True,
-                                                       step=step, index=index,
-                                                       list_index=idx)
-                        if is_list:
-                            found_file = found_file[0]
-                        if not found_file:
-                            self.logger.error(f"Parameter {keypath} path {check_file} is invalid")
-                            error = True
+        package_map = self.get("package", field="schema").get_resolvers()
 
-        return not error
+        return self.schema.check_filepaths(
+            ignore_keys=ignore_keys,
+            logger=self.logger,
+            packages=package_map,
+            collection_dir=self._getcollectdir(),
+            cwd=self.cwd)
 
     ###########################################################################
     def check_manifest(self):
@@ -1646,7 +1552,7 @@ class Chip:
                         NodeStatus.SUCCESS:
                     # this task has already completed successfully, OK
                     continue
-                self.logger.error(f'{step}{index} relies on {in_step}{in_index}, '
+                self.logger.error(f'{step}/{index} relies on {in_step}/{in_index}, '
                                   'but this task has not been run and is not in the '
                                   'current nodes to execute.')
                 error = True
@@ -1691,12 +1597,12 @@ class Chip:
             if not self._get_tool_module(step, index, flow=flow, error=False):
                 error = True
                 self.logger.error(f"Tool module {tool_name} could not be found or "
-                                  f"loaded for {step}{index}.")
+                                  f"loaded for {step}/{index}.")
             if not self._get_task_module(step, index, flow=flow, error=False):
                 error = True
                 task_module = self.get('flowgraph', flow, step, index, 'taskmodule')
                 self.logger.error(f"Task module {task_module} for {tool_name}/{task_name} "
-                                  f"could not be found or loaded for {step}{index}.")
+                                  f"could not be found or loaded for {step}/{index}.")
 
         # 5. Check per tool parameter requirements (when tool exists)
         for (step, index) in nodes:
@@ -1796,10 +1702,7 @@ class Chip:
             schema.write_manifest(filepath)
             return
 
-        tcl_record = False
-        if isinstance(schema, JournalingSchema):
-            tcl_record = "get" in schema.get_journaling_types()
-            schema = schema.get_base_schema()
+        tcl_record = "get" in Journal.access(schema).get_types()
 
         is_csv = re.search(r'(\.csv)(\.gz)*$', filepath)
 
@@ -1929,156 +1832,13 @@ class Chip:
             >>> status = chip.check_checklist('iso9000', 'd000')
             Returns status.
         '''
-        error = False
-
-        self.logger.info(f'Checking checklist {standard}')
-
         if standard not in self.getkeys('checklist'):
             self.logger.error(f'{standard} has not been loaded.')
             return False
 
-        if items is None:
-            items = self.getkeys('checklist', standard)
-
-        # these tasks are recorded by SC so there are no reports
-        metrics_without_reports = (
-            'tasktime',
-            'totaltime',
-            'exetime',
-            'memory')
-
-        for item in items:
-            if item not in self.getkeys('checklist', standard):
-                self.logger.error(f'{item} is not a check in {standard}.')
-                error = True
-                continue
-
-            allow_missing_reports = True
-
-            has_check = False
-
-            all_criteria = self.get('checklist', standard, item, 'criteria')
-            for criteria in all_criteria:
-                m = re.match(r'^(\w+)\s*([\>\=\<]+)\s*([+\-]?\d+(\.\d+)?(e[+\-]?\d+)?)$',
-                             criteria.strip())
-                if not m:
-                    self.error(f"Illegal checklist criteria: {criteria}")
-                    return False
-                elif m.group(1) not in self.getkeys('metric'):
-                    self.error(f"Criteria must use legal metrics only: {criteria}")
-                    return False
-
-                metric = m.group(1)
-                op = m.group(2)
-                if self.get('metric', metric, field='type') == 'int':
-                    goal = int(m.group(3))
-                    number_format = 'd'
-                else:
-                    goal = float(m.group(3))
-
-                    if goal == 0.0 or (abs(goal) > 1e-3 and abs(goal) < 1e5):
-                        number_format = '.3f'
-                    else:
-                        number_format = '.3e'
-
-                if metric not in metrics_without_reports:
-                    allow_missing_reports = False
-
-                tasks = self.get('checklist', standard, item, 'task')
-                for job, step, index in tasks:
-                    if job not in self.getkeys('history'):
-                        self.error(f'{job} not found in history')
-
-                    flow = self.get('option', 'flow', job=job)
-
-                    if step not in self.getkeys('flowgraph', flow, job=job):
-                        self.error(f'{step} not found in flowgraph')
-
-                    if index not in self.getkeys('flowgraph', flow, step, job=job):
-                        self.error(f'{step}{index} not found in flowgraph')
-
-                    if self.get('record', 'status', step=step, index=index, job=job) == \
-                            NodeStatus.SKIPPED:
-                        if verbose:
-                            self.logger.warning(f'{step}{index} was skipped')
-                        continue
-
-                    has_check = True
-
-                    # Automated checks
-                    flow = self.get('option', 'flow', job=job)
-                    tool = self.get('flowgraph', flow, step, index, 'tool', job=job)
-                    task = self.get('flowgraph', flow, step, index, 'task', job=job)
-
-                    value = self.get('metric', metric, job=job, step=step, index=index)
-                    criteria_ok = utils.safecompare(self, value, op, goal)
-                    if metric in self.getkeys('checklist', standard, item, 'waiver'):
-                        waivers = self.get('checklist', standard, item, 'waiver', metric)
-                    else:
-                        waivers = []
-
-                    criteria_str = f'{metric}{op}{goal:{number_format}}'
-                    compare_str = f'{value:{number_format}}{op}{goal:{number_format}}'
-                    step_desc = f'job {job} with step {step}{index} and task {tool}/{task}'
-                    if not criteria_ok and waivers:
-                        self.logger.warning(f'{item} criteria {criteria_str} ({compare_str}) unmet '
-                                            f'by {step_desc}, but found waivers.')
-                    elif not criteria_ok:
-                        self.logger.error(f'{item} criteria {criteria_str} ({compare_str}) unmet '
-                                          f'by {step_desc}.')
-                        error = True
-                    elif verbose and criteria_ok:
-                        self.logger.info(f'{item} criteria {criteria_str} met by {step_desc}.')
-
-                    has_reports = \
-                        self.valid('tool', tool, 'task', task, 'report', metric, job=job) and \
-                        self.get('tool', tool, 'task', task, 'report', metric, job=job,
-                                 step=step, index=index)
-
-                    if allow_missing_reports and not has_reports:
-                        # No reports available and it is allowed
-                        continue
-
-                    reports = []
-                    try:
-                        if has_reports:
-                            reports = self.find_files('tool', tool, 'task', task, 'report', metric,
-                                                      job=job,
-                                                      step=step, index=index,
-                                                      missing_ok=not require_reports)
-                    except SiliconCompilerError:
-                        reports = []
-                        continue
-
-                    if require_reports and not reports:
-                        self.logger.error(f'No EDA reports generated for metric {metric} in '
-                                          f'{step_desc}')
-                        error = True
-
-                    for report in reports:
-                        if not report:
-                            continue
-
-                        report = os.path.relpath(report, self.cwd)
-                        if report not in self.get('checklist', standard, item, 'report'):
-                            self.add('checklist', standard, item, 'report', report)
-
-            if has_check:
-                if require_reports and \
-                        not allow_missing_reports and \
-                        not self.get('checklist', standard, item, 'report'):
-                    # TODO: validate that report exists?
-                    self.logger.error(f'No report documenting item {item}')
-                    error = True
-
-                if check_ok and not self.get('checklist', standard, item, 'ok'):
-                    self.logger.error(f"Item {item} 'ok' field not checked")
-                    error = True
-
-        if not error:
-            self.logger.info('Check succeeded!')
-
-        return not error
+        return self.get("checklist", standard, field="schema").check(
+            items=items, check_ok=check_ok, require_reports=require_reports
+        )
 
     ###########################################################################
     def __import_library(self, libname, library, job=None, clobber=True, keep_input=True):
@@ -2691,7 +2451,7 @@ class Chip:
     ###########################################################################
     def _archive_node(self, tar, step, index, include=None, verbose=True):
         if verbose:
-            self.logger.info(f'Archiving {step}{index}...')
+            self.logger.info(f'Archiving {step}/{index}...')
 
         basedir = self.getworkdir(step=step, index=index)
 
@@ -2700,7 +2460,7 @@ class Chip:
 
         if not os.path.isdir(basedir):
             if self.get('record', 'status', step=step, index=index) != NodeStatus.SKIPPED:
-                self.logger.error(f'Unable to archive {step}{index} due to missing node directory')
+                self.logger.error(f'Unable to archive {step}/{index} due to missing node directory')
             return
 
         if include:
@@ -2774,7 +2534,7 @@ class Chip:
 
         if not archive_name:
             if step and index:
-                archive_name = f"{design}_{jobname}_{step}{index}.tgz"
+                archive_name = f"{design}_{jobname}_{step}_{index}.tgz"
             elif step:
                 archive_name = f"{design}_{jobname}_{step}.tgz"
             else:
@@ -3201,9 +2961,14 @@ class Chip:
             >>> run()
             Runs the execution flow defined by the flowgraph dictionary.
         '''
+        from siliconcompiler.remote.client import ClientScheduler
 
         try:
-            sc_runner(self)
+            if self.get('option', 'remote'):
+                scheduler = ClientScheduler(self)
+            else:
+                scheduler = Scheduler(self)
+            scheduler.run()
         except Exception as e:
             if raise_exception:
                 raise e
@@ -3335,7 +3100,7 @@ class Chip:
         self.unset('option', 'prune')
         self.unset('option', 'from')
         # build new job name
-        self.set('option', 'jobname', f'_{taskname}_{sc_job}_{sc_step}{sc_index}', clobber=True)
+        self.set('option', 'jobname', f'_{taskname}_{sc_job}_{sc_step}_{sc_index}', clobber=True)
 
         # Setup in step/index variables
         for step, index in self.get("flowgraph", "showflow", field="schema").get_nodes():
@@ -3464,6 +3229,7 @@ class Chip:
         # We have to remove the chip's logger before serializing the object
         # since the logger object is not serializable.
         del attributes['logger']
+        del attributes['_logger_console']
         return attributes
 
     #######################################
@@ -3471,4 +3237,7 @@ class Chip:
         self.__dict__ = state
 
         # Reinitialize logger on restore
-        self._init_logger()
+        self.__init_logger()
+
+    def copy(self):
+        return copy.deepcopy(self)

siliconcompiler/data/templates/email/general.j2

@@ -9,11 +9,11 @@
     </head>
 
     <body>
-        <h2>Node Summary: "{{ step }}{{ index }}"</h2>
+        <h2>Node Summary: "{{ step }}/{{ index }}"</h2>
         <span>
             <p><b>Design</b>: {{ design }}</p>
             <p><b>Job</b>: {{ job }}</p>
-            <p><b>Node</b>: {{ step }}{{ index }}</p>
+            <p><b>Node</b>: {{ step }}/{{ index }}</p>
             <p><b>Status</b>: {{ status }}</p>
         </span>
         <span>
@@ -36,7 +36,7 @@
                     <th>Metrics</th>
                     <th>units</th>
                     {% for step, index in nodes %}
-                    <th align="center">{{ step }}{{ index }}</th>
+                    <th align="center">{{ step }}/{{ index }}</th>
                     {% endfor %}
                 </tr>
                 {% for metric in metric_keys %}

siliconcompiler/data/templates/email/summary.j2

@@ -14,7 +14,7 @@
                 <th>-</th>
                 <th>units</th>
                 {% for step, index in nodes %}
-                <th align="center">{{ step }}{{ index }}</th>
+                <th align="center">{{ step }}/{{ index }}</th>
                 {% endfor %}
             </tr>
             {% for metric in metric_keys %}

siliconcompiler/data/templates/issue/README.txt

@@ -13,7 +13,7 @@ Schema: {{ version['schema'] }}
 Testcase built: {{ date }}
 Tool: {{ run['tool'] }} {% if run['toolversion'] %}{{ run['toolversion'] }}{% endif %}
 Task: {{ run['task'] }}
-Node: {{ run['step'] }}{{ run['index'] }}
+Node: {{ run['step'] }}/{{ run['index'] }}
 
 ** Python **
 Version: {{ python['version'] }}

siliconcompiler/data/templates/report/sc_report.j2

@@ -271,12 +271,12 @@
         {% for metric in metric_keys %}
           {% for step, index in nodes %}
             {% if reports[step, index][metric] %}
-              var sim_log_btn = document.getElementById("{{ step }}{{ index }}_{{ metric }}_metlink");
+              var sim_log_btn = document.getElementById("{{ step }}_{{ index }}_{{ metric }}_metlink");
               sim_log_btn.addEventListener('click', () => {
                 log_link = "{{ step }}/{{ index }}/{{ reports[step, index][metric][0] }}";
                 window.open(log_link, "_blank");
               });
-              var sim_log_btn = document.getElementById("{{ step }}{{ index }}_{{ metric }}_ddmetlink");
+              var sim_log_btn = document.getElementById("{{ step }}_{{ index }}_{{ metric }}_ddmetlink");
               sim_log_btn.addEventListener('click', () => {
                 log_link = "{{ step }}/{{ index }}/{{ reports[step, index][metric][0] }}";
                 window.open(log_link, "_blank");
@@ -333,7 +333,7 @@
               <th>-</th>
               <th>units</th>
               {% for step, index in nodes %}
-                <th>{{ step }}{{ index }}</th>
+                <th>{{ step }}/{{ index }}</th>
               {% endfor %}
             </tr>
             {% for metric in metric_keys %}
@@ -372,12 +372,12 @@
         <h2>Metrics for {{ design }} Tasks</h2>
         {% for step, index in nodes %}
           <div>
-            <a class="btn btn-success" data-bs-toggle="collapse" href="#{{ step }}{{ index }}_dropdown_div", role="button", aria-controls="{{ step }}{{ index }}_dropdown_div">
-              Toggle {{ step }}{{ index }} Metrics
+            <a class="btn btn-success" data-bs-toggle="collapse" href="#{{ step }}_{{ index }}_dropdown_div", role="button", aria-controls="{{ step }}_{{ index }}_dropdown_div">
+              Toggle {{ step }}/{{ index }} Metrics
             </a>
           </div>
-          <div id="{{ step }}{{ index }}_dropdown_div" class="collapse">
-            <table id="{{ step }}{{ index }}_metrics_table" class="table table-dark table-striped table-bordered">
+          <div id="{{ step }}_{{ index }}_dropdown_div" class="collapse">
+            <table id="{{ step }}_{{ index }}_metrics_table" class="table table-dark table-striped table-bordered">
               <tr>
                 {% for metric in metric_keys %}
                   <th>{{ metric }}</th>