siliconcompiler 0.34.0__py3-none-any.whl → 0.34.2__py3-none-any.whl

siliconcompiler/core.py

@@ -1,5 +1,6 @@
 # Copyright 2020 Silicon Compiler Authors. All Rights Reserved.
 
+import copy
 import tarfile
 import os
 import pathlib
@@ -13,7 +14,6 @@ import shutil
 import importlib
 import textwrap
 import graphviz
-import codecs
 import csv
 import yaml
 from inspect import getfullargspec
@@ -23,10 +23,7 @@ from siliconcompiler.schema.parametertype import NodeType
 from siliconcompiler.schema.parametervalue import FileNodeValue
 from siliconcompiler.schema import utils as schema_utils
 from siliconcompiler import utils
-from siliconcompiler.utils.logging import SCColorLoggerFormatter, \
-    SCLoggerFormatter, SCInRunLoggerFormatter, \
-    SCDebugLoggerFormatter, SCDebugInRunLoggerFormatter, \
-    SCBlankLoggerFormatter
+from siliconcompiler.utils.logging import get_console_formatter, SCLoggerFormatter
 from siliconcompiler import _metadata
 from siliconcompiler import NodeStatus, SiliconCompilerError
 from siliconcompiler.report import _show_summary_table
@@ -40,6 +37,7 @@ from siliconcompiler.utils.flowgraph import _check_flowgraph_io, _get_flowgraph_
 from siliconcompiler.tools._common import get_tool_task
 from types import FunctionType, ModuleType
 from siliconcompiler.flowgraph import RuntimeFlowgraph
+from siliconcompiler.package import Resolver
 
 
 class Chip:
@@ -62,6 +60,8 @@ class Chip:
         self.scversion = _metadata.version
         self.schemaversion = SCHEMA_VERSION
 
+        Resolver.reset_cache(self)
+
         # Local variables
         self.scroot = os.path.dirname(os.path.abspath(__file__))
         self._error = False
@@ -72,14 +72,13 @@ class Chip:
                 "SiliconCompiler must be run from a directory that exists. "
                 "If you are sure that your working directory is valid, try running `cd $(pwd)`.")
 
-        # Initialize custom error handling for codecs. This has to be called
-        # by each spawned (as opposed to forked) subprocess
-        self._init_codecs()
-
-        self._init_logger()
+        self.__init_logger()
 
         self.schema = Schema(logger=self.logger)
 
+        # Setup console formatting
+        self._logger_console.setFormatter(get_console_formatter(self, False, None, None))
+
         self.register_source('siliconcompiler',
                              'python://siliconcompiler')
 
@@ -205,96 +204,21 @@ class Chip:
         else:
             return module
 
-    def _add_file_logger(self, filename):
-        # Add a file handler for logging
-        file_handler = logging.FileHandler(filename)
-        self.logger.addHandler(file_handler)
-
-        self._init_logger_formats()
-
-        return file_handler
-
     ###########################################################################
-    def _init_logger(self, step=None, index=None, in_run=False):
-
+    def __init_logger(self):
         # Check if the logger exists and create
         if not hasattr(self, 'logger') or not self.logger:
             self.logger = logging.getLogger(f'sc_{id(self)}')
 
         self.logger.propagate = False
 
-        loglevel = 'info'
-        if hasattr(self, 'schema'):
-            loglevel = self.schema.get('option', 'loglevel', step=step, index=index)
-        else:
-            in_run = False
-
-        # Save in run flag
-        self.logger._in_run = in_run
-        self.logger._in_step = step
-        self.logger._in_index = index
-
-        self.logger.setLevel(schema_utils.translate_loglevel(loglevel))
-
-        if not self.logger.hasHandlers():
-            stream_handler = logging.StreamHandler(stream=sys.stdout)
-            # Save console handler
-            self.logger._console = stream_handler
-            self.logger.addHandler(stream_handler)
-
-            self.logger._support_color = SCColorLoggerFormatter.supports_color(stream_handler)
-
-        self._init_logger_formats(loglevel=loglevel)
-
-    def _init_logger_formats(self, loglevel=None):
-        if not loglevel:
-            loglevel = self.schema.get('option', 'loglevel',
-                                       step=self.logger._in_step, index=self.logger._in_index)
-
-        if loglevel == 'quiet':
-            base_format = SCBlankLoggerFormatter()
-        elif self.logger._in_run:
-            if loglevel == 'debug':
-                base_format = SCDebugInRunLoggerFormatter(
-                    self,
-                    self.get('option', 'jobname'),
-                    self.logger._in_step, self.logger._in_index)
-            else:
-                base_format = SCInRunLoggerFormatter(
-                    self,
-                    self.get('option', 'jobname'),
-                    self.logger._in_step, self.logger._in_index)
-        else:
-            if loglevel == 'debug':
-                base_format = SCDebugLoggerFormatter()
-            else:
-                base_format = SCLoggerFormatter()
+        self.logger.setLevel(logging.INFO)
 
-        for handler in self.logger.handlers.copy():
-            if handler == self.logger._console and self.logger._support_color:
-                formatter = SCColorLoggerFormatter(base_format)
-            else:
-                formatter = base_format
-            handler.setFormatter(formatter)
-
-    ###########################################################################
-    def _init_codecs(self):
-        # Custom error handlers used to provide warnings when invalid characters
-        # are encountered in a file for a given encoding. The names
-        # 'replace_with_warning' and 'ignore_with_warning' are supplied to
-        # open() via the 'errors' kwarg.
-
-        # Warning message/behavior for invalid characters while running tool
-        def display_error_handler(e):
-            self.logger.warning('Invalid character in tool output, displaying as �')
-            return codecs.replace_errors(e)
-        codecs.register_error('replace_with_warning', display_error_handler)
-
-        # Warning message/behavior for invalid characters while processing log
-        def log_error_handler(e):
-            self.logger.warning('Ignoring invalid character found while reading log')
-            return codecs.ignore_errors(e)
-        codecs.register_error('ignore_with_warning', log_error_handler)
+        stream_handler = logging.StreamHandler(stream=sys.stdout)
+        stream_handler.setFormatter(SCLoggerFormatter())
+        # Save console handler
+        self._logger_console = stream_handler
+        self.logger.addHandler(stream_handler)
 
     ###########################################################################
     def create_cmdline(self,
@@ -1557,12 +1481,9 @@ class Chip:
             if keypath[-2:] == ('option', 'builddir'):
                 ignore_keys.append(keypath)
 
-        package_map = self.get("package", field="schema").get_resolvers()
-
         return self.schema.check_filepaths(
             ignore_keys=ignore_keys,
             logger=self.logger,
-            packages=package_map,
             collection_dir=self._getcollectdir(),
             cwd=self.cwd)
 
@@ -1631,7 +1552,7 @@ class Chip:
                         NodeStatus.SUCCESS:
                     # this task has already completed successfully, OK
                     continue
-                self.logger.error(f'{step}{index} relies on {in_step}{in_index}, '
+                self.logger.error(f'{step}/{index} relies on {in_step}/{in_index}, '
                                   'but this task has not been run and is not in the '
                                   'current nodes to execute.')
                 error = True
@@ -1676,12 +1597,12 @@ class Chip:
             if not self._get_tool_module(step, index, flow=flow, error=False):
                 error = True
                 self.logger.error(f"Tool module {tool_name} could not be found or "
-                                  f"loaded for {step}{index}.")
+                                  f"loaded for {step}/{index}.")
             if not self._get_task_module(step, index, flow=flow, error=False):
                 error = True
                 task_module = self.get('flowgraph', flow, step, index, 'taskmodule')
                 self.logger.error(f"Task module {task_module} for {tool_name}/{task_name} "
-                                  f"could not be found or loaded for {step}{index}.")
+                                  f"could not be found or loaded for {step}/{index}.")
 
         # 5. Check per tool parameter requirements (when tool exists)
         for (step, index) in nodes:
@@ -1911,156 +1832,13 @@ class Chip:
             >>> status = chip.check_checklist('iso9000', 'd000')
             Returns status.
         '''
-        error = False
-
-        self.logger.info(f'Checking checklist {standard}')
-
         if standard not in self.getkeys('checklist'):
             self.logger.error(f'{standard} has not been loaded.')
             return False
 
-        if items is None:
-            items = self.getkeys('checklist', standard)
-
-        # these tasks are recorded by SC so there are no reports
-        metrics_without_reports = (
-            'tasktime',
-            'totaltime',
-            'exetime',
-            'memory')
-
-        for item in items:
-            if item not in self.getkeys('checklist', standard):
-                self.logger.error(f'{item} is not a check in {standard}.')
-                error = True
-                continue
-
-            allow_missing_reports = True
-
-            has_check = False
-
-            all_criteria = self.get('checklist', standard, item, 'criteria')
-            for criteria in all_criteria:
-                m = re.match(r'^(\w+)\s*([\>\=\<]+)\s*([+\-]?\d+(\.\d+)?(e[+\-]?\d+)?)$',
-                             criteria.strip())
-                if not m:
-                    self.error(f"Illegal checklist criteria: {criteria}")
-                    return False
-                elif m.group(1) not in self.getkeys('metric'):
-                    self.error(f"Criteria must use legal metrics only: {criteria}")
-                    return False
-
-                metric = m.group(1)
-                op = m.group(2)
-                if self.get('metric', metric, field='type') == 'int':
-                    goal = int(m.group(3))
-                    number_format = 'd'
-                else:
-                    goal = float(m.group(3))
-
-                    if goal == 0.0 or (abs(goal) > 1e-3 and abs(goal) < 1e5):
-                        number_format = '.3f'
-                    else:
-                        number_format = '.3e'
-
-                if metric not in metrics_without_reports:
-                    allow_missing_reports = False
-
-                tasks = self.get('checklist', standard, item, 'task')
-                for job, step, index in tasks:
-                    if job not in self.getkeys('history'):
-                        self.error(f'{job} not found in history')
-
-                    flow = self.get('option', 'flow', job=job)
-
-                    if step not in self.getkeys('flowgraph', flow, job=job):
-                        self.error(f'{step} not found in flowgraph')
-
-                    if index not in self.getkeys('flowgraph', flow, step, job=job):
-                        self.error(f'{step}{index} not found in flowgraph')
-
-                    if self.get('record', 'status', step=step, index=index, job=job) == \
-                            NodeStatus.SKIPPED:
-                        if verbose:
-                            self.logger.warning(f'{step}{index} was skipped')
-                        continue
-
-                    has_check = True
-
-                    # Automated checks
-                    flow = self.get('option', 'flow', job=job)
-                    tool = self.get('flowgraph', flow, step, index, 'tool', job=job)
-                    task = self.get('flowgraph', flow, step, index, 'task', job=job)
-
-                    value = self.get('metric', metric, job=job, step=step, index=index)
-                    criteria_ok = utils.safecompare(self, value, op, goal)
-                    if metric in self.getkeys('checklist', standard, item, 'waiver'):
-                        waivers = self.get('checklist', standard, item, 'waiver', metric)
-                    else:
-                        waivers = []
-
-                    criteria_str = f'{metric}{op}{goal:{number_format}}'
-                    compare_str = f'{value:{number_format}}{op}{goal:{number_format}}'
-                    step_desc = f'job {job} with step {step}{index} and task {tool}/{task}'
-                    if not criteria_ok and waivers:
-                        self.logger.warning(f'{item} criteria {criteria_str} ({compare_str}) unmet '
-                                            f'by {step_desc}, but found waivers.')
-                    elif not criteria_ok:
-                        self.logger.error(f'{item} criteria {criteria_str} ({compare_str}) unmet '
-                                          f'by {step_desc}.')
-                        error = True
-                    elif verbose and criteria_ok:
-                        self.logger.info(f'{item} criteria {criteria_str} met by {step_desc}.')
-
-                    has_reports = \
-                        self.valid('tool', tool, 'task', task, 'report', metric, job=job) and \
-                        self.get('tool', tool, 'task', task, 'report', metric, job=job,
-                                 step=step, index=index)
-
-                    if allow_missing_reports and not has_reports:
-                        # No reports available and it is allowed
-                        continue
-
-                    reports = []
-                    try:
-                        if has_reports:
-                            reports = self.find_files('tool', tool, 'task', task, 'report', metric,
-                                                      job=job,
-                                                      step=step, index=index,
-                                                      missing_ok=not require_reports)
-                    except SiliconCompilerError:
-                        reports = []
-                        continue
-
-                    if require_reports and not reports:
-                        self.logger.error(f'No EDA reports generated for metric {metric} in '
-                                          f'{step_desc}')
-                        error = True
-
-                    for report in reports:
-                        if not report:
-                            continue
-
-                        report = os.path.relpath(report, self.cwd)
-                        if report not in self.get('checklist', standard, item, 'report'):
-                            self.add('checklist', standard, item, 'report', report)
-
-            if has_check:
-                if require_reports and \
-                        not allow_missing_reports and \
-                        not self.get('checklist', standard, item, 'report'):
-                    # TODO: validate that report exists?
-                    self.logger.error(f'No report documenting item {item}')
-                    error = True
-
-                if check_ok and not self.get('checklist', standard, item, 'ok'):
-                    self.logger.error(f"Item {item} 'ok' field not checked")
-                    error = True
-
-        if not error:
-            self.logger.info('Check succeeded!')
-
-        return not error
+        return self.get("checklist", standard, field="schema").check(
+            items=items, check_ok=check_ok, require_reports=require_reports
+        )
 
     ###########################################################################
     def __import_library(self, libname, library, job=None, clobber=True, keep_input=True):
@@ -2673,7 +2451,7 @@ class Chip:
     ###########################################################################
     def _archive_node(self, tar, step, index, include=None, verbose=True):
         if verbose:
-            self.logger.info(f'Archiving {step}{index}...')
+            self.logger.info(f'Archiving {step}/{index}...')
 
         basedir = self.getworkdir(step=step, index=index)
 
@@ -2682,7 +2460,7 @@ class Chip:
 
         if not os.path.isdir(basedir):
             if self.get('record', 'status', step=step, index=index) != NodeStatus.SKIPPED:
-                self.logger.error(f'Unable to archive {step}{index} due to missing node directory')
+                self.logger.error(f'Unable to archive {step}/{index} due to missing node directory')
             return
 
         if include:
@@ -2756,7 +2534,7 @@ class Chip:
 
         if not archive_name:
             if step and index:
-                archive_name = f"{design}_{jobname}_{step}{index}.tgz"
+                archive_name = f"{design}_{jobname}_{step}_{index}.tgz"
             elif step:
                 archive_name = f"{design}_{jobname}_{step}.tgz"
             else:
@@ -3322,7 +3100,7 @@ class Chip:
         self.unset('option', 'prune')
         self.unset('option', 'from')
         # build new job name
-        self.set('option', 'jobname', f'_{taskname}_{sc_job}_{sc_step}{sc_index}', clobber=True)
+        self.set('option', 'jobname', f'_{taskname}_{sc_job}_{sc_step}_{sc_index}', clobber=True)
 
         # Setup in step/index variables
         for step, index in self.get("flowgraph", "showflow", field="schema").get_nodes():
@@ -3451,6 +3229,7 @@ class Chip:
         # We have to remove the chip's logger before serializing the object
         # since the logger object is not serializable.
         del attributes['logger']
+        del attributes['_logger_console']
         return attributes
 
     #######################################
@@ -3458,4 +3237,7 @@ class Chip:
         self.__dict__ = state
 
         # Reinitialize logger on restore
-        self._init_logger()
+        self.__init_logger()
+
+    def copy(self):
+        return copy.deepcopy(self)

siliconcompiler/data/templates/email/general.j2

@@ -9,11 +9,11 @@
     </head>
 
     <body>
-        <h2>Node Summary: "{{ step }}{{ index }}"</h2>
+        <h2>Node Summary: "{{ step }}/{{ index }}"</h2>
         <span>
             <p><b>Design</b>: {{ design }}</p>
             <p><b>Job</b>: {{ job }}</p>
-            <p><b>Node</b>: {{ step }}{{ index }}</p>
+            <p><b>Node</b>: {{ step }}/{{ index }}</p>
             <p><b>Status</b>: {{ status }}</p>
         </span>
         <span>
@@ -36,7 +36,7 @@
                     <th>Metrics</th>
                     <th>units</th>
                     {% for step, index in nodes %}
-                    <th align="center">{{ step }}{{ index }}</th>
+                    <th align="center">{{ step }}/{{ index }}</th>
                     {% endfor %}
                 </tr>
                 {% for metric in metric_keys %}

siliconcompiler/data/templates/email/summary.j2

@@ -14,7 +14,7 @@
                 <th>-</th>
                 <th>units</th>
                 {% for step, index in nodes %}
-                <th align="center">{{ step }}{{ index }}</th>
+                <th align="center">{{ step }}/{{ index }}</th>
                 {% endfor %}
             </tr>
             {% for metric in metric_keys %}

siliconcompiler/data/templates/issue/README.txt

@@ -13,7 +13,7 @@ Schema: {{ version['schema'] }}
 Testcase built: {{ date }}
 Tool: {{ run['tool'] }} {% if run['toolversion'] %}{{ run['toolversion'] }}{% endif %}
 Task: {{ run['task'] }}
-Node: {{ run['step'] }}{{ run['index'] }}
+Node: {{ run['step'] }}/{{ run['index'] }}
 
 ** Python **
 Version: {{ python['version'] }}

siliconcompiler/data/templates/report/sc_report.j2

@@ -271,12 +271,12 @@
         {% for metric in metric_keys %}
           {% for step, index in nodes %}
             {% if reports[step, index][metric] %}
-              var sim_log_btn = document.getElementById("{{ step }}{{ index }}_{{ metric }}_metlink");
+              var sim_log_btn = document.getElementById("{{ step }}_{{ index }}_{{ metric }}_metlink");
               sim_log_btn.addEventListener('click', () => {
                 log_link = "{{ step }}/{{ index }}/{{ reports[step, index][metric][0] }}";
                 window.open(log_link, "_blank");
               });
-              var sim_log_btn = document.getElementById("{{ step }}{{ index }}_{{ metric }}_ddmetlink");
+              var sim_log_btn = document.getElementById("{{ step }}_{{ index }}_{{ metric }}_ddmetlink");
               sim_log_btn.addEventListener('click', () => {
                 log_link = "{{ step }}/{{ index }}/{{ reports[step, index][metric][0] }}";
                 window.open(log_link, "_blank");
@@ -333,7 +333,7 @@
               <th>-</th>
               <th>units</th>
               {% for step, index in nodes %}
-                <th>{{ step }}{{ index }}</th>
+                <th>{{ step }}/{{ index }}</th>
               {% endfor %}
             </tr>
             {% for metric in metric_keys %}
@@ -372,12 +372,12 @@
         <h2>Metrics for {{ design }} Tasks</h2>
         {% for step, index in nodes %}
           <div>
-            <a class="btn btn-success" data-bs-toggle="collapse" href="#{{ step }}{{ index }}_dropdown_div", role="button", aria-controls="{{ step }}{{ index }}_dropdown_div">
-              Toggle {{ step }}{{ index }} Metrics
+            <a class="btn btn-success" data-bs-toggle="collapse" href="#{{ step }}_{{ index }}_dropdown_div", role="button", aria-controls="{{ step }}_{{ index }}_dropdown_div">
+              Toggle {{ step }}/{{ index }} Metrics
             </a>
           </div>
-          <div id="{{ step }}{{ index }}_dropdown_div" class="collapse">
-            <table id="{{ step }}{{ index }}_metrics_table" class="table table-dark table-striped table-bordered">
+          <div id="{{ step }}_{{ index }}_dropdown_div" class="collapse">
+            <table id="{{ step }}_{{ index }}_metrics_table" class="table table-dark table-striped table-bordered">
               <tr>
                 {% for metric in metric_keys %}
                   <th>{{ metric }}</th>