opencos-eda 0.3.9__py3-none-any.whl → 0.3.11__py3-none-any.whl

opencos/commands/deps_help.py

@@ -10,32 +10,96 @@ it is generally run as simply
 uses no tools and will print a help text regarding DEPS markup files to stdout.
 '''
 
+# pylint: disable=line-too-long
 
 import os
+import re
 
 from opencos.eda_base import Command
 from opencos import util
-
-
-BASIC_DEPS_HELP = '''
-
---------------------------------------------------------------------
-
-  What is a DEPS.yml file and why does `eda` use this?
-  - DEPS.yml is a fancy filelist.
+from opencos.util import Colors
+
+def get_deps_md_file() -> str:
+    '''Tries to get docs/DEPS.md from our pypackage dist'''
+    opencos_dir, _ = os.path.split(util.__file__)
+
+    # Try to get it from site-packages dir, which should have docs/ alongside
+    # commands/ and tools/
+    filename = os.path.join(opencos_dir, 'docs', 'DEPS.md')
+    if os.path.isfile(filename):
+        return filename
+
+    # If you're running directly from the git checkout dir, you won't be getting
+    # this dist, so it's not in opencos/docs, it will simply be in ./docs
+    filename = os.path.join(opencos_dir, '..', 'docs', 'DEPS.md')
+    if os.path.isfile(filename):
+        return filename
+    return ''
+
+def get_deps_md_contents() -> str:
+    '''Tries to get the docs/DEPS.md file and returns the str contents
+
+    This also performs some limited colorization of markdown and YAML
+    (assuming util was not disabled with --no-color)
+    '''
+    filename = get_deps_md_file()
+    if not filename:
+        return ''
+
+    def make_byellow(match):
+        '''Used by re.sub to wrap the match with bold yellow and return to normal yellow'''
+        return f'{Colors.byellow}{match.group(0)}{Colors.normal}{Colors.yellow}'
+
+    lines = []
+    with open(filename, encoding='utf-8') as f:
+        for line in f.readlines():
+
+            if line.startswith('# '):
+                # colors for markdown headings
+                line = f'{Colors.bgreen}{line}{Colors.normal}{Colors.yellow}'
+            elif line.startswith('## '):
+                # colors for markdown headings
+                line = f'{Colors.bcyan}{line}{Colors.normal}{Colors.yellow}'
+
+            elif '#' in line:
+                # colors for comments
+                line = line.replace('#', f'{Colors.normal}{Colors.cyan}#') + Colors.yellow
+
+            # colors for starting a line with:
+            #  key: value
+            #  - key : value
+            # try to make "key:"  or "- key:" as bold:
+            line = re.sub(
+                r'^( *\-? ?[^ ]+):', make_byellow, line
+            )
+
+            lines.append(line)
+
+    return ''.join(lines)
+
+
+BASIC_DEPS_HELP = f'''
+{Colors.yellow}
+Note: you can run with one of: {Colors.cyan}--verbose, --help, --debug{Colors.yellow} to show full
+schema supported, or {Colors.cyan}--no-color{Colors.yellow} to avoid printing this text with colors.
+
+{Colors.green}--------------------------------------------------------------------{Colors.yellow}
+
+  What is a {Colors.byellow}DEPS.yml{Colors.normal}{Colors.yellow} file and why does `eda` use this?
+  - {Colors.byellow}DEPS.yml{Colors.normal}{Colors.yellow} is a fancy filelist.
   - Used to organize a project into "targets", a tool can run on a "target".
   - Allows for more than just source files attached to a "target".
     -- incdirs, defines, and args can be applied to a "target".
 
---------------------------------------------------------------------
+{Colors.green}--------------------------------------------------------------------{Colors.yellow}
 
   Hello World example:
 
-  The following example is a DEPS.yml file example for a SystemVerilog simulation of
-  hello_world_tb.sv. DEPS.yml is, in short, a fancy filelist. We use them in the `eda`
+  The following example is a {Colors.byellow}DEPS.yml{Colors.normal}{Colors.yellow} file example for a SystemVerilog simulation of
+  hello_world_tb.sv. {Colors.byellow}DEPS.yml{Colors.normal}{Colors.yellow} is, in short, a fancy filelist. We use them in the `eda`
   app to organize projects.
 
---- DEPS.yml: ---
+--- {Colors.byellow}DEPS.yml{Colors.normal}{Colors.yellow}: ---
 
 hello-world:           # <-- this is a named target that will be run
 
@@ -64,20 +128,20 @@ endmodule : hello_world_tb
 
 
   hello-world:
-    The target name in the DEPS.yml we named is hello-world. That is a valid target
+    The target name in the {Colors.byellow}DEPS.yml{Colors.normal}{Colors.yellow} we named is hello-world. That is a valid target
     that `eda` can use. Such as:
 
        eda sim --tool=verilator hello-world
 
 
---------------------------------------------------------------------
+{Colors.green}--------------------------------------------------------------------{Colors.yellow}
 
    Beyond Hello World example:
 
    The following example is a DEPS.yml file for a more complex module simulation.
    It has two files in ./DEPS.yml and ./lib/DEPS.yml.
 
---- ./DEPS.yml: ---
+--- {Colors.byellow}./DEPS.yml{Colors.normal}{Colors.yellow}: ---
 
 my_fifo:                         # <-- this is a design
   incdirs: . lib                 # <-- 'incdirs' define the paths searched to find `include files
@@ -102,7 +166,7 @@ my_fifo_stress_test:             # <-- this is another TEST
   deps:
     - my_fifo_test               # aside from the define, this is same as "my_fifo_test"
 
---- lib/DEPS.yml: ---
+--- {Colors.byellow}lib/DEPS.yml{Colors.normal}{Colors.yellow}: ---
 
 lib_pkg:                         # <-- this is a package required by bin_to_gray below
   deps:
@@ -115,113 +179,16 @@ bin_to_gray:                     # <-- this is the target that was required by .
                                  # to be read before the code that uses them
     - bin_to_gray.sv             # an SV module pulled in directly
 
---------------------------------------------------------------------
+{Colors.green}--------------------------------------------------------------------{Colors.yellow}
 '''
 
 
-FULL_DEPS_HELP = '''
-
---------------------------------------------------------------------
-
-   Full DEPS.yml schema:
-
-```
-DEFAULTS: # <table> defaults applied to ALL targets in this file, local targets ** override ** the defaults.
-
-METADATA: # <table> unstructured data, any UPPERCASE first level key is not considered a target.
-
-target-spec:
-
-  args: # <array or | separated str>
-    - --waves
-    - --sim_plusargs="+info=500"
-
-  defines: # <table>
-    SOME_DEFINE: value
-    SOME_DEFINE_NO_VALUE:   # we just leave this blank, or use nil (yaml's None)
-
-  incdirs: # <array>
-    - some/relative/path
-
-  top: # <string>
-
-  deps: # <array or | space separated string>
-    - some_relative_target       # <string> aka, a target
-    - some_file.sv               # <string> aka, a file
-    - sv@some_file.txt           # <string> aka, ext@file where we'd like a file not ending in .sv to be
-                                 # treated as a .sv file for tools.
-                                 # Supported for sv@, v@, vhdl@, cpp@
-    - commands:                  # <table> with key 'commands' for a <array>:  support for built-in commands
-                                 # Note this cannot be confused for other targets or files.
-      - shell: # <string>
-        var-subst-args: # <bool> default false. If true, substitute vars in commands, such as {fpga}
-                        # substituted from eda arg --fpga=SomeFpga, such that {fpga} becomes SomeFpga
-        var-subst-os-env:  #<bool> default false. If true, substitute vars in commands using os.environ vars,
-                           # such as {FPGA} could get substituted by env value for $FPGA
-        tee: # <string> optional filename, otherwise shell commands write to {target-spec}__shell_0.log
-        run-from-work-dir: #<bool> default true. If false, runs from the directory of this DEPS file.
-        filepath-subst-target-dir: #<bool> default true. If false, disables shell file path
-	                               substituion on this target's directory (this DEPS file dir).
-        dirpath-subst-target-dir: #<bool> default false. If true, enables shell directory path
-	                              substituion on this target's directory (this DEPS file dir).
-      - shell: echo "Hello World!"
-      - work-dir-add-sources: # <array or | space separated string>, this is how to add generated files
-                              # to compile order list.
-      - peakrdl:              # <string>     ## peakrdl command to generate CSRs
-
-  reqs: # <array or | space separated string>
-    - some_file.mem           # <string> aka, a non-source file required for this target.
-                              # This file is checked for existence prior to invoking the tool involved, for example,
-                              # in a simulation this would be done prior to a compile step.
-
-  multi:
-    ignore-this-target:  # <array of tables> eda commands to be ignored in `eda multi <command>` for this target only
-                         # this is checked in the matching multi targets list, and is not inherited through dependencies.
-      - commands: synth  # space separated strings
-        tools: vivado    # space separated strings
-
-      - commands: sim # omit tools, ignores 'sim' commands for all tools, for this target only, when this target
-                      # is in the target list called by `eda multi`.
-
-      - tools: vivado # omit commands, ignores all commands if tool is vivado, for this target only, when this target
-                      # is in the target list called by `eda multi`.
-
-    args: # <array> additional args added to all multi commands of this target.
-          # Note that all args are POSIX with dashes, --sim-plusargs=value, etc.
-
-  <eda-command>: # key is one of sim, flist, build, synth, etc.
-                 # can be used instead of 'tags' to support different args or deps.
-    disable-tools: # Note: not implemented yet.
-    only-tools:    # Note: not implemented yet.
-    args: # <array or | space separated string>
-    deps: # <array or | space separated string> # Note: not implemented yet
-    defines: ## <table>
-    incdirs: ## <array>
-
-  tags: # <table> this is the currently support tags features in a target.
-    <tag-name>: # <string> key for table, can be anything, name is not used.
-      with-tools: <array or | space separated string>
-                  # If using one of these tools, apply these values.
-                  # entries can be in the form: vivado, or vivado:2024.1
-      with-commands: <array or | space separated string>
-                  # apply if this was the `eda` command, such as: sim
-      with-args: # <table> (optional) arg key/value pairs to match for this tag.
-                 # this would be an alternative to running eda with --tags=value
-                 # The existence of an argument with correct value would enable a tag.
-                 # And example would be:
-                 #   with-args:
-                 #     waves: true
-      args: <array or | space separated string> # args to be applied if this target is used, with a matching
-                                          # tool in 'with-tools'.
-      deps: <array or | space separated string, applied with tag>
-      defines: <table, applied with tag>
-      incdirs: <array, applied with tag>
-      replace-config-tools: <table>  # spec matching eda_config_defaults.yml::tools.<tool> (replace merge strategy)
-      additive-config-tools: <table> # spec matching eda_config_defaults.yml::tools.<tool> (additive merge strategy)
-
-
-```
-'''
+FULL_DEPS_HELP = f'''
+
+{Colors.green}--------------------------------------------------------------------{Colors.yellow}
+
+''' + get_deps_md_contents()
+
 
 
 class CommandDepsHelp:
@@ -256,4 +223,6 @@ class CommandDepsHelp:
         '''
         Command.help(self, tokens=tokens, no_targets=True)
         print()
+        print(BASIC_DEPS_HELP)
+        print()
         print(FULL_DEPS_HELP)

opencos/commands/export.py

@@ -58,9 +58,14 @@ class CommandExport(CommandDesign):
 
     def do_it(self) -> None:
 
-        # decide output dir name
+        # decide output dir name, note this does not follow the work-dir naming of
+        # eda.work/{target}.{command}
         if not self.args['output']:
-            self.args['output'] = os.path.join('.', 'eda.export', self.args['top'] + '.export')
+            if self.target:
+                name = f'{self.target}.export'
+            else:
+                name = self.args.get('top', '') + '.export'
+            self.args['output'] = os.path.join('.', 'eda.export', name)
         out_dir = self.args['output']
 
         if not self.target:

opencos/commands/multi.py

@@ -6,14 +6,14 @@ These are not intended to be overriden by child classes. They do not inherit Too
 import argparse
 import glob
 import os
-import shutil
 from pathlib import Path
 
 from opencos import util, eda_base, eda_config, export_helper, \
     eda_tool_helper
-from opencos.eda_base import CommandParallel, get_eda_exec
 from opencos.deps.deps_file import get_deps_markup_file, deps_markup_safe_load, \
     deps_data_get_all_targets, deps_list_target_sanitize
+from opencos.eda_base import CommandParallel, get_eda_exec
+from opencos.files import safe_shutil_which
 from opencos.utils.str_helpers import fnmatch_or_re, dep_str2list
 
 class CommandMulti(CommandParallel):
@@ -425,7 +425,7 @@ class CommandMulti(CommandParallel):
         '''
         eda_path = get_eda_exec('multi')
         command = self.single_command
-        timeout = shutil.which('timeout')
+        timeout = safe_shutil_which('timeout')
 
         # Built-in support for running > 1 tool.
         all_multi_tools = self.multi_which_tools(command)

opencos/commands/sim.py

@@ -300,7 +300,6 @@ class CommandSim(CommandDesign): # pylint: disable=too-many-public-methods
         tool = self.args.get('tool', None)
         # Certain args are allow-listed here
         deps_file_args = []
-        print(f'SUPER DREW DEBUG: {self.get_command_line_args()=}')
         for a in self.get_command_line_args():
             if any(a.startswith(x) for x in [
                     '--compile-args',
@@ -371,7 +370,8 @@ class CommandSim(CommandDesign): # pylint: disable=too-many-public-methods
     def check_logs_for_errors( # pylint: disable=dangerous-default-value,too-many-locals,too-many-branches
             self,
             sim_retcode: int = 0,
-            filename: str = '', file_contents_str: str = '',
+            filename: str = '',
+            file_contents_str: str = '',
             bad_strings: list = [], must_strings: list = [],
             warning_strings: list = [],
             use_bad_strings: bool = True, use_must_strings: bool = True
@@ -405,32 +405,30 @@ class CommandSim(CommandDesign): # pylint: disable=too-many-public-methods
         lines = []
 
         log_fpath = ''
-        if os.path.exists(filename):
+        if os.path.isfile(filename):
             log_fpath = filename
 
         if file_contents_str:
             lines = file_contents_str.split('\n')
             log_fname = log_fpath + '(STDOUT)'
             util.debug(f'Checking log for errors: {log_fpath=} but checking from STDOUT string...')
-        elif filename:
+        elif log_fpath:
             log_fname = log_fpath
-            util.debug(f'Checking log for errors: {log_fpath=} opening file...')
-            if not os.path.exists(log_fname):
-                self.error(f'sim.check_logs_for_errors: log {log_fpath} does not exist, cannot',
-                           'check it for errors or passing strings')
-                return
+            util.info(f'Checking log filename: {log_fpath}')
             with open(log_fpath, 'r', encoding='utf-8') as f:
                 lines = f.read().splitlines()
         else:
-            self.error(f'sim.check_logs_for_errors: {log_fpath=} does not exist, and no',
-                       'file_contents_str exists to check')
+            log_fname = filename
+            self.error(f'sim.check_logs_for_errors: {log_fname=} is not a file or does not exist,',
+                       'and no file_contents_str exists to check')
 
         self.update_tool_warn_err_counts_from_log_lines(
             log_lines=lines, bad_strings=_bad_strings, warning_strings=_warning_strings
         )
 
-        if isinstance(self, Tool):
-            self.report_tool_warn_error_counts()
+        func = getattr(self, 'report_tool_warn_error_counts', None)
+        if func and isinstance(self, Tool) and callable(func):
+            self.report_tool_warn_error_counts() # pylint: disable=no-member
 
         if sim_retcode > 0:
             # We have to update artifacts first, have the caller set the error.
@@ -447,14 +445,14 @@ class CommandSim(CommandDesign): # pylint: disable=too-many-public-methods
                             hit_must_string_dict[k] = True
                 if any(bad_str in line for bad_str in _bad_strings):
                     self.error(
-                        f"log {log_fname}:{lineno} contains one of {_bad_strings=}",
+                        f"log {log_fname}:{lineno} contains one of: {_bad_strings}",
                         error_code=status_constants.EDA_SIM_LOG_HAS_BAD_STRING
                     )
 
         if any(x is None for x in hit_must_string_dict.values()):
             self.error(
-                f"Didn't get all passing patterns in log {log_fname}: {_must_strings=}",
-                f" {hit_must_string_dict=}",
+                f"Didn't get all passing patterns in log {log_fname}: {_must_strings},",
+                f" {hit_must_string_dict}",
                 error_code=status_constants.EDA_SIM_LOG_MISSING_MUST_STRING
             )
 

opencos/commands/synth.py

@@ -67,9 +67,8 @@ class CommandSynth(CommandDesign):
         if self.stop_process_tokens_before_do_it():
             return unparsed
 
-        # add defines for this job type
         if self.args['top']:
-            # create our work dir
+            # create our work dir (from self.args['top'])
             self.create_work_dir()
             self.run_dep_commands()
             self.do_it()

opencos/commands/upload.py

@@ -4,20 +4,73 @@ Intended to be overriden by Tool based classes (such as CommandUploadVivado, etc
 '''
 
 import os
+import re
+from datetime import datetime
+from pathlib import Path
 
 from opencos.eda_base import Command, Tool
+from opencos.util import Colors, debug, info, warning, safe_emoji, import_class_from_string
+
 
 class CommandUpload(Command):
-    '''Base class command handler for: eda upload ...'''
+    '''Base class command handler for: eda upload ...
 
-    CHECK_REQUIRES = [Tool]
+    If a --tool arg is not specified, this is the default handler for 'eda upload'
+    and will attempt to choose a derived class based on the bit files found
+    '''
 
     command_name = 'upload'
 
+    # SUPPORTED_TOOLS is used
+    SUPPORTED_TOOLS = {
+        'vivado': ['.bit'],
+        'quartus': ['.sof'],
+    }
+    BIT_EXT_TO_TOOL = {}
+
+    # Child classes can set SUPPORTED_BIT_EXT = ['.bit', ..] because they
+    # should only represent one tool
+    SUPPORTED_BIT_EXT = [item for value in SUPPORTED_TOOLS.values() for item in value]
+
+
     def __init__(self, config: dict):
         Command.__init__(self, config=config)
         self.unparsed_args = []
 
+        self.args.update({
+            'bitfile': "",
+            'list-bitfiles': False,
+        })
+
+        self.str_ext = '/'.join(self.SUPPORTED_BIT_EXT).replace('.', '').upper()
+
+        help_upload_tools = '|'.join(self.config.get('auto_tools_found', []))
+        if not help_upload_tools:
+            help_upload_tools = 'TOOL'
+
+        self.args_help.update({
+            'bitfile': (
+                f'Tool specific {self.str_ext} files to upload (auto-detected if not specified)'
+                ' If you would like see full help for a given tool, use:'
+                f' {Colors.yellow}eda upload --help'
+                f' {Colors.byellow}--tool={help_upload_tools}{Colors.green}'
+            ),
+            'list-bitfiles': (
+                f'List available {self.str_ext} files.'
+                ' If you would like see full help for a given tool, use:'
+                f' {Colors.yellow}eda upload --help'
+                f' {Colors.byellow}--tool={help_upload_tools}{Colors.green}'
+            )
+        })
+
+        self.bitfiles = []
+
+        if not getattr(self, '_TOOL', ''):
+            for tool, bit_exts in self.SUPPORTED_TOOLS.items():
+                for ext in bit_exts:
+                    self.BIT_EXT_TO_TOOL[ext] = tool
+
+
     def process_tokens(
             self, tokens: list, process_all: bool = True, pwd: str = os.getcwd()
     ) -> list:
@@ -25,9 +78,144 @@ class CommandUpload(Command):
         self.unparsed_args = Command.process_tokens(
             self, tokens=tokens, process_all=False, pwd=pwd
         )
+
         if self.stop_process_tokens_before_do_it():
             return []
 
-        self.create_work_dir()
-        self.do_it()
+        self.bitfiles = self.get_list_bitfiles(display=True)
+
+        # If someone called --list-bitfiles, stop now.
+        if self.args['list-bitfiles']:
+            if not self.bitfiles:
+                self.error('No bitfiles found')
+            return []
+
+        sco = self._get_child_handling_class()
+
+        if sco is None or not isinstance(sco, Tool):
+            self.error('Could not find a suitable tool to process bitfiles')
+            return []
+
+        sco.unparsed_args = Command.process_tokens(
+            sco, tokens=tokens, process_all=False, pwd=pwd
+        )
+        sco.bitfiles = self.bitfiles
+        sco.create_work_dir()
+        sco.do_it()
         return []
+
+    def get_targets_or_files_from_unparsed_args(self) -> (list, list):
+        '''Returns (list of targets, list of files) from unparsed args or --bitfile'''
+
+        targets = []
+        files = []
+        for f in self.unparsed_args + [self.args['bitfile']]:
+            if not f:
+                continue
+            if os.path.isfile(f):
+                files.append(f)
+            elif not f.startswith('-'):
+                # avoid a arg
+                targets.append(f)
+        return targets, files
+
+
+    def get_list_bitfiles(self, display: bool = True) -> list:
+        '''Returns a list of bit files (ending with self.SUPPORTED_BIT_EXT)'''
+
+        bitfiles: list[Path] = []
+
+        targets, files = self.get_targets_or_files_from_unparsed_args()
+        targets.extend(files)
+
+        debug(f"Looking for bitfiles in {os.path.abspath('.')=}")
+        for root, _, files in os.walk("."):
+            for f in files:
+                if any(f.endswith(x) for x in self.SUPPORTED_BIT_EXT):
+                    fullpath = os.path.abspath(Path(root) / f)
+                    if os.path.isfile(fullpath) and fullpath not in bitfiles:
+                        bitfiles.append(fullpath)
+
+        matched: list[Path] = []
+        for cand in bitfiles:
+            debug(f"Looking for {cand=} in {targets=}")
+            passing = all(re.search(t, str(cand)) for t in targets)
+            if passing:
+                matched.append(cand)
+                mod_time_string = datetime.fromtimestamp(
+                    os.path.getmtime(cand)).strftime('%Y-%m-%d %H:%M:%S')
+                tool_guess = getattr(self, '_TOOL', '')
+                if not tool_guess:
+                    ext = os.path.splitext(cand)[1]
+                    tool_guess = self.BIT_EXT_TO_TOOL.get(ext, '')
+                if tool_guess:
+                    tool_guess = f'({tool_guess})'
+                if display:
+                    info(
+                        f"{safe_emoji('⏩ ')}Found matching bitfile {tool_guess}:",
+                        f"{Colors.cyan}{mod_time_string}{Colors.normal} :",
+                        f"{Colors.byellow}{cand}"
+                    )
+
+        if display and not matched:
+            if self.args['list-bitfiles']:
+                warning(f'{safe_emoji("❕ ")}--list-bitfiles: no {self.str_ext} found that matched',
+                        f'{targets}')
+            else:
+                warning(f'{safe_emoji("❕ ")} Searched for bitfiles with {self.str_ext}: none found',
+                        f'that matched {targets}')
+
+        return matched
+
+
+
+
+    def _get_child_handling_class(self) -> object:
+        '''Returns a class handle of a child to process this, which should be a Tool class
+
+        if no appropriate child is found, returns self.
+        '''
+
+        if isinstance(self, Tool):
+            # We're already a tool handling class.
+            return self
+
+        tools_found = set()
+        for bitfile in self.bitfiles:
+            ext = os.path.splitext(bitfile)[1]
+            tool_guess = self.BIT_EXT_TO_TOOL.get(ext, '')
+            if tool_guess:
+                tools_found.add(tool_guess)
+            else:
+                warning(f'For bitfile {bitfile} no tool found for it')
+                return self
+
+        if not tools_found:
+            # Probably not an error?
+            warning(f'No tools found to process bitfiles: {self.bitfiles}')
+            return self
+
+        if len(tools_found) > 1:
+            warning(f'More than one tool found ({tools_found}) to to process bitfiles:',
+                         f'{self.bitfiles}')
+            return self
+
+
+        tool = tools_found.pop() # only item in set
+        # Do we have a handler for this in our config?
+        if tool in self.config.get('tools_loaded', []):
+            auto_tools_order = self.config.get('auto_tools_order', [{}])[0]
+            if tool in auto_tools_order:
+                cls_str = auto_tools_order[tool].get('handlers', {}).get(self.command_name, None)
+                if cls_str:
+                    cls = import_class_from_string(cls_str)
+                    if issubclass(cls, Command):
+                        info(f'For found bitfiles, can use tool={tool} and handler {cls}')
+                        sco = cls(config=self.config)
+                        return sco
+
+
+        warning(f'No handler found for tool={tool} to process bitfiles: {self.bitfiles}')
+        debug(f'config -- tools_loaded: {self.config["tools_loaded"]}')
+        debug(f'config -- auto_tools_order for tool: {self.config["auto_tools_order"][0][tool]}')
+        return self