opencos-eda 0.3.3__py3-none-any.whl → 0.3.6__py3-none-any.whl

opencos/eda_config.py

@@ -15,10 +15,14 @@ import shutil
 import mergedeep
 
 from opencos import util
+from opencos.util import safe_emoji
 from opencos.utils.markup_helpers import yaml_safe_load, yaml_safe_writer
 
 class Defaults:
-    '''Defaults is a global placeholder for constants and supported features.'''
+    '''Defaults is a global namespace for constants and supported features.
+
+    Defaults.config_yml is set depending on search order for default eda_config[_defaults].yml
+    '''
 
     environ_override_config_yml = os.environ.get('EDA_CONFIG_YML', '')
     home_override_config_yml = os.path.join(
@@ -63,14 +67,15 @@ class Defaults:
         'simulate-args',
         'simulate-waves-args',
         'simulate-waivers',
+        'simulate-coverage-tcl',
         'coverage-args',
     ])
 
 EDA_OUTPUT_CONFIG_FNAME = 'eda_output_config.yml'
 
-if os.path.exists(Defaults.environ_override_config_yml):
+if os.path.isfile(Defaults.environ_override_config_yml):
     Defaults.config_yml = Defaults.environ_override_config_yml
-elif os.path.exists(Defaults.home_override_config_yml):
+elif os.path.isfile(Defaults.home_override_config_yml):
     Defaults.config_yml = Defaults.home_override_config_yml
 else:
     Defaults.config_yml = Defaults.opencos_config_yml
@@ -149,29 +154,9 @@ def update_config_auto_tool_order_for_tool(tool: str, config: dict) -> str:
     Input arg tool can be in the form (for example):
       tool='verlator', tool='verilator=/path/to/verilator.exe'
 
-    Performs no update if tool has no = in it. Returns tool (str) w/out = in it
+    Performs no update if tool has no = or : in it. Returns tool (str) w/out = in it
     '''
-    if not tool or '=' not in tool:
-        return tool
-
-    tool, user_exe = tool.split('=')[0:2]
-
-    user_exe = shutil.which(user_exe)
-
-    # try adding to $PATH if in form --tool=/path/to/exe
-    tool_try_add_to_path(tool)
-
-    if tool not in config['auto_tools_order'][0]:
-        return tool
-    if not user_exe:
-        return tool
-
-    old_exe = config['auto_tools_order'][0][tool].get('exe', str())
-    if isinstance(old_exe, list):
-        config['auto_tools_order'][0][tool]['exe'][0] = user_exe
-    else:
-        config['auto_tools_order'][0][tool]['exe'] = user_exe
-    return tool
+    return tool_try_add_to_path(tool=tool, config=config, update_config=True)
 
 
 def update_config_auto_tool_order_for_tools(tools: list, config: dict) -> list:
@@ -247,7 +232,7 @@ def get_config_merged_with_defaults(config:dict) -> dict:
 def get_argparser() -> argparse.ArgumentParser:
     '''Returns an ArgumentParser, handles --config-yml=<filename> arg'''
     parser = argparse.ArgumentParser(
-        prog='opencos eda config options', add_help=False, allow_abbrev=False
+        prog=f'{safe_emoji("🔎 ")}opencos eda config options', add_help=False, allow_abbrev=False
     )
     parser.add_argument('--config-yml', type=str, default=Defaults.config_yml,
                         help=('YAML filename to use for configuration (default'
@@ -327,32 +312,92 @@ def write_eda_config_and_args(
     yaml_safe_writer(data=data, filepath=fullpath)
 
 
-def tool_try_add_to_path(tool: str) -> None:
-    '''Since we support --tool=<name>=/path/to/bin/exe, attempt to prepend $PATH
+def tool_arg_get_parts(tool: str) -> list:
+    '''Given a tool (str or None) that may be in form <name>=/path/to/something
+
+    Return the parts [<name>, <path>, ..]
+    '''
+    if not tool or ('=' not in tool and ':' not in tool):
+        return [tool]
+
+    if '=' in tool:
+        parts = tool.split('=')
+    else:
+        parts = tool.split(':')
+
+    return parts
+
+def tool_arg_remove_path_information(tool: str) -> str:
+    '''Given a tool (str or None) that may be in form <name>=/path/to/something
+
+    Return the <name> only
+    '''
+    if not tool:
+        return tool
+    return tool_arg_get_parts(tool)[0]
+
+
+def tool_try_add_to_path( # pylint: disable=too-many-branches
+        tool: str, config: dict, update_config: bool
+) -> str:
+    '''Since we support --tool=<name>=/path/to/bin[/exe], attempt to prepend $PATH
+
+    (also works for --tool=<name>:/path/to/bin[/exe] )
 
     with this information for this tool (which will nicely affect all subprocesses,
     but not wreck our original shell).'''
 
-    if not tool or '=' not in tool:
-        return
+    name_path_parts = tool_arg_get_parts(tool)
+    if len(name_path_parts) == 1:
+        return name_path_parts[0]
+
+    name, path_arg = name_path_parts[0:2]
+
+    if name not in config['auto_tools_order'][0]:
+        return name
+
+    config_exe = config['auto_tools_order'][0][name].get('exe', str())
+    if isinstance(config_exe, list):
+        orig_exe = config_exe[0]
+    else:
+        orig_exe = config_exe
 
-    name, exe = tool.split('=')
-    if os.path.isdir(name):
+    if path_arg and os.path.isfile(path_arg):
+        # Someone passes us --tool=<name>=/path/to/bin/exe, remove the exe from path:
+        path, exe = os.path.split(path_arg)
+    elif path_arg and os.path.isdir(path_arg):
         # Someone passes us --tool=<name>=/path/to/bin/ (did not have exe)
-        path = name
+        path, exe = path_arg, orig_exe
     else:
-        # Someone passes us --tool=<name>=/path/to/bin/exe, remove the exe.
-        path, _ = os.path.split(exe)
-    if not path:
-        return
+        path, exe = '', ''
+
+    if not path or not exe:
+        util.error(f'Can not find path or exe for --tool={tool}: {name=} path={path_arg}')
+        return name
 
     path = os.path.abspath(path)
     if os.path.isdir(path):
-        paths = os.environ['PATH'].split(':')
+        paths = os.environ.get('PATH', '').split(':')
         if path not in paths:
-            os.environ['PATH'] = path + ':' + os.environ['PATH']
             util.info(f'--tool={tool} has path information, prepending PATH with: {path}')
+            os.environ['PATH'] = path + ':' + os.environ.get('PATH', '')
         else:
             util.info(f'--tool={tool} has path information, but {path} already in $PATH')
-    if exe and os.path.isfile(exe):
-        util.info(f'--tool={tool} has path information, using exe {shutil.which(exe)}')
+
+    user_exe = os.path.join(path, exe)
+    if not os.access(user_exe, os.X_OK):
+        util.error(f'--tool setting for {tool}: {user_exe} is not an executable')
+        return name
+
+    user_exe = shutil.which(exe)
+
+    if update_config:
+        if isinstance(config_exe, list):
+            config['auto_tools_order'][0][name]['exe'][0] = user_exe
+        else:
+            config['auto_tools_order'][0][name]['exe'] = user_exe
+        util.debug(f'For {tool=}, auto_tools_order config updated')
+
+    util.debug(f'For {tool=}, final {user_exe=}')
+
+    return name

opencos/eda_config_defaults.yml

@@ -94,6 +94,11 @@ file_extensions:
   dotf:
     - .f
     - .vc
+  python:
+    - .py
+  makefile:
+    - .mk
+
 
   inferred_top:
     # file extensions that we can infer "top" module from, if --top omitted.
@@ -247,6 +252,9 @@ tools:
       +accb +accr +access +r+w
     coverage-args: |
       -acdb -acdb_cov sbfectapm
+    simulate-coverage-tcl:
+      - acdb save
+      - acdb report -db work.acdb -txt -o cov.txt
 
 
   modelsim_ase:
@@ -329,7 +337,6 @@ tools:
       - "COCOTB_TEST_FAILED"
     log-must-strings:
       - "passed"
-      - "Cocotb test completed successfully!"
 
 
   quartus:

opencos/eda_tool_helper.py

@@ -68,3 +68,22 @@ def get_all_handler_commands(config=None, tools_loaded=None) -> dict:
                 all_handler_commands[command].append(tool)
 
     return all_handler_commands
+
+
+def get_handler_tool_version(tool: str, eda_command: str, config: dict) -> str:
+    '''Attempts to get a Command Handler's version given tool + eda_command'''
+
+    entry = config['auto_tools_order'][0].get(tool, {})
+    if not entry:
+        return ''
+
+    handler_name = entry.get('handlers', {}).get(eda_command, '')
+    if not handler_name:
+        return ''
+
+    module = util.import_class_from_string(handler_name)
+    obj = module(config=config)
+    if not getattr(obj, 'get_versions', None):
+        return ''
+
+    return obj.get_versions()

opencos/export_helper.py

@@ -42,12 +42,18 @@ def json_paths_to_jsonl(
     with open(output_json_path, 'w', encoding='utf-8') as outf:
 
         # jsonl is every line of the file is a json.
+        # We would expect the JSON to have "tests": [ ... ] with >= 1 test.
         for json_file_path in json_file_paths:
             with open(json_file_path, encoding='utf-8') as f:
                 data = json.load(f)
                 if len(assert_json_types) > 0 and type(data) not in assert_json_types:
                     error(f'{json_file_path=} JSON data is not a Table (py dict) {type(data)=}')
-                json.dump(data, outf)
+                if 'tests' in data and isinstance(data['tests'], list):
+                    for test in data['tests']:
+                        json.dump(test, outf)
+                else:
+                    json.dump(data, outf)
+
                 outf.write('\n')
         info(f'Wrote {len(json_file_paths)} tests to {output_json_path=}')
 
@@ -78,7 +84,12 @@ def json_paths_to_single_json(
                 data = json.load(f)
                 if len(assert_json_types) > 0 and type(data) not in assert_json_types:
                     error(f'{json_file_path=} JSON data is not a Table (py dict) {type(data)=}')
-                out_json_data['tests'].append(data)
+                if 'tests' in data and isinstance(data['tests'], list):
+                    for test in data['tests']:
+                        out_json_data['tests'].append(test)
+                else:
+                    out_json_data['tests'].append(data)
+
         json.dump(out_json_data, outf)
         outf.write('\n')
         info(f'Wrote {len(json_file_paths)} tests {output_json_path=}')
@@ -471,7 +482,52 @@ class ExportHelper:
         yaml_safe_writer(data=data, filepath=dst)
 
 
-    def create_export_json_in_out_dir( # pylint: disable=unused-argument
+    def create_combined_env_file_in_out_dir(self) -> list:
+        ''' Returns list of all .env lines to be put in output directory
+
+        Creates single exported .env file if any --env-file(s)
+        were loaded by CLI or DEPS targets. --input-file/-f was handled as args
+        already, but --env-files need to be special cased. We do not add this
+        combined ".env" file to the all_files list, just build it on the fly if needed.
+        '''
+        env_lines = []
+        for filepath in util.env_files_loaded:
+            with open(filepath, encoding='utf-8') as f:
+                env_lines.extend(f.readlines() + ['\n'])
+
+        if env_lines:
+            dst = os.path.join(self.out_dir, '.env')
+
+            # Write this to the export directory too:
+            with open(dst, 'w', encoding='utf-8') as f:
+                for lineno, line in enumerate(env_lines):
+
+                    # modify VERILOG_SOURCES line if present, although it is known via DEPS.yml,
+                    # we can set it according to files_v and files_sv:
+                    if line.strip().startswith('VERILOG_SOURCES'):
+                        base_verilog_filenames = [
+                            os.path.split(x)[1] for x in \
+                            self.cmd_design_obj.files_v + self.cmd_design_obj.files_sv
+                        ]
+                        env_lines[lineno] = (
+                            f'VERILOG_SOURCES = {" ".join(base_verilog_filenames)}'
+                        )
+                        continue
+
+                    # remove PYTHONPATH from .env
+                    if line.strip().startswith('PYTHONPATH'):
+                        env_lines[lineno] = ''
+                        continue
+
+                    f.write(line)
+
+
+            info(f'export_helper: Wrote {dst}')
+
+        return env_lines
+
+
+    def create_export_json_in_out_dir( # pylint: disable=unused-argument,too-many-locals,too-many-branches
             self, eda_config:dict={}, **kwargs
     ) -> None:
         '''Optionally creates an exported JSON file in the output directory'''
@@ -482,40 +538,43 @@ class ExportHelper:
         # assumes we've run self.create_deps_yml_in_out_dir():
         assert self.target
         assert self.out_deps_file
+        full_eda_cmd_list = ['eda', self.eda_command]
+        if self.args.get('waves', False):
+            full_eda_cmd_list.append('--waves')
+        if self.args.get('tool', ''):
+            full_eda_cmd_list.append(f'--tool={self.args["tool"]}')
+        full_eda_cmd_list.append(self.target)
 
         data = {
-            'name': self.target,
-            'eda': {
-                'enable': True,
-                'multi': False, # Not yet implemented.
-                'command': self.eda_command,
-                'targets': [self.target],
-                'args': [],
-                'waves': self.args.get('waves', False),
-                # tool - eda.CommandSimVerilator has this set in self.args:
-                'tool': self.args.get('tool', None),
-            },
-            'files': [],
+            'correlationId': self.target,
+            'jobType': 'edaCmd',
+            'cmd': ' '.join(full_eda_cmd_list),
+            'timeout': 600,
+            'filesList': [], # filename (str), content (str)
         }
 
-        # allow caller to override eda - tool, or eda - args, etc.
-        for k,v in eda_config.items():
-            if k in data['eda'] and v is not None:
-                data['eda'][k] = v
-
         # Note that args may already be set via:
         #   create_deps_yml_in_out_dir(deps_file_args=some_list)
         # For example, eda.CommandSim.do_export() will set certain allow-listed
         # args if present with non-default values.
 
-
         all_files = [self.out_deps_file] + self.included_files \
             + self.cmd_design_obj.files_sv + self.cmd_design_obj.files_v \
             + self.cmd_design_obj.files_vhd + self.cmd_design_obj.files_cpp \
+            + self.cmd_design_obj.files_sdc + self.cmd_design_obj.files_py \
+            + self.cmd_design_obj.files_makefile + self.cmd_design_obj.files_non_source
+
+        all_files = list(dict.fromkeys(all_files)) # uniqify list.
+
+        # The last file we handle is a single exported .env file if any --env-file(s)
+        env_lines = self.create_combined_env_file_in_out_dir()
+        if env_lines:
+            # write the updated env_lines to the export.json file.
+            data['filesList'].append({
+                'filename': '.env',
+                'content': ''.join(env_lines), # already has \n per line
+            })
 
-        for x in self.cmd_design_obj.files_non_source:
-            if x not in all_files:
-                all_files.append(x)
 
         for somefile in all_files:
 
@@ -532,18 +591,17 @@ class ExportHelper:
 
             assert os.path.exists(somefile)
             with open(somefile, encoding='utf-8') as f:
-                filestr = ''.join(f.readlines())
-                data['files'].append({
-                    'name': os.path.split(somefile)[1],
-                    'content': filestr,
+                data['filesList'].append({
+                    'filename': os.path.split(somefile)[1],
+                    'content': ''.join(f.readlines()),
                 })
 
-
+        test_runner_data = {'tests': [data]} # single test for test runner.
         dst = os.path.join(self.out_dir, 'export.json')
         with open(dst, 'w', encoding='utf-8') as f:
-            json.dump(data, f)
+            json.dump(test_runner_data, f)
             f.write('\n')
-        info(f'export_helper: Wrote {dst=}')
+        info(f'export_helper: Wrote {dst}')
 
         # If this was from an `export` command, and the self.out_dir != self.args['work-dir'], then
         # copy the export.json to the work-dir:

opencos/files.py

@@ -24,7 +24,9 @@ FORCE_PREFIX_DICT = {
     'vhdl@': 'vhdl',
     'cpp@': 'cpp',
     'sdc@': 'synth_constraints',
-    'f@': 'dotf'
+    'f@': 'dotf',
+    'py@' : 'python',
+    'makefile@': 'makefile'
 }
 
 ALL_FORCED_PREFIXES = set(list(FORCE_PREFIX_DICT.keys()))

opencos/tests/helpers.py

@@ -16,6 +16,7 @@ from opencos.utils.subprocess_helpers import subprocess_run_background
 
 # Figure out what tools the system has available, without calling eda.main(..)
 config, tools_loaded = eda_tool_helper.get_config_and_tools_loaded()
+known_tool_versions = {}
 
 
 def eda_wrap_is_sim_fail(rc: int, quiet: bool = False) -> bool:
@@ -32,6 +33,16 @@ def eda_wrap_is_sim_fail(rc: int, quiet: bool = False) -> bool:
         status_constants.EDA_DEFAULT_ERROR
     )
 
+def handle_tool_version(tool: str, eda_command: str, cfg: dict = config) -> None:
+    '''Attempts to use a command handler for tool + eda_command and we'll
+    track the version globally'''
+
+    if not known_tool_versions.get(tool, ''):
+        handler_version = eda_tool_helper.get_handler_tool_version(
+            tool=tool, eda_command=eda_command, config=cfg
+        )
+        known_tool_versions[tool] = handler_version
+
 def can_run_eda_command(*commands, cfg: dict = config) -> bool:
     '''Returns True if we have any installed tool that can run: eda <command>'''
     runnable = []
@@ -50,10 +61,36 @@ def can_run_eda_command(*commands, cfg: dict = config) -> bool:
             if entry and entry.get('disable-auto', False):
                 # This tool cannot automatically run our command.
                 return False
+            # While we're here, set known tool versions.
+            handle_tool_version(tool=tool, eda_command=command, cfg=cfg)
 
         runnable.append(True)
     return runnable and all(runnable)
 
+def can_uvm(tool: str) -> bool:
+    '''Returns True if we can run UVM, per tool'''
+
+    if tool not in tools_loaded:
+        return False
+
+    if tool == 'verilator':
+        # requires UVM_HOME to be set with uvm_pkg.sv existing.
+        uvm_home = os.environ.get('UVM_HOME', '')
+        uvm_pkg = os.path.join(uvm_home, 'uvm_pkg.sv')
+        if not all((os.path.isdir(uvm_home), os.path.isfile(uvm_pkg))):
+            return False
+
+        handle_tool_version(tool=tool, eda_command='sim', cfg=config)
+        version_list = known_tool_versions.get('verilator', '').split('.')
+        if int(version_list[0]) < 5 or \
+           (int(version_list[0]) == 5 and int(version_list[1]) < 42):
+            return False
+
+        return True
+
+    # default return False:
+    return False
+
 def can_run_eda_sim(cfg: dict = config) -> bool:
     '''Returns True if we have any installed tool that can run: eda sim'''
     return can_run_eda_command('sim', cfg=cfg)
@@ -127,38 +164,43 @@ def assert_gen_deps_yml_good(filepath:str, want_target:str='') -> dict:
 
 def assert_export_json_good(filepath:str) -> dict:
     '''Checks that an exported JSON (from eda export, or eda <command> --export) has known keys'''
-    assert os.path.exists(filepath), f'{filepath=} does not exist'
+    assert os.path.isfile(filepath), f'{filepath=} does not exist'
     with open(filepath, encoding='utf-8') as f:
         data = json.load(f)
-    assert 'name' in data
-    assert 'eda' in data
-    assert any(x in data for x in ['files', 'tb'])
+    assert 'tests' in data
+    assert len(data.get('tests', [])) >= 1
+    for test in data.get('tests', []):
+        check_test_runner_schema(test)
     return data
 
+def check_test_runner_schema(test: dict) -> None:
+    '''Confirm that a single test's JSON/JSONL schema is OK.'''
+    assert 'correlationId' in test
+    assert 'jobType' in test
+    assert 'cmd' in test
+    assert 'filesList' in test # 0 files is OK.
+
 
 def assert_export_jsonl_good(filepath:str, jsonl:bool=True) -> list:
     '''Checks that an exported JSONL (from eda multi --export) has known keys'''
-    assert os.path.exists(filepath), f'{filepath=} does not exist'
+    assert os.path.isfile(filepath), f'{filepath=} does not exist'
     ret = []
     with open(filepath, encoding='utf-8') as f:
         if jsonl:
+            print(f'Using JSONL for {filepath=}')
             for line in f.readlines():
                 line = line.strip()
-                data = json.loads(line)
-                assert 'name' in data
-                assert 'eda' in data
-                assert any(x in data for x in ['files', 'tb'])
-                ret.append(data)
+                test = json.loads(line)
+                check_test_runner_schema(test)
+                ret.append(test)
         else:
+            print(f'Using JSON for {filepath=}')
             data = json.load(f)
             assert 'tests' in data
-            assert isinstance(data['tests'], list)
-            for entry in data['tests']:
-                assert 'name' in entry
-                assert 'eda' in entry
-                assert any(x in entry for x in ['files', 'tb'])
-                ret.append(entry)
-
+            assert len(data.get('tests', [])) >= 1
+            for test in data.get('tests', []):
+                check_test_runner_schema(test)
+                ret.append(test)
 
     return ret
 
@@ -228,6 +270,7 @@ class Helpers:
                 background=True,
                 tee_fpath=logfile
             )
+            print(f'Wrote: {os.path.abspath(logfile)=}')
         else:
             with open(logfile, 'w', encoding='utf-8') as f:
                 with redirect_stdout(f), redirect_stderr(f):

opencos/tests/test_eda.py

@@ -100,9 +100,19 @@ class TestsRequiresVerilator( # pylint: disable=too-many-public-methods
     def test_args_sim_tool_with_path(self):
         '''Test for calling a tool as --tool=<tool>=</path/to/tool-exe>'''
         verilator_fullpath = shutil.which('verilator')
+        verilator_path, _ = os.path.split(verilator_fullpath)
+
         chdir_remove_work_dir('../../lib/tests')
         rc = eda_sim_wrap('--tool', f'verilator={verilator_fullpath}', 'oclib_fifo_test')
-        print(f'{rc=}')
+        assert rc == 0
+
+        rc = eda_sim_wrap('--tool', f'verilator:{verilator_fullpath}', 'oclib_fifo_test')
+        assert rc == 0
+
+        rc = eda_sim_wrap('--tool', f'verilator={verilator_path}', 'oclib_fifo_test')
+        assert rc == 0
+
+        rc = eda_sim_wrap('--tool', f'verilator:{verilator_fullpath}', 'oclib_fifo_test')
         assert rc == 0
 
     def test_args_sim_with_coverage(self):