opencos-eda 0.2.46__tar.gz → 0.2.48__tar.gz

{opencos_eda-0.2.46/opencos_eda.egg-info → opencos_eda-0.2.48}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opencos-eda
-Version: 0.2.46
+Version: 0.2.48
 Summary: A simple Python package for wrapping RTL simuliatons and synthesis
 Author-email: Simon Sabato <simon@cognichip.ai>, Drew Ranck <drew@cognichip.ai>
 Project-URL: Homepage, https://github.com/cognichip/opencos

{opencos_eda-0.2.46 → opencos_eda-0.2.48}/opencos/commands/multi.py

@@ -68,7 +68,7 @@ class CommandMulti(CommandParallel):
         return command, level
 
 
-    def resolve_path_and_target_patterns(
+    def resolve_path_and_target_patterns( # pylint: disable=too-many-locals
             self, base_path: str, target: str, level: int = -1
     ) -> dict:
         '''Returns a dict of: key = matching path, value = set of matched targets.
@@ -91,10 +91,20 @@ class CommandMulti(CommandParallel):
 
         matching_targets_dict = {}
 
+        # Let's not glob.glob if the path_pattern and target_pattern are
+        # exact, aka if it does not have special characters for glob: * or ?
+        # for the target, we also support re, so: * + ?
+        if any(x in path_pattern for x in ['*', '?']):
+            paths_from_pattern = list(glob.glob(path_pattern, recursive=True))
+        else:
+            paths_from_pattern = [path_pattern]
+
+        target_pattern_needs_lookup = any(x in target_pattern for x in ['*', '?', '+'])
+
         # resolve the path_pattern portion using glob.
         # we'll have to check for DEPS markup files in path_pattern, to match the target_wildcard
         # using fnmatch or re.
-        for path in glob.glob(path_pattern, recursive=True):
+        for path in paths_from_pattern:
 
             if self.path_hidden_or_work_dir(path):
                 continue
@@ -108,7 +118,11 @@ class CommandMulti(CommandParallel):
                 debug(f'in {rel_path=} looking for {target_pattern=} in {deps_targets=}')
 
                 for t in deps_targets:
-                    if deps_helpers.fnmatch_or_re(pattern=target_pattern, string=t):
+                    if target_pattern_needs_lookup:
+                        matched = deps_helpers.fnmatch_or_re(pattern=target_pattern, string=t)
+                    else:
+                        matched = t == target_pattern
+                    if matched:
                         if rel_path not in matching_targets_dict:
                             matching_targets_dict[rel_path] = set()
                         matching_targets_dict[rel_path].add(t)
@@ -392,9 +406,13 @@ class CommandMulti(CommandParallel):
             # Special case for 'multi' --export-jsonl, run reach child with --export-json
             command_list.append('--export-json')
         if tool and len(all_multi_tools) > 1:
-            command_list.append(f'--job-name={short_target}.{command}.{tool}')
+            jobname = f'{short_target}.{command}.{tool}'
         else:
-            command_list.append(f'--job-name={short_target}.{command}')
+            jobname = f'{short_target}.{command}'
+        command_list.append(f'--job-name={jobname}')
+        logfile = os.path.join(self.args['eda-dir'], f'eda.{jobname}.log')
+        command_list.append(f'--force-logfile={logfile}')
+
 
     def append_jobs_from_targets(self, args:list):
         '''Helper method in CommandMulti to apply 'args' (list) to all self.targets,

{opencos_eda-0.2.46 → opencos_eda-0.2.48}/opencos/commands/sweep.py

@@ -206,14 +206,18 @@ class CommandSweep(CommandDesign, CommandParallel):
             snapshot_name = snapshot_name.replace(os.sep, '_') \
                 + f'.{self.single_command}{sweep_string}'
             eda_path = get_eda_exec('sweep')
+            logfile = os.path.join(self.args['eda-dir'], f'eda.{snapshot_name}.log')
             self.jobs.append({
                 'name' : snapshot_name,
                 'index' : len(self.jobs),
                 'command': self.single_command,
                 'target': self.sweep_target,
                 'command_list' : (
-                    [eda_path, self.single_command, self.sweep_target,
-                     f'--job-name={snapshot_name}'] + arg_tokens
+                    [
+                        eda_path, self.single_command, self.sweep_target,
+                        f'--job-name={snapshot_name}',
+                        f'--force-logfile={logfile}',
+                    ] + arg_tokens
                 )
             })
             return

{opencos_eda-0.2.46 → opencos_eda-0.2.48}/opencos/commands/waves.py

@@ -143,7 +143,7 @@ class CommandWaves(CommandDesign):
             else:
                 self.error(f"Don't know how to open {wave_file} without GtkWave in PATH")
         elif wave_file.endswith('.vcd'):
-            if 'gtkwave' in self.config['tools_loaded'] and shutil.which('gktwave'):
+            if 'gtkwave' in self.config['tools_loaded'] and shutil.which('gtkwave'):
                 command_list = ['gtkwave', wave_file]
                 self.exec(os.path.dirname(wave_file), command_list)
             elif self._vsim_available(from_tools=self.VSIM_VCD_TOOLS):

{opencos_eda-0.2.46 → opencos_eda-0.2.48}/opencos/deps_helpers.py

@@ -68,7 +68,9 @@ class Defaults:
         'shell',
         'work-dir-add-srcs', 'work-dir-add-sources',
         'peakrdl',
-        'run-from-work-dir',
+        'run-from-work-dir',     # default True
+        'filepath-subst-target-dir', # default True
+        'dirpath-subst-target-dir',  # default False
         'var-subst-args',
         'var-subst-os-env',
         'tee',
@@ -384,7 +386,13 @@ def deps_list_target_sanitize(entry, default_key:str='deps', target_node:str='',
     assert False, f"Can't convert to list {entry=} {default_key=} {target_node=} {deps_file=}"
 
 
-def path_substitutions_relative_to_work_dir(exec_list : list, info_str : str, target_path : str):
+def path_substitutions_relative_to_work_dir(
+        exec_list: list, info_str: str, target_path: str,
+        enable_filepath_subst: bool, enable_dirpath_subst: bool
+) -> list:
+
+    if not enable_filepath_subst and not enable_dirpath_subst:
+        return exec_list
 
     # Look for path substitutions, b/c we later "work" in self.args['work-dir'], but
     # files should be relative to our target_path.
@@ -398,10 +406,18 @@ def path_substitutions_relative_to_work_dir(exec_list : list, info_str : str, ta
             # need this, and we can't assume dir levels in the work-dir.
             try:
                 try_path = os.path.abspath(os.path.join(os.path.abspath(target_path), m.group(1)))
-                if os.path.isfile(try_path) or os.path.isdir(try_path):
+                if enable_filepath_subst and  os.path.isfile(try_path):
+                    # make the substitution
+                    exec_list[i] = word.replace(m.group(1), try_path)
+                    debug(f'file path substitution {info_str=} {target_path=}: replaced - {word=}'
+                          f'is now ={exec_list[i]}. This can be disabled in DEPS with:',
+                          '"filepath-subst-targetdir: false"')
+                elif enable_dirpath_subst and os.path.isdir(try_path):
                     # make the substitution
                     exec_list[i] = word.replace(m.group(1), try_path)
-                    debug(f'path substitution {info_str=} {target_path=}: replaced - {word=} is now ={exec_list[i]}')
+                    debug(f'dir path substitution {info_str=} {target_path=}: replaced - {word=}'
+                          f'is now ={exec_list[i]}. This can be disabled in DEPS with:',
+                          '"dirpath-subst-targetdir: false"')
             except:
                 pass
 
@@ -1034,7 +1050,10 @@ class DepsProcessor:
 
 
 
-def parse_deps_shell_str(line : str, target_path : str, target_node : str, enable : bool = True):
+def parse_deps_shell_str(line: str, target_path: str, target_node: str,
+                         enable_filepath_subst_target_dir: bool = True,
+                         enable_dirpath_subst_target_dir: bool = False,
+                         enable: bool = True) -> dict:
     '''Returns None or a dict of a possible shell command from line (str)
 
      Examples of 'line' str:
@@ -1050,25 +1069,30 @@ def parse_deps_shell_str(line : str, target_path : str, target_node : str, enabl
     target_node (str) -- from dependency parsing, the target containing this 'line' str.
     '''
     if not enable:
-        return None
+        return {}
 
     m = re.match(r'^\s*shell\@(.*)\s*$', line)
     if not m:
-        return None
+        return {}
 
     exec_str = m.group(1)
     exec_list = exec_str.split()
 
     # Look for path substitutions, b/c we later "work" in self.args['work-dir'], but
     # files should be relative to our target_path.
-    exec_list = path_substitutions_relative_to_work_dir(exec_list=exec_list, info_str='shell@', target_path=target_path)
+    # Note this can be disable in DEPS via path-subst-target-dir=False
+    exec_list = path_substitutions_relative_to_work_dir(
+        exec_list=exec_list, info_str='shell@', target_path=target_path,
+        enable_filepath_subst=enable_filepath_subst_target_dir,
+        enable_dirpath_subst=enable_dirpath_subst_target_dir
+    )
 
-    d = {'target_path': os.path.abspath(target_path),
-         'target_node': target_node,
-         'run_from_work_dir': True,
-         'exec_list': exec_list,
-         }
-    return d
+    return {
+        'target_path': os.path.abspath(target_path),
+        'target_node': target_node,
+        'run_from_work_dir': True, # may be overriden later.
+        'exec_list': exec_list,
+    }
 
 
 def parse_deps_work_dir_add_srcs(line : str, target_path : str, target_node : str, enable : bool = True):
@@ -1100,8 +1124,12 @@ def parse_deps_work_dir_add_srcs(line : str, target_path : str, target_node : st
     return d
 
 
-def parse_deps_peakrdl(line : str, target_path : str, target_node : str, enable : bool = True,
-                       tool : str = ''):
+def parse_deps_peakrdl(
+        line: str, target_path: str, target_node: str, enable: bool = True,
+        enable_filepath_subst_target_dir: bool = True,
+        enable_dirpath_subst_target_dir: bool = False,
+        tool: str = ''
+) -> dict:
     '''Returns None or a dict describing a PeakRDL CSR register generator dependency
 
      Examples of 'line' str:
@@ -1172,10 +1200,13 @@ def parse_deps_peakrdl(line : str, target_path : str, target_node : str, enable
     # Make these look like a dep_shell_command:
     for one_cmd_as_list in shell_commands:
         ret_dict['shell_commands_list'].append(
-            parse_deps_shell_str(line = ' shell@ ' + ' '.join(one_cmd_as_list),
-                                 target_path = target_path,
-                                 target_node = target_node
-                                 )
+            parse_deps_shell_str(
+                line=(' shell@ ' + ' '.join(one_cmd_as_list)),
+                target_path=target_path,
+                target_node=target_node,
+                enable_filepath_subst_target_dir=enable_filepath_subst_target_dir,
+                enable_dirpath_subst_target_dir=enable_dirpath_subst_target_dir
+            )
         )
 
     # Make the work_dir_add_srcs dict:
@@ -1237,14 +1268,22 @@ def deps_commands_handler(config: dict, eda_args: dict,
             var_subst_dict = eda_args
 
         tee_fpath = command.get('tee', None)
-        run_from_work_dir = command.get('run-from-work-dir', True) # for shell, default True
+
+        # These are both optional bools, default True, would have to explicitly be set to False
+        # to take effect:
+        run_from_work_dir = command.get('run-from-work-dir', True)
+        filepath_subst_target_dir = command.get('filepath-subst-target-dir', True)
+        dirpath_subst_target_dir = command.get('dirpath-subst-target-dir', False)
 
         for key,item in command.items():
 
-            # skip the var-subst-* keys, since these types are bools
-            if key.startswith('var-subst') or \
-               key.startswith('tee') or \
-               key.startswith('run-from-work-dir'):
+            # skip the tee and var-subst-* keys, since these types are bools and not commands.
+            if key in ['tee',
+                       'var-subst-os-env',
+                       'var-subst-args',
+                       'run-from-work-dir',
+                       'filepath-subst-target-dir',
+                       'dirpath-subst-target-dir']:
                 continue
 
             # Optional variable substituion in commands
@@ -1255,9 +1294,11 @@ def deps_commands_handler(config: dict, eda_args: dict,
             if key == 'shell':
                 # For now, piggyback on parse_deps_shell_str:
                 ret_dict = parse_deps_shell_str(
-                    line = 'shell@ ' + item,
-                    target_path = target_path,
-                    target_node = target_node,
+                    line=('shell@ ' + item),
+                    target_path=target_path,
+                    target_node=target_node,
+                    enable_filepath_subst_target_dir=filepath_subst_target_dir,
+                    enable_dirpath_subst_target_dir=dirpath_subst_target_dir,
                     enable=config['dep_command_enables']['shell'],
                 )
                 # To support 'tee: <some-file>' need to append it to last
@@ -1286,6 +1327,8 @@ def deps_commands_handler(config: dict, eda_args: dict,
                     line = 'peakrdl@ ' + item,
                     target_path = target_path,
                     target_node = target_node,
+                    enable_filepath_subst_target_dir=filepath_subst_target_dir,
+                    enable_dirpath_subst_target_dir=dirpath_subst_target_dir,
                     enable=config['dep_command_enables']['peakrdl'],
                     tool=eda_args.get('tool', '')
                 )

{opencos_eda-0.2.46 → opencos_eda-0.2.48}/opencos/deps_schema.py

@@ -71,6 +71,12 @@ my_target_name:
         var-subst-os-env:            <---- bool, perform var substitution using os.environ
         run-from-work-dir:           <---- bool, default True, if False runs from target dir
                                            instead of work-dir.
+        filepath-subst-target-dir:   <---- bool, default True, if False does not perform
+                                           file path substitution relative to target dir
+                                           (if substituted file exists).
+        dirpath-subst-target-dir:    <---- bool, default False, if True performs
+                                           directory path substitution relative to target dir
+                                           (if substituted directory exists).
         tee:                         <---- string, filename to write logs to
       - work-dir-add-sources:        <---- work-dir-add-sources, optional list (or string)
         - some_file_gen_from_sh.sv   <---- string filename that we created with sh command
@@ -153,6 +159,8 @@ DEPS_COMMANDS_LIST = [
         Optional('var-subst-args'): bool,
         Optional('var-subst-os-env'): bool,
         Optional('run-from-work-dir'): bool,
+        Optional('filepath-subst-target-dir'): bool,
+        Optional('dirpath-subst-target-dir'): bool,
         Optional('tee'): Or(str, type(None)),
     },
     {

{opencos_eda-0.2.46 → opencos_eda-0.2.48}/opencos/eda.py

@@ -18,9 +18,10 @@ from opencos import eda_config
 from opencos import eda_base
 from opencos.eda_base import Tool, which_tool
 
-# Globals
-
+# Configure util:
 util.progname = "EDA"
+util.global_log.default_log_enabled = True
+util.global_log.default_log_filepath = os.path.join('eda.work', 'eda.log')
 
 
 # ******************************************************************************
@@ -453,10 +454,14 @@ def main(*args):
         if not config:
             util.error(f'eda.py main: problem loading config, {args=}')
             return 3
-        return interactive(config=config)
+        main_ret = interactive(config=config)
     else:
-        return process_tokens(tokens=list(unparsed), original_args=original_args,
+        main_ret =  process_tokens(tokens=list(unparsed), original_args=original_args,
                               config=config)
+    # Stop the util log, needed for pytests that call eda.main directly that otherwise
+    # won't close the log file via util's atexist.register(stop_log)
+    util.global_log.stop()
+    return main_ret
 
 
 def main_cli() -> None:

{opencos_eda-0.2.46 → opencos_eda-0.2.48}/opencos/eda_base.py

@@ -212,12 +212,30 @@ class Command:
         self.target = ""
         self.target_path = ""
         self.status = 0
+        self.errors_log_f = None
 
-    def error(self, *args, **kwargs):
-        '''Returns None, child classes can call self.error(..) instead of util.error, which updates their self.status.
+    def error(self, *args, **kwargs) -> None:
+        '''Returns None, child classes can call self.error(..) instead of util.error,
 
-        Please consider using Command.error(..) (or self.error(..)) in place of util.error so self.status is updated.
+        which updates their self.status. Also will write out to eda.errors.log if the work-dir
+        exists. Please consider using Command.error(..) (or self.error(..)) in place of util.error
+        so self.status is updated.
         '''
+
+        if self.args['work-dir']:
+            if not self.errors_log_f:
+                try:
+                    self.errors_log_f = open(
+                        os.path.join(self.args['work-dir'], 'eda.errors.log'), 'w'
+                    )
+                except:
+                    pass
+            if self.errors_log_f:
+                print(
+                    f'ERROR: [eda] ({self.command_name}) {" ".join(list(args))}',
+                    file=self.errors_log_f
+                )
+
         self.status = util.error(*args, **kwargs)
 
     def status_any_error(self, report=True) -> bool:
@@ -1480,6 +1498,9 @@ class CommandParallel(Command):
                 jobs_launched += 1
                 anything_done = True
                 job = self.jobs.pop(0)
+                # TODO(drew): it might be nice to pass more items on 'job' dict, like
+                # logfile or job-name, so CommandSweep or CommandMulti don't have to set
+                # via args. on their command_list.
                 if job['name'].startswith(multi_cwd): job['name'] = job['name'][len(multi_cwd):]
                 # in all but fancy mode, we will print this text at the launch of a job.  It may get a newline below
                 job_text = sprint_job_line(jobs_launched, job['name'], hide_stats=run_parallel)
@@ -1663,34 +1684,36 @@ class CommandParallel(Command):
         '''Examines list self.jobs, and if leaf target names are duplicate will
         patch each command's job-name to:
             --job-name=path.leaf.command[.tool]
+
+        Also do for --force-logfile
         '''
 
-        def get_job_name(job_dict: dict) -> str:
-            '''Fishes the job-name out of an entry in self.jobs'''
+        def get_job_arg(job_dict: dict, arg_name: str) -> str:
+            '''Fishes the arg_name out of an entry in self.jobs'''
             for i, item in enumerate(job_dict['command_list']):
-                if item.startswith('--job-name='):
-                    _, name = item.split('--job-name=')
+                if item.startswith(f'--{arg_name}='):
+                    _, name = item.split(f'--{arg_name}=')
                     return name
-                elif item == '--job-name':
+                elif item == f'--{arg_name}':
                     return job_dict['command_list'][i + 1]
             return ''
 
-        def replace_job_name(job_dict: dict, new_job_name: str) -> dict:
-            '''Replaces the job-name in an entry in self.jobs'''
+        def replace_job_arg(job_dict: dict, arg_name: str, new_value: str) -> bool:
+            '''Replaces the arg_name's value in an entry in self.jobs'''
             for i, item in enumerate(job_dict['command_list']):
-                if item.startswith('--job-name='):
-                    job_dict['command_list'][i] = '--job-name=' + new_job_name
-                    return job_dict
-                elif item == '--job-name':
-                    job_dict['command_list'][i + 1] = new_job_name
-                    return job_dict
-            return job_dict
+                if item.startswith(f'--{arg_name}='):
+                    job_dict['command_list'][i] = f'--{arg_name}=' + new_value
+                    return True
+                elif item == f'--{arg_name}':
+                    job_dict['command_list'][i + 1] = new_value
+                    return True
+            return False
 
 
         job_names_count_dict = {}
         for job_dict in self.jobs:
 
-            key = get_job_name(job_dict)
+            key = get_job_arg(job_dict, arg_name='job-name')
             if not key:
                 self.error(f'{job_dict=} needs to have a --job-name= arg attached')
             if key not in job_names_count_dict:
@@ -1699,7 +1722,7 @@ class CommandParallel(Command):
                 job_names_count_dict[key] += 1
 
         for i, job_dict in enumerate(self.jobs):
-            key = get_job_name(job_dict)
+            key = get_job_arg(job_dict, 'job-name')
             if job_names_count_dict[key] < 2:
                 continue
 
@@ -1709,6 +1732,20 @@ class CommandParallel(Command):
             patched_sub_work_dir = False
             patched_target_path = os.path.relpath(tpath).replace(os.sep, '_')
             new_job_name = f'{patched_target_path}.{key}'
-            new_job_dict = replace_job_name(job_dict, new_job_name)
-            self.jobs[i] = new_job_dict
+            replace_job_arg(job_dict, arg_name='job-name', new_value=new_job_name)
+
+            # prepend path information to force-logfile (if present):
+            force_logfile = get_job_arg(job_dict, arg_name='force-logfile')
+            if force_logfile:
+                left, right = os.path.split(force_logfile)
+                new_force_logfile = os.path.join(left, f'{patched_target_path}.{right}')
+                replace_job_arg(
+                    job_dict, arg_name='force-logfile', new_value=new_force_logfile
+                )
+                util.debug(
+                    f'Patched job {job_dict["name"]}: --force-logfile={new_force_logfile}'
+                )
+
+
+            self.jobs[i] = job_dict
             util.debug(f'Patched job {job_dict["name"]}: --job-name={new_job_name}')

{opencos_eda-0.2.46 → opencos_eda-0.2.48}/opencos/tests/helpers.py

@@ -124,20 +124,35 @@ def assert_export_jsonl_good(filepath:str, jsonl:bool=True) -> list:
 class Helpers:
     '''We do so much with logging in this file, might as well make it reusable'''
     DEFAULT_DIR = ''
-    DEFAULT_LOG = 'eda.log'
+    DEFAULT_LOG_DIR = os.getcwd()
+    DEFAULT_LOG = os.path.join(DEFAULT_LOG_DIR, '.pytest.eda.log')
     def chdir(self):
         '''Changes directory to self.DEFAULT_DIR and removes eda.work, eda.export paths'''
         chdir_remove_work_dir('', self.DEFAULT_DIR)
 
+    def _resolve_logfile(self, logfile=None) -> str:
+        '''Returns the logfile's filepath'''
+        ret = logfile
+        if ret is None:
+            ret = self.DEFAULT_LOG
+        else:
+            left, right = os.path.split(logfile)
+            if not left or left in [os.path.sep, '.', '..']:
+                # relative logfile put in DEFAULT_LOG_DIR:
+                ret = os.path.join(self.DEFAULT_LOG_DIR, right)
+        return ret
+
     def log_it(self, command_str:str, logfile=None, use_eda_wrap=True) -> int:
         '''Replacement for calling eda.main or eda_wrap, when you want an internal logfile
 
         Usage:
-            rc = self.log_it('sim foo', logfile='yes.log')
+            rc = self.log_it('sim foo')
             assert rc == 0
+
+        Note this will run with --no-default-log to avoid a Windows problem with stomping
+        on a log file.
         '''
-        if logfile is None:
-            logfile = self.DEFAULT_LOG
+        logfile = self._resolve_logfile(logfile)
         rc = 50
 
         # TODO(drew): There are some issues with log_it redirecting stdout from vivado
@@ -148,16 +163,15 @@ class Helpers:
         with open(logfile, 'w', encoding='utf-8') as f:
             with redirect_stdout(f), redirect_stderr(f):
                 if use_eda_wrap:
-                    rc = eda_wrap(*(command_str.split()))
+                    rc = eda_wrap('--no-default-log', *(command_str.split()))
                 else:
-                    rc = eda.main(*(command_str.split()))
+                    rc = eda.main('--no-default-log', *(command_str.split()))
             print(f'Wrote: {os.path.abspath(logfile)=}')
         return rc
 
     def is_in_log(self, *want_str, logfile=None, windows_path_support=False):
         '''Check if any of want_str args are in the logfile, or self.DEFAULT_LOG'''
-        if logfile is None:
-            logfile = self.DEFAULT_LOG
+        logfile = self._resolve_logfile(logfile)
         want_str0 = ' '.join(list(want_str))
         want_str1 = want_str0.replace('/', '\\')
         with open(logfile, encoding='utf-8') as f:
@@ -169,8 +183,7 @@ class Helpers:
 
     def get_log_lines_with(self, *want_str, logfile=None, windows_path_support=False):
         '''gets all log lines with any of want_str args are in the logfile, or self.DEFAULT_LOG'''
-        if logfile is None:
-            logfile = self.DEFAULT_LOG
+        logfile = self._resolve_logfile(logfile)
         ret_list = []
         want_str0 = ' '.join(list(want_str))
         want_str1 = want_str0.replace('/', '\\')
@@ -186,8 +199,7 @@ class Helpers:
         '''gets all log lines with any of *want_str within a single word
         in the logfile or self.DEFAULT_LOG
         '''
-        if logfile is None:
-            logfile = self.DEFAULT_LOG
+        logfile = self._resolve_logfile(logfile)
         ret_list = []
         want_str0 = ' '.join(list(want_str))
         want_str1 = want_str0.replace('/', '\\')