opencos-eda 0.2.48__py3-none-any.whl → 0.2.50__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. opencos/__init__.py +4 -2
  2. opencos/_version.py +10 -7
  3. opencos/commands/flist.py +8 -7
  4. opencos/commands/multi.py +14 -15
  5. opencos/commands/sim.py +5 -0
  6. opencos/commands/sweep.py +3 -2
  7. opencos/deps/__init__.py +0 -0
  8. opencos/deps/defaults.py +69 -0
  9. opencos/deps/deps_commands.py +419 -0
  10. opencos/deps/deps_file.py +326 -0
  11. opencos/deps/deps_processor.py +670 -0
  12. opencos/deps_schema.py +7 -8
  13. opencos/eda.py +84 -64
  14. opencos/eda_base.py +585 -316
  15. opencos/eda_config.py +85 -14
  16. opencos/eda_config_defaults.yml +36 -4
  17. opencos/eda_extract_targets.py +22 -14
  18. opencos/eda_tool_helper.py +33 -7
  19. opencos/export_helper.py +166 -86
  20. opencos/export_json_convert.py +31 -23
  21. opencos/files.py +2 -1
  22. opencos/hw/__init__.py +0 -0
  23. opencos/{oc_cli.py → hw/oc_cli.py} +9 -4
  24. opencos/names.py +0 -4
  25. opencos/peakrdl_cleanup.py +13 -7
  26. opencos/seed.py +19 -11
  27. opencos/tests/helpers.py +3 -2
  28. opencos/tests/test_deps_helpers.py +35 -32
  29. opencos/tests/test_eda.py +36 -29
  30. opencos/tests/test_eda_elab.py +7 -4
  31. opencos/tests/test_eda_synth.py +1 -1
  32. opencos/tests/test_oc_cli.py +1 -1
  33. opencos/tests/test_tools.py +4 -2
  34. opencos/tools/iverilog.py +2 -2
  35. opencos/tools/modelsim_ase.py +24 -2
  36. opencos/tools/questa.py +5 -3
  37. opencos/tools/questa_fse.py +57 -0
  38. opencos/tools/riviera.py +1 -1
  39. opencos/tools/slang.py +9 -3
  40. opencos/tools/surelog.py +1 -1
  41. opencos/tools/verilator.py +26 -1
  42. opencos/tools/vivado.py +34 -27
  43. opencos/tools/yosys.py +4 -3
  44. opencos/util.py +532 -474
  45. opencos/utils/__init__.py +0 -0
  46. opencos/utils/markup_helpers.py +98 -0
  47. opencos/utils/str_helpers.py +111 -0
  48. opencos/utils/subprocess_helpers.py +108 -0
  49. {opencos_eda-0.2.48.dist-info → opencos_eda-0.2.50.dist-info}/METADATA +1 -1
  50. opencos_eda-0.2.50.dist-info/RECORD +89 -0
  51. {opencos_eda-0.2.48.dist-info → opencos_eda-0.2.50.dist-info}/entry_points.txt +1 -1
  52. opencos/deps_helpers.py +0 -1346
  53. opencos_eda-0.2.48.dist-info/RECORD +0 -79
  54. /opencos/{pcie.py → hw/pcie.py} +0 -0
  55. {opencos_eda-0.2.48.dist-info → opencos_eda-0.2.50.dist-info}/WHEEL +0 -0
  56. {opencos_eda-0.2.48.dist-info → opencos_eda-0.2.50.dist-info}/licenses/LICENSE +0 -0
  57. {opencos_eda-0.2.48.dist-info → opencos_eda-0.2.50.dist-info}/licenses/LICENSE.spdx +0 -0
  58. {opencos_eda-0.2.48.dist-info → opencos_eda-0.2.50.dist-info}/top_level.txt +0 -0
opencos/eda_base.py CHANGED
@@ -1,22 +1,36 @@
1
+ '''opencos.eda_base holds base classes for opencos.tools and opencos.commands
2
+
3
+ Note that opencos.Command.process_tokens (on a derived class) is the main entrypoint
4
+ from eda.py, it will find a command handler class for a command + tool, and call
5
+ its obj.process_tokens(args)
6
+ '''
7
+
8
+ # TODO(drew): Clean up the following pylint
9
+ # pylint: disable=too-many-lines
10
+
1
11
  import argparse
2
12
  import copy
3
- import glob
4
13
  import subprocess
5
14
  import os
6
15
  import queue
7
16
  import re
8
17
  import shutil
9
- import shlex
10
18
  import signal
11
19
  import sys
12
20
  import threading
13
21
  import time
14
22
  from pathlib import Path
15
23
 
16
- import opencos
17
- from opencos import seed, deps_helpers, util, files
24
+ from opencos import seed, util, files
18
25
  from opencos import eda_config
19
- from opencos.deps_helpers import get_deps_markup_file, deps_markup_safe_load
26
+
27
+ from opencos.util import Colors
28
+ from opencos.utils.str_helpers import sprint_time, strip_outer_quotes, string_or_space, \
29
+ indent_wrap_long_text
30
+ from opencos.utils.subprocess_helpers import subprocess_run_background
31
+
32
+ from opencos.deps.deps_file import DepsFile, deps_data_get_all_targets
33
+ from opencos.deps.deps_processor import DepsProcessor
20
34
 
21
35
 
22
36
  def print_base_help() -> None:
@@ -28,9 +42,12 @@ def print_base_help() -> None:
28
42
 
29
43
 
30
44
  def get_argparser() -> argparse.ArgumentParser:
45
+ '''Returns the ArgumentParser for general eda CLI'''
31
46
  parser = argparse.ArgumentParser(prog='eda options', add_help=False, allow_abbrev=False)
32
47
  parser.add_argument('-q', '--quit', action='store_true',
33
- help='For interactive mode (eda called with no options, command, or targets)')
48
+ help=(
49
+ 'For interactive mode (eda called with no options, command, or'
50
+ ' targets)'))
34
51
  parser.add_argument('--exit', action='store_true', help='same as --quit')
35
52
  parser.add_argument('-h', '--help', action='store_true')
36
53
 
@@ -40,22 +57,25 @@ def get_argparser() -> argparse.ArgumentParser:
40
57
  help='Tool to use for this command, such as: modelsim_ase, verilator,' \
41
58
  + ' modelsim_ase=/path/to/bin/vsim, verilator=/path/to/bin/verilator')
42
59
  parser.add_argument('--eda-safe', action='store_true',
43
- help='disable all DEPS file deps shell commands, overrides values from --config-yml')
60
+ help=('disable all DEPS file deps shell commands, overrides values from'
61
+ ' --config-yml'))
44
62
  return parser
45
63
 
46
64
 
47
65
  def get_argparser_short_help() -> str:
66
+ '''Returns str for the eda_base argparser (--help, --tool, etc)'''
48
67
  return util.get_argparser_short_help(parser=get_argparser())
49
68
 
50
69
 
51
- def get_eda_exec(command:str=''):
70
+ def get_eda_exec(command: str = '') -> str:
71
+ '''Returns the full path of `eda` executable to be used for a given eda <command>'''
52
72
  # NOTE(drew): This is kind of flaky. 'eda multi' reinvokes 'eda'. But the executable for 'eda'
53
73
  # is one of:
54
74
  # 1. pip3 install opencos-eda
55
75
  # -- script 'eda', installed from PyPi
56
76
  # 2. pip3 uninstall .; python3 -m build; pip3 install
57
77
  # -- script 'eda' but installed from local.
58
- # 2. (opencos repo)/bin/eda - a python wrapper to link to (opencos repo)/opencos/eda.py (package)
78
+ # 2. (opencos repo)/bin/eda - a python wrapper to link to (opencos repo)/opencos/eda.py, but
59
79
  # packages cannot be run standalone, they need to be called as: python3 -m opencos.eda,
60
80
  # and do not work with relative paths. This only works if env OC_ROOT is set or can be found.
61
81
  # 3. If you ran 'source bin/addpath' then you are always using the local (opencos repo)/bin/eda
@@ -64,11 +84,13 @@ def get_eda_exec(command:str=''):
64
84
  # Can we run from OC_ROOT/bin/eda?
65
85
  oc_root = util.get_oc_root()
66
86
  if not oc_root:
67
- util.error(f"Need 'eda' in our path to run 'eda {command}', could not find env OC_ROOT, {eda_path=}, {oc_root=}")
87
+ util.error(f"Need 'eda' in our path to run 'eda {command}', could not find env",
88
+ f"OC_ROOT, {eda_path=}, {oc_root=}")
68
89
  else:
69
90
  bin_eda = os.path.join(oc_root, 'bin', 'eda')
70
91
  if not os.path.exists(bin_eda):
71
- util.error(f"Need 'eda' in our path to run 'eda {command}', cound not find bin/, {eda_path=}, {oc_root=}, {bin_eda=}")
92
+ util.error(f"Need 'eda' in our path to run 'eda {command}', cound not find",
93
+ f"bin/, {eda_path=}, {oc_root=}, {bin_eda=}")
72
94
  else:
73
95
  util.info(f"'eda' not in path, using {bin_eda=} for 'eda' {command} executable")
74
96
  eda_path = os.path.abspath(bin_eda)
@@ -76,9 +98,10 @@ def get_eda_exec(command:str=''):
76
98
  return eda_path
77
99
 
78
100
 
79
- def which_tool(command, config):
80
- '''Returns which tool will be used for a command, given the command_handlers in config dict.'''
81
- from opencos import eda_tool_helper
101
+ def which_tool(command: str, config: dict) -> str:
102
+ '''Returns which tool (str) will be used for a command, given the command_handlers in
103
+
104
+ config dict.'''
82
105
  if config is None:
83
106
  util.error(f'which_tool({command=}) called w/out config')
84
107
  if not command in config.get('command_handler', {}):
@@ -136,9 +159,10 @@ class Tool:
136
159
  self.set_tool_config_from_config()
137
160
 
138
161
  def set_exe(self, config: dict) -> None:
162
+ '''Sets self._EXE based on config'''
139
163
  if self._TOOL and self._TOOL in config.get('auto_tools_order', [{}])[0]:
140
164
  exe = config.get('auto_tools_order', [{}])[0][self._TOOL].get('exe', '')
141
- if exe and type(exe) is list:
165
+ if exe and isinstance(exe, list):
142
166
  exe = exe[0] # pick first
143
167
  if exe and exe != self._EXE:
144
168
  util.info(f'Override for {self._TOOL} using exe {exe}')
@@ -155,9 +179,16 @@ class Tool:
155
179
  return self._VERSION
156
180
 
157
181
  def set_tool_defines(self) -> None:
158
- pass
182
+ '''Derived classes may override, sets any additional defines based on tool.'''
183
+ return
184
+
159
185
 
160
186
  class Command:
187
+ '''Base class for all: eda <command>
188
+
189
+ The Command class should be used when you don't require files, otherwise consider
190
+ CommandDesign.
191
+ '''
161
192
 
162
193
  command_name: str = ''
163
194
 
@@ -170,20 +201,20 @@ class Command:
170
201
  "keep" : False,
171
202
  "force" : False,
172
203
  "fake" : False,
173
- "stop-before-compile": False, # Usually in the self.do_it() method, stop prior to compile/elaborate/simulate
204
+ "stop-before-compile": False,
174
205
  "stop-after-compile": False,
175
- "stop-after-elaborate": False, # Set to True to only run compile + elaboration (aka compile + lint)
206
+ "stop-after-elaborate": False,
176
207
  "lint": False, # Same as stop-after-elaborate
177
208
  "eda-dir" : "eda.work", # all eda jobs go in here
178
209
  "job-name" : "", # this is used to create a certain dir under "eda_dir"
179
- "work-dir" : "", # this can be used to run the job in a certain dir, else it will be <eda-dir>/<job-name> else <eda-dir>/<target>_<command>
180
- "sub-work-dir" : "", # this can be used to name the dir built under <eda-dir>, which seems to be same function as job-name??
210
+ "work-dir" : "", # default is usually <eda-dir>/<job-name> or <eda-dir>/<target>.<cmd>
211
+ "sub-work-dir" : "", # this can be used to name the dir built under <eda-dir>
181
212
  "work-dir-use-target-dir": False,
182
213
  "suffix" : "",
183
214
  "design" : "", # not sure how this relates to top
184
215
  'export': False,
185
- 'export-run': False, # run from the exported location if possible, if not possible run the command in usual place.
186
- 'export-json': False, # generates an export.json suitable for a testrunner, if possible for self.command.
216
+ 'export-run': False, # run from the exported location if possible
217
+ 'export-json': False, # generates an export.json suitable for a testrunner
187
218
  'enable-tags': [],
188
219
  'disable-tags': [],
189
220
  'test-mode': False,
@@ -214,6 +245,7 @@ class Command:
214
245
  self.status = 0
215
246
  self.errors_log_f = None
216
247
 
248
+
217
249
  def error(self, *args, **kwargs) -> None:
218
250
  '''Returns None, child classes can call self.error(..) instead of util.error,
219
251
 
@@ -225,10 +257,16 @@ class Command:
225
257
  if self.args['work-dir']:
226
258
  if not self.errors_log_f:
227
259
  try:
228
- self.errors_log_f = open(
229
- os.path.join(self.args['work-dir'], 'eda.errors.log'), 'w'
260
+ fullpath = os.path.join(self.args['work-dir'], 'eda.errors.log')
261
+ self.errors_log_f = open( # pylint: disable=consider-using-with
262
+ fullpath, 'w', encoding='utf-8'
230
263
  )
231
- except:
264
+ util.artifacts.add(
265
+ name=fullpath,
266
+ typ='text', description='EDA reported errors'
267
+ )
268
+
269
+ except FileNotFoundError:
232
270
  pass
233
271
  if self.errors_log_f:
234
272
  print(
@@ -245,17 +283,29 @@ class Command:
245
283
  util.error(f"command '{self.command_name}' has previous errors")
246
284
  return self.status > 0
247
285
 
248
- def which_tool(self, command:str):
286
+ def which_tool(self, command:str) -> str:
287
+ '''Returns a str for the tool name used for the requested command'''
249
288
  return which_tool(command, config=self.config)
250
289
 
251
- def create_work_dir(self):
290
+ def create_work_dir( # pylint: disable=too-many-branches,too-many-statements
291
+ self
292
+ ) -> str:
293
+ '''Creates the working directory and populates self.args['work-dir']
294
+
295
+ Generally uses ./ self.args['eda-dir'] / <target-name>.<command> /
296
+ however, self.args['job-name'] or ['sub-work-dir'] can override that.
297
+
298
+ Additionally, the work-dir is attempted to be deleted if it already exists
299
+ AND it is beneath the caller's original working directory. We also will
300
+ not delete 'work-dir' that is ./ or $PWD.
301
+ '''
252
302
  util.debug(f"create_work_dir: {self.args['eda-dir']=} {self.args['work-dir']=}")
253
303
  if self.args['work-dir-use-target-dir']:
254
304
  if not self.target_path:
255
305
  self.target_path = '.'
256
- util.info(f"create_work_dir: --work-dir-use-target-dir: using:",
257
- f"{os.path.abspath(self.target_path)}",
258
- f'target={self.target}')
306
+ util.info("create_work_dir: --work-dir-use-target-dir: using:",
307
+ f"{os.path.abspath(self.target_path)}",
308
+ f'target={self.target}')
259
309
  self.args['work-dir'] = self.target_path
260
310
  self.args['sub-work-dir'] = ''
261
311
  return self.args['work-dir']
@@ -263,28 +313,37 @@ class Command:
263
313
  if not os.path.exists(self.args['eda-dir']):
264
314
  util.safe_mkdir(self.args['eda-dir'])
265
315
  util.info(f"create_work_dir: created {self.args['eda-dir']}")
316
+
266
317
  if self.args['design'] == "":
267
318
  if ('top' in self.args) and (self.args['top'] != ""):
268
319
  self.args['design'] = self.args['top']
269
- util.debug(f"create_work_dir: set {self.args['design']=} from {self.args['top']=}, since it was empty")
320
+ util.debug(f"create_work_dir: set {self.args['design']=} from",
321
+ f"{self.args['top']=}, since it was empty")
270
322
  else:
271
323
  self.args['design'] = "design" # generic, i.e. to create work dir "design_upload"
272
- util.debug(f"create_work_dir: set {self.args['design']=} to 'design', since it was empty and we have no top")
324
+ util.debug(f"create_work_dir: set {self.args['design']=} to 'design', since it",
325
+ "was empty and we have no top")
326
+
273
327
  if self.target == "":
274
328
  self.target = self.args['design']
275
329
  util.debug(f"create_work_dir: set {self.target=} from design name, since it was empty")
330
+
276
331
  if self.args['work-dir'] == '':
277
332
  if self.args['sub-work-dir'] == '':
278
333
  if self.args['job-name'] != '':
279
334
  self.args['sub-work-dir'] = self.args['job-name']
280
- util.debug(f"create_work_dir: set {self.args['sub-work-dir']=} from {self.args['job-name']=}, since it was empty")
335
+ util.debug(f"create_work_dir: set {self.args['sub-work-dir']=} from",
336
+ f"{self.args['job-name']=}, since it was empty")
281
337
  else:
282
338
  self.args['sub-work-dir'] = f'{self.target}.{self.command_name}'
283
- util.debug(f"create_work_dir: set {self.args['sub-work-dir']=} from {self.target=} and {self.command_name=}, since it was empty and we have no job-name")
339
+ util.debug(f"create_work_dir: set {self.args['sub-work-dir']=} from",
340
+ f"{self.target=} and {self.command_name=}, since it was empty and",
341
+ "we have no job-name")
284
342
  self.args['work-dir'] = os.path.join(self.args['eda-dir'], self.args['sub-work-dir'])
285
343
  util.debug(f"create_work_dir: set {self.args['work-dir']=}")
344
+
286
345
  keep_file = os.path.join(self.args['work-dir'], "eda.keep")
287
- if (os.path.exists(self.args['work-dir'])):
346
+ if os.path.exists(self.args['work-dir']):
288
347
  if os.path.exists(keep_file) and not self.args['force']:
289
348
  self.error(f"Cannot remove old work dir due to '{keep_file}'")
290
349
  elif os.path.abspath(self.args['work-dir']) in os.getcwd():
@@ -293,16 +352,19 @@ class Command:
293
352
  # --work-dir=$PWD
294
353
  # --work-dir=/some/path/almost/here
295
354
  # Allow it, but preserve the existing directory, we don't want to blow away
296
- # files up-heir from us.
355
+ # files up-hier from us.
297
356
  # Enables support for --work-dir=.
298
- util.info(f"Not removing existing work-dir: '{self.args['work-dir']}' is within {os.getcwd()=}")
357
+ util.info(f"Not removing existing work-dir: '{self.args['work-dir']}' is within",
358
+ f"{os.getcwd()=}")
299
359
  elif str(Path(self.args['work-dir'])).startswith(str(Path('/'))):
300
360
  # Do not allow other absolute path work dirs if it already exists.
301
361
  # This prevents you from --work-dir=~ and eda wipes out your home dir.
302
- self.error(f'Cannot use work-dir={self.args["work-dir"]} starting with absolute path "/"')
362
+ self.error(f'Cannot use work-dir={self.args["work-dir"]} starting with',
363
+ 'fabsolute path "/"')
303
364
  elif str(Path('..')) in str(Path(self.args['work-dir'])):
304
365
  # Do not allow other ../ work dirs if it already exists.
305
- self.error(f'Cannot use work-dir={self.args["work-dir"]} with up-hierarchy ../ paths')
366
+ self.error(f'Cannot use work-dir={self.args["work-dir"]} with up-hierarchy'
367
+ '../ paths')
306
368
  else:
307
369
  # If we made it this far, on a directory that exists, that appears safe
308
370
  # to delete and re-create:
@@ -313,22 +375,40 @@ class Command:
313
375
  else:
314
376
  util.safe_mkdir(self.args['work-dir'])
315
377
  util.debug(f'create_work_dir: created {self.args["work-dir"]}')
316
- if (self.args['keep']):
317
- open(keep_file, 'w').close()
378
+
379
+ if self.args['keep']:
380
+ open(keep_file, 'w', encoding='utf-8').close() # pylint: disable=consider-using-with
318
381
  util.debug(f'create_work_dir: created {keep_file=}')
382
+
383
+ # Set the util.artifacts path with our work-dir:
384
+ util.artifacts.set_artifacts_json_dir(self.args['work-dir'])
385
+
319
386
  return self.args['work-dir']
320
387
 
321
- def exec(self, work_dir, command_list, background=False, stop_on_error=True,
322
- quiet=False, tee_fpath=None, shell=False):
388
+ def artifacts_add(self, name: str, typ: str, description: str) -> None:
389
+ '''Adds a file to util.artifacts, derived classes may override'''
390
+ util.artifacts.add(name=name, typ=typ, description=description)
391
+
392
+
393
+ def exec(self, work_dir: str, command_list: list, background: bool = False,
394
+ stop_on_error: bool = True, quiet: bool = False, tee_fpath: str = '',
395
+ shell: bool = False) -> (str, str, int):
396
+ '''Runs a command via subprocess and returns a tuple of (stderr, stdout, rc)
397
+
398
+ - work_dir: is passed to the subprocess call, so os.chdir(..) is not invoked.
399
+ - command_list: should be a list of individual strings w/out spaces.
400
+ - background: if False, does not print to stdout (but will print to any
401
+ tee_fpath or the global log
402
+ - shell: arg passed to subprocess, defaults to False.
403
+ '''
404
+
323
405
  if not tee_fpath and getattr(command_list, 'tee_fpath', None):
324
406
  tee_fpath = getattr(command_list, 'tee_fpath', '')
325
407
  if not quiet:
326
408
  util.info(f"exec: {' '.join(command_list)} (in {work_dir}, {tee_fpath=})")
327
- original_cwd = util.getcwd()
328
- os.chdir(work_dir)
329
409
 
330
- stdout, stderr, return_code = util.subprocess_run_background(
331
- work_dir=None, # we've already called os.chdir(work_dir).
410
+ stdout, stderr, return_code = subprocess_run_background(
411
+ work_dir=work_dir,
332
412
  command_list=command_list,
333
413
  background=background,
334
414
  fake=self.args.get('fake', False),
@@ -336,16 +416,23 @@ class Command:
336
416
  shell=shell
337
417
  )
338
418
 
339
- os.chdir(original_cwd)
340
419
  if return_code:
341
420
  self.status += return_code
342
- if stop_on_error: self.error(f"exec: returned with error (return code: {return_code})")
343
- else : util.debug(f"exec: returned with error (return code: {return_code})")
421
+ if stop_on_error:
422
+ self.error(f"exec: returned with error (return code: {return_code})")
423
+ else:
424
+ util.debug(f"exec: returned with error (return code: {return_code})")
344
425
  else:
345
426
  util.debug(f"exec: returned without error (return code: {return_code})")
346
427
  return stderr, stdout, return_code
347
428
 
348
- def set_arg(self, key, value):
429
+ def set_arg( # pylint: disable=too-many-branches
430
+ self, key, value
431
+ ) -> None:
432
+ '''Sets self.args[key] with value, and self.modified_args[key]=True
433
+
434
+ Does type checking, handles list type append
435
+ '''
349
436
 
350
437
  # Do some minimal type handling, preserving the type(self.args[key])
351
438
  if key not in self.args:
@@ -378,14 +465,14 @@ class Command:
378
465
  else:
379
466
  self.args[key] = True
380
467
  else:
381
- raise Exception(f'set_arg({key=}, {value=}) bool, {type(cur_value)=} {type(value)=}')
468
+ assert False, f'set_arg({key=}, {value=}) bool {type(cur_value)=} {type(value)=}'
382
469
 
383
470
  elif isinstance(cur_value, int):
384
471
  # if int, attempt to convert string or bool --> int
385
472
  if isinstance(value, (bool, int, str)):
386
473
  self.args[key] = int(value)
387
474
  else:
388
- raise Exception(f'set_arg({key=}, {value=}) int, {type(cur_value)=} {type(value)=}')
475
+ assert False, f'set_arg({key=}, {value=}) int {type(cur_value)=} {type(value)=}'
389
476
 
390
477
 
391
478
  else:
@@ -396,7 +483,9 @@ class Command:
396
483
  util.debug(f'Set arg["{key}"]="{self.args[key]}"')
397
484
 
398
485
 
399
- def get_argparser(self, parser_arg_list=None) -> argparse.ArgumentParser:
486
+ def get_argparser( # pylint: disable=too-many-branches
487
+ self, parser_arg_list=None
488
+ ) -> argparse.ArgumentParser:
400
489
  ''' Returns an argparse.ArgumentParser() based on self.args (dict)
401
490
 
402
491
  If parser_arg_list is not None, the ArgumentParser() is created using only the keys in
@@ -404,8 +493,8 @@ class Command:
404
493
  '''
405
494
 
406
495
  # Preference is --args-with-dashes, which then become parsed.args_with_dashes, b/c
407
- # parsed.args-with-dashes is not legal python. Some of self.args.keys() still have - or _, so
408
- # this will handle both.
496
+ # parsed.args-with-dashes is not legal python. Some of self.args.keys() still have - or _,
497
+ # so this will handle both.
409
498
  # Also, preference is for self.args.keys(), to be str with - dashes
410
499
  parser = argparse.ArgumentParser(prog='eda', add_help=False, allow_abbrev=False)
411
500
  bool_action_kwargs = util.get_argparse_bool_action_kwargs()
@@ -439,19 +528,19 @@ class Command:
439
528
  help_kwargs = {'help': f'{type(value).__name__} default={value}'}
440
529
 
441
530
 
442
- # It's important to set the default=None on these, except for list types where default is []
443
- # If the parsed Namespace has values set to None or [], we do not update. This means that as deps
444
- # are processed that have args set, they cannot override the top level args that were already set.
445
- # nor be overriden by defaults.
446
- if type(value) is bool:
447
- # For bool, support --key and --no-key with this action=argparse.BooleanOptionalAction.
448
- # Note, this means you cannot use --some-bool=True, or --some-bool=False, has to be --some-bool
449
- # or --no-some-bool.
531
+ # It's important to set the default=None on these, except for list types where default
532
+ # is []. If the parsed Namespace has values set to None or [], we do not update. This
533
+ # means that as deps are processed that have args set, they cannot override the top
534
+ # level args that were already set, nor be overriden by defaults.
535
+ if isinstance(value, bool):
536
+ # For bool, support --key and --no-key with action=argparse.BooleanOptionalAction.
537
+ # Note, this means you cannot use --some-bool=True, or --some-bool=False, has to
538
+ # be --some-bool or --no-some-bool.
450
539
  parser.add_argument(
451
540
  *arguments, default=None, **bool_action_kwargs, **help_kwargs)
452
- elif type(value) is list:
541
+ elif isinstance(value, list):
453
542
  parser.add_argument(*arguments, default=value, action='append', **help_kwargs)
454
- elif type(value) in [int, str]:
543
+ elif isinstance(value, (int, str)):
455
544
  parser.add_argument(*arguments, default=value, type=type(value), **help_kwargs)
456
545
  elif value is None:
457
546
  parser.add_argument(*arguments, default=None, **help_kwargs)
@@ -462,10 +551,10 @@ class Command:
462
551
 
463
552
 
464
553
  def run_argparser_on_list(
465
- self, tokens: list, parser_arg_list=None, apply_parsed_args:bool=True
554
+ self, tokens: list, parser_arg_list=None, apply_parsed_args: bool = True
466
555
  ) -> (dict, list):
467
- ''' Creates an argparse.ArgumentParser() for all the keys in self.args, and attempts to parse
468
- from the provided list. Parsed args are applied to self.args.
556
+ ''' Creates an argparse.ArgumentParser() for all the keys in self.args, and attempts to
557
+ parse from the provided list. Parsed args are applied to self.args.
469
558
 
470
559
  Returns a tuple of (parsed argparse.Namespace obj, list of unparsed args)
471
560
 
@@ -479,17 +568,16 @@ class Command:
479
568
  parsed, unparsed = parser.parse_known_args(tokens + [''])
480
569
  unparsed = list(filter(None, unparsed))
481
570
  except argparse.ArgumentError:
482
- self.error(f'problem {command_name=} attempting to parse_known_args for {tokens=}')
571
+ self.error(f'problem {self.command_name=} attempting to parse_known_args for {tokens=}')
483
572
 
484
573
  parsed_as_dict = vars(parsed)
485
574
 
486
575
  args_to_be_applied = {}
487
576
 
488
- for key,value in parsed_as_dict.items():
489
- # key should have _ instead of POSIX dashes, but we still support dashes like self.args['build-file'],
490
- # etc.
577
+ for key, value in parsed_as_dict.items():
578
+ # key currently has _ instead of POSIX dashes, so we convert to dashes, for
579
+ # most self.args, like self.args['build-file'], etc.
491
580
  if key not in self.args and '_' in key:
492
- # try with dashes instead of _
493
581
  key = key.replace('_', '-')
494
582
  assert key in self.args, f'{key=} not in {self.args=}'
495
583
 
@@ -501,7 +589,8 @@ class Command:
501
589
  return parsed, unparsed
502
590
 
503
591
 
504
- def apply_args_from_dict(self, args_to_be_applied:dict) -> list:
592
+ def apply_args_from_dict(self, args_to_be_applied: dict) -> list:
593
+ '''Given a dict of key/value for self.args, apply them to self.args'''
505
594
  util.debug('apply_args_from_dict() -- called by:',
506
595
  f'{self.command_name=}, {self.__class__.__name__=},',
507
596
  f'{args_to_be_applied=}')
@@ -523,7 +612,7 @@ class Command:
523
612
  f"with different cur value (cur value={self.args.get(key, None)})")
524
613
  continue
525
614
  if self.args[key] != value:
526
- util.debug(f"Command.run_argparser_on_list - setting set_arg b/c",
615
+ util.debug("Command.run_argparser_on_list - setting set_arg b/c",
527
616
  f" argparse -- {key=} {value=} (cur value={self.args[key]})")
528
617
  self.set_arg(key, value) # Note this has special handling for lists already.
529
618
  self.modified_args[key] = True
@@ -541,7 +630,7 @@ class Command:
541
630
  _, unparsed = self.run_argparser_on_list(tokens)
542
631
  if process_all and len(unparsed) > 0:
543
632
  self.error(f"Didn't understand argument: '{unparsed=}' in",
544
- f" {self.command_name=} context")
633
+ f" {self.command_name=} context, {pwd=}")
545
634
 
546
635
  return unparsed
547
636
 
@@ -574,27 +663,33 @@ class Command:
574
663
 
575
664
  Allows command handlers to make thier own customizations from --config-yaml=YAML.
576
665
  '''
577
- pass
666
+ return
578
667
 
579
668
  def update_tool_config(self) -> None:
580
669
  '''Returns None. Hook for classes like CommandSim to make tool specific overrides.'''
581
- pass
670
+ return
582
671
 
583
672
  def write_eda_config_and_args(self):
673
+ '''Attempts to write eda_output_config.yml to our work-dir'''
584
674
  if not self.args.get('work-dir', None):
585
675
  util.warning(f'Ouput work-dir not set, saving ouput eda_config to {os.getcwd()}')
586
- util.write_eda_config_and_args(dirpath=self.args.get('work-dir', os.getcwd()),
587
- command_obj_ref=self)
676
+ eda_config.write_eda_config_and_args(
677
+ dirpath=self.args.get('work-dir', os.getcwd()), command_obj_ref=self
678
+ )
588
679
 
589
680
  def is_export_enabled(self) -> bool:
590
- # check if any self.args['export'] is set in any way (but not set to False
591
- # or empty list)
592
- return any([arg.startswith('export') and v for arg,v in self.args.items()])
681
+ '''Returns True if any self.args['export'] is set in any way (but not set to False
682
+ or empty list)'''
683
+ return any(arg.startswith('export') and v for arg,v in self.args.items())
593
684
 
594
685
  def run(self) -> None:
686
+ '''Alias for do_it(self)'''
595
687
  self.do_it()
596
688
 
597
689
  def do_it(self) -> None:
690
+ '''Main entrypoint of running a command, usually called by process_tokens.
691
+
692
+ process_tokens(..) is the starting entrypoint from eda.py'''
598
693
  self.write_eda_config_and_args()
599
694
  self.error(f"No tool bound to command '{self.command_name}', you",
600
695
  " probably need to setup tool, or use '--tool <name>'")
@@ -610,7 +705,9 @@ class Command:
610
705
  self.set_tool_defines()
611
706
 
612
707
 
613
- def help(self, tokens: list = []) -> None:
708
+ def help( # pylint: disable=dangerous-default-value,too-many-branches
709
+ self, tokens: list = []
710
+ ) -> None:
614
711
  '''Since we don't quite follow standard argparger help()/usage(), we'll format our own
615
712
 
616
713
  if self.args_help has additional help information.
@@ -618,7 +715,7 @@ class Command:
618
715
 
619
716
  # Indent long lines (>100) to indent=56 (this is where we leave off w/ {vstr:12} below.
620
717
  def indent_me(text:str):
621
- return util.indent_wrap_long_text(text, width=100, indent=56)
718
+ return indent_wrap_long_text(text, width=100, indent=56)
622
719
 
623
720
  util.info('Help:')
624
721
  # using bare 'print' here, since help was requested, avoids --color and --quiet
@@ -637,14 +734,14 @@ class Command:
637
734
  lines.append(f"Generic help for command='{self.command_name}'"
638
735
  f" (using '{self.__class__.__name__}')")
639
736
  else:
640
- lines.append(f"Generic help (from class Command):")
737
+ lines.append("Generic help (from class Command):")
641
738
 
642
739
  # Attempt to run argparser on args, but don't error if it fails.
643
740
  unparsed = []
644
741
  if tokens:
645
742
  try:
646
743
  _, unparsed = self.run_argparser_on_list(tokens=tokens)
647
- except:
744
+ except Exception:
648
745
  pass
649
746
 
650
747
  for k in sorted(self.args.keys()):
@@ -653,13 +750,13 @@ class Command:
653
750
  khelp = self.args_help.get(k, '')
654
751
  if khelp:
655
752
  khelp = f' - {khelp}'
656
- if type(v) == bool :
753
+ if isinstance(v, bool):
657
754
  lines.append(indent_me(f" --{k:20} : boolean : {vstr:12}{khelp}"))
658
- elif type(v) == int:
755
+ elif isinstance(v, int):
659
756
  lines.append(indent_me(f" --{k:20} : integer : {vstr:12}{khelp}"))
660
- elif type(v) == list:
757
+ elif isinstance(v, list):
661
758
  lines.append(indent_me(f" --{k:20} : list : {vstr:12}{khelp}"))
662
- elif type(v) == str:
759
+ elif isinstance(v, str):
663
760
  vstr = "'" + v + "'"
664
761
  lines.append(indent_me(f" --{k:20} : string : {vstr:12}{khelp}"))
665
762
  else:
@@ -672,7 +769,10 @@ class Command:
672
769
  print(f'Unparsed args: {unparsed}')
673
770
 
674
771
 
675
- class CommandDesign(Command):
772
+ class CommandDesign(Command): # pylint: disable=too-many-instance-attributes
773
+ '''CommandDesign is the eda base class for command handlers that need to track files.
774
+
775
+ This is the base class for CommandSim, CommandSynth, and others.'''
676
776
 
677
777
  # Used by for DEPS work_dir_add_srcs@ commands, by class methods:
678
778
  # update_file_lists_for_work_dir(..), and resolve_target(..)
@@ -693,11 +793,16 @@ class CommandDesign(Command):
693
793
  self.args_help.update({
694
794
  'seed': 'design seed, default is 31-bit non-zero urandom',
695
795
  'top': 'TOP level verilog/SV module or VHDL entity for this target',
696
- 'all-sv': ('Maintain .sv and .v in single file list.'
697
- ' False: .sv flist separate from .v flist and separate compile(s)'
698
- ' True: .sv and .v files compiled together if possible'),
699
- 'unprocessed-plusargs': 'Args that began with +, but were not +define+ or +incdir+, +<name>, ' \
700
- + ' or +<name>=<value>. These become tool dependent, for example "sim" commands will treat as sim-plusargs',
796
+ 'all-sv': (
797
+ 'Maintain .sv and .v in single file list.'
798
+ ' False: .sv flist separate from .v flist and separate compile(s)'
799
+ ' True: .sv and .v files compiled together if possible'
800
+ ),
801
+ 'unprocessed-plusargs': (
802
+ 'Args that began with +, but were not +define+ or +incdir+, +<name>, '
803
+ ' or +<name>=<value>. These become tool dependent, for example "sim" commands will'
804
+ ' treat as sim-plusargs'
805
+ ),
701
806
  })
702
807
  self.defines = {}
703
808
  self.incdirs = []
@@ -715,12 +820,18 @@ class CommandDesign(Command):
715
820
  for (d,v) in self.config.get('defines', {}).items():
716
821
  self.defines[d] = v
717
822
 
718
- self.cached_deps = {} # key = abspath of DEPS markup file, has sub-dicts for 'data' and 'line_numbers'.
823
+ # cached_deps: key = abspath of DEPS markup file, value is a dict with
824
+ # keys 'data' and 'line_numbers'
825
+ self.cached_deps = {}
719
826
  self.targets_dict = {} # key = targets that we've already processed in DEPS files
720
827
  self.last_added_source_file_inferred_top = ''
721
828
 
722
- def run_dep_commands(self):
723
- # Run any shell@ commands from DEPS files
829
+
830
+ def run_dep_commands(self) -> None:
831
+ '''Run shell/peakrdl style commands from DEPS files
832
+
833
+ These are deferred to maintain the deps ordering, and run in that order.
834
+ '''
724
835
  self.run_dep_shell_commands()
725
836
  # Update any work_dir_add_srcs@ in our self.files, self.files_v, etc, b/c
726
837
  # self.args['work-dir'] now exists.
@@ -728,7 +839,10 @@ class CommandDesign(Command):
728
839
  # Link any non-sources to our work-dir:
729
840
  self.update_non_source_files_in_work_dir()
730
841
 
731
- def run_dep_shell_commands(self):
842
+
843
+ def run_dep_shell_commands(self) -> None:
844
+ '''Specifically runs shell command from DEPS files'''
845
+
732
846
  # Runs from self.args['work-dir']
733
847
  all_cmds_lists = []
734
848
 
@@ -747,7 +861,8 @@ class CommandDesign(Command):
747
861
  log_fnames_count.update({target_node: lognum + 1})
748
862
  all_cmds_lists += [
749
863
  [], # blank line
750
- # comment, where it came from, log to {node}__shell_{lognum}.log (or tee name from DEPS.yml)
864
+ # comment, where it came from, log to {node}__shell_{lognum}.log
865
+ # (or tee name from DEPS.yml)
751
866
  [f'# command {i}: target: {d["target_path"]} : {target_node} --> {log}'],
752
867
  ]
753
868
  if not run_from_work_dir:
@@ -761,10 +876,12 @@ class CommandDesign(Command):
761
876
 
762
877
  d['exec_list'] = clist # update to tee_fpath is set.
763
878
 
764
- util.write_shell_command_file(dirpath=self.args['work-dir'], filename='pre_compile_dep_shell_commands.sh',
765
- command_lists=all_cmds_lists)
879
+ util.write_shell_command_file(
880
+ dirpath=self.args['work-dir'], filename='pre_compile_dep_shell_commands.sh',
881
+ command_lists=all_cmds_lists
882
+ )
766
883
 
767
- for i,d in enumerate(self.dep_shell_commands):
884
+ for i, d in enumerate(self.dep_shell_commands):
768
885
  util.info(f'run_dep_shell_commands {i=}: {d=}')
769
886
  clist = util.ShellCommandList(d['exec_list'])
770
887
  tee_fpath=clist.tee_fpath
@@ -782,16 +899,23 @@ class CommandDesign(Command):
782
899
  self.exec(run_from_dir, clist, tee_fpath=tee_fpath,
783
900
  shell=self.config.get('deps_subprocess_shell', False))
784
901
 
785
- def update_file_lists_for_work_dir(self):
786
- if len(self.dep_work_dir_add_srcs) == 0:
902
+
903
+ def update_file_lists_for_work_dir(self) -> None:
904
+ '''Handles any source files that were creating by "shell" style commands in the
905
+
906
+ work-dir, these need to be added via self.add_file(..) in the correct order
907
+ of the dependencies. Basically, any files that were left with @EDA-WORK-DIR@ prefix
908
+ need to be patched and added.
909
+ '''
910
+ if not self.dep_work_dir_add_srcs:
787
911
  return
788
912
 
789
- # If we encounter any @EDA-WORK_DIR@some_file.v in self.files, self.files_v, etc, then replace it with:
790
- # self.args['work-dir'] / some_file.v:
913
+ # If we encounter any @EDA-WORK_DIR@some_file.v in self.files, self.files_v, etc,
914
+ # then replace it with: self.args['work-dir'] / some_file.v:
791
915
  _work_dir_add_srcs_path_string_len = len(self._work_dir_add_srcs_path_string)
792
916
  work_dir_abspath = os.path.abspath(self.args['work-dir'])
793
- for key in list(self.files.keys()): # list so it's not an iterator, we're updating self.files.
794
- if type(key) is str and key.startswith(self._work_dir_add_srcs_path_string):
917
+ for key in list(self.files.keys()): # list so it's not an iterator, updates self.files.
918
+ if isinstance(key, str) and key.startswith(self._work_dir_add_srcs_path_string):
795
919
  new_key = os.path.join(work_dir_abspath, key[_work_dir_add_srcs_path_string_len :])
796
920
  self.files.pop(key)
797
921
  self.files[new_key] = True
@@ -800,12 +924,23 @@ class CommandDesign(Command):
800
924
  self.files_sdc]
801
925
  for my_file_list in my_file_lists_list:
802
926
  for i,value in enumerate(my_file_list):
803
- if value and type(value) is str and value.startswith(self._work_dir_add_srcs_path_string):
804
- new_value = os.path.join(work_dir_abspath, value[_work_dir_add_srcs_path_string_len :])
927
+ if value and isinstance(value, str) and \
928
+ value.startswith(self._work_dir_add_srcs_path_string):
929
+ new_value = os.path.join(
930
+ work_dir_abspath, value[_work_dir_add_srcs_path_string_len :]
931
+ )
805
932
  my_file_list[i] = new_value
806
933
  util.debug(f"file lists: replaced {value} with {new_value}")
807
934
 
808
- def update_non_source_files_in_work_dir(self):
935
+
936
+ def update_non_source_files_in_work_dir(self) -> None:
937
+ '''For non-source files (that are tracked as 'reqs' in DEPS markup files) these
938
+
939
+ need to be copied or linked to the work-dir. For example, if some SV assumes it
940
+ can $readmemh('file_that_is_here.txt') but we're running out of work-dir. Linking
941
+ is the easy work-around vs trying to run-in-place of all SV files.
942
+ '''
943
+
809
944
  for fname in self.files_non_source:
810
945
  _, leaf_fname = os.path.split(fname)
811
946
  destfile = os.path.join(self.args['work-dir'], leaf_fname)
@@ -815,16 +950,25 @@ class CommandDesign(Command):
815
950
  util.info(f'{fname=} {self.files_caller_info=}')
816
951
  self.error(f'Non-source file (reqs?) {relfname=} does not exist from {caller_info}')
817
952
  elif not os.path.exists(destfile):
818
- util.debug(f'updating non-source file to work-dir: Linked {fname=} to {destfile=}, from {caller_info}')
953
+ util.debug(f'updating non-source file to work-dir: Linked {fname=} to {destfile=},',
954
+ f'from {caller_info}')
819
955
  if sys.platform == "win32":
820
956
  shutil.copyfile(fname, destfile) # On Windows, fall back to copying
821
957
  else:
822
958
  os.symlink(src=fname, dst=destfile)
823
959
 
824
- def get_top_name(self, name):
960
+ @staticmethod
961
+ def get_top_name(name: str) -> str:
962
+ '''Attempt to get the 'top' module name from a file, such as path/to/mine.sv will
963
+
964
+ return "mine"'''
965
+ # TODO(drew): Use the helper method in util for this instead to peek in file contents?
825
966
  return os.path.splitext(os.path.basename(name))[0]
826
967
 
827
- def process_plusarg(self, plusarg: str, pwd: str = os.getcwd()):
968
+
969
+ def process_plusarg( # pylint: disable=too-many-branches
970
+ self, plusarg: str, pwd: str = os.getcwd()
971
+ ) -> None:
828
972
  '''Retuns str, parses a +define+, +incdir+, +key=value str; adds to internal.
829
973
 
830
974
  Adds to self.defines, self.incdirs,
@@ -836,12 +980,12 @@ class CommandDesign(Command):
836
980
  # args that come from shlex.quote(token), such as:
837
981
  # token = '\'+define+OC_ROOT="/foo/bar/opencos"\''
838
982
  # So we strip all outer ' or " on the plusarg:
839
- plusarg = util.strip_outer_quotes(plusarg)
983
+ plusarg = strip_outer_quotes(plusarg)
840
984
  if not pwd:
841
985
  pwd = ''
842
986
 
843
987
  if plusarg.startswith('+define+'):
844
- plusarg = plusarg.lstrip('+define+')
988
+ plusarg = plusarg[len('+define+'):]
845
989
  m = re.match(r'^(\w+)$', plusarg)
846
990
  if m:
847
991
  k = m.group(1)
@@ -849,11 +993,12 @@ class CommandDesign(Command):
849
993
  util.debug(f"Defined {k}")
850
994
  return None
851
995
  m = re.match(r'^(\w+)\=(\S+)$', plusarg)
852
- if not m: m = re.match(r'^(\w+)\=(\"[^\"]*\")$', plusarg)
996
+ if not m:
997
+ m = re.match(r'^(\w+)\=(\"[^\"]*\")$', plusarg)
853
998
  if m:
854
999
  k = m.group(1)
855
1000
  v = m.group(2)
856
- if v and type(v) is str:
1001
+ if v and isinstance(v, str):
857
1002
  if v.startswith('%PWD%/'):
858
1003
  v = v.replace('%PWD%', os.path.abspath(pwd))
859
1004
  if v.startswith('%SEED%'):
@@ -865,7 +1010,7 @@ class CommandDesign(Command):
865
1010
  return None
866
1011
 
867
1012
  if plusarg.startswith('+incdir+'):
868
- plusarg = plusarg.lstrip('+incdir+')
1013
+ plusarg = plusarg[len('+incdir+'):]
869
1014
  m = re.match(r'^(\S+)$', plusarg)
870
1015
  if m:
871
1016
  incdir = m.group(1)
@@ -881,40 +1026,59 @@ class CommandDesign(Command):
881
1026
  if not self.config.get('bare_plusarg_supported', False):
882
1027
  self.error(f"bare plusarg(s) are not supported: {plusarg}'")
883
1028
  return None
884
- elif plusarg not in self.args['unprocessed-plusargs']:
1029
+ if plusarg not in self.args['unprocessed-plusargs']:
885
1030
  self.args['unprocessed-plusargs'].append(plusarg)
886
1031
  # For anything added to unprocessed-plusarg, we have to return it, to let
887
1032
  # derived classes have the option to handle it
888
1033
  return plusarg
889
- else:
890
- self.error(f"Didn't understand +plusarg: '{plusarg}'")
1034
+
1035
+ self.error(f"Didn't understand +plusarg: '{plusarg}'")
891
1036
  return None
892
1037
 
893
1038
 
894
- def append_shell_commands(self, cmds : list):
895
- # Each entry in cmds (list) should be a dict with keys ['target_node', 'target_path', 'exec_list']
1039
+ def append_shell_commands(self, cmds : list) -> None:
1040
+ ''' Given a cmds (list), where each item is a dict with:
1041
+
1042
+ { 'target_node': str,
1043
+ 'target_path': str,
1044
+ 'exec_list': list
1045
+ }
1046
+ add to this class's dep_shell_commands for deferred processing
1047
+ '''
1048
+
896
1049
  for entry in cmds:
897
- if entry is None or type(entry) is not dict:
1050
+ if entry is None or not isinstance(entry, dict):
898
1051
  continue
899
1052
  if entry in self.dep_shell_commands:
900
- # we've already run this exact command (target node, target path, exec list), don't run it
901
- # again
1053
+ # we've already run this exact command (target node, target path, exec list),
1054
+ # don't run it again
902
1055
  continue
903
1056
 
904
1057
  assert 'exec_list' in entry, f'{entry=}'
905
1058
  util.debug(f'adding - dep_shell_command: {entry=}')
906
1059
  self.dep_shell_commands.append(entry)
907
1060
 
908
- def append_work_dir_add_srcs(self, add_srcs: list, caller_info: str):
909
- # Each entry in add_srcs (list) should be a dict with keys ['target_node', 'target_path', 'file_list']
1061
+
1062
+ def append_work_dir_add_srcs(self, add_srcs: list, caller_info: str) -> None:
1063
+ '''Given add_srcs (list), where each item is a dict with:
1064
+
1065
+ { 'target_node': str,
1066
+ 'target_path': str,
1067
+ 'file_list': list
1068
+ }
1069
+
1070
+ adds files to set self.dep_work_dir_add_src, and call add_file on it, basically
1071
+ resolving adding source files that were generated and residing in the work-dir.
1072
+ '''
910
1073
  for entry in add_srcs:
911
- if entry is None or type(entry) is not dict:
1074
+ if entry is None or not isinstance(entry, dict):
912
1075
  continue
913
1076
 
914
1077
  work_dir_files = entry['file_list']
915
1078
  for filename in work_dir_files:
916
- # Unfortunately, self.args['work-dir'] doesn't exist yet and hasn't been set, so we'll add these
917
- # files as '@EDA-WORK_DIR@' + filename, and have to replace the EDA-WORK_DIR@ string later in our flow.
1079
+ # Unfortunately, self.args['work-dir'] doesn't exist yet and hasn't been set,
1080
+ # so we'll add these files as '@EDA-WORK_DIR@' + filename, and have to replace
1081
+ # the EDA-WORK_DIR@ string later in our flow.
918
1082
  filename_use = self._work_dir_add_srcs_path_string + filename
919
1083
  dep_key_tuple = (
920
1084
  entry['target_path'],
@@ -929,14 +1093,25 @@ class CommandDesign(Command):
929
1093
  self.dep_work_dir_add_srcs.add(dep_key_tuple) # add to set()
930
1094
  elif dep_key_tuple not in self.dep_work_dir_add_srcs:
931
1095
  # we've already added the file so this dep was skipped for this one file.
932
- util.warning(f'work_dir_add_srcs@ {dep_key_tuple=} but {filename_use=}' \
933
- + 'is already in self.files (duplicate dependency on generated file?)')
1096
+ util.warning(f'work_dir_add_srcs@ {dep_key_tuple=} but {filename_use=}',
1097
+ 'is already in self.files (duplicate dependency on generated',
1098
+ 'file?)')
934
1099
 
935
1100
 
936
- def resolve_target(self, target, no_recursion=False, caller_info=''):
937
- util.debug("Entered resolve_target(%s)" % (target))
938
- # self.target is a name we grab for the job (i.e. for naming work dir etc). we don't want the path prefix.
939
- # TODO: too messy -- there's also a self.target, args['job-name'], args['work-dir'], args['design'], args['top'], args['sub-work-dir'] ...
1101
+ def resolve_target(
1102
+ self, target: str, no_recursion: bool = False, caller_info: str = ''
1103
+ ) -> bool:
1104
+ '''Returns True if target is found. Entry point for resolving a CLI path based target name.
1105
+
1106
+ Will recursively call resolve_target_core, looking for a filename existing
1107
+ or DEPS markup file in that path and a target (key) in the markup table.
1108
+
1109
+ This will populate all source file dependencies, commands, and other DEPS markup
1110
+ features as the target is resolved.
1111
+ '''
1112
+ util.debug(f"Entered resolve_target({target})")
1113
+ # self.target is a name we grab for the job (i.e. for naming work dir etc). we don't want
1114
+ # the path prefix.
940
1115
 
941
1116
  self.target_path, self.target = os.path.split(target)
942
1117
 
@@ -950,7 +1125,7 @@ class CommandDesign(Command):
950
1125
  # If the target is a file (we're at the root here processing CLI arg tokens)
951
1126
  # and that file exists and has an extension, then there's no reason to go looking
952
1127
  # in DEPS files, add the file and return True.
953
- file_base, file_ext = os.path.splitext(fpath)
1128
+ _, file_ext = os.path.splitext(fpath)
954
1129
  if forced_extension or file_ext:
955
1130
  self.add_file(fpath, caller_info=caller_info,
956
1131
  forced_extension=forced_extension)
@@ -958,23 +1133,25 @@ class CommandDesign(Command):
958
1133
 
959
1134
  return self.resolve_target_core(target, no_recursion, caller_info)
960
1135
 
961
- def resolve_target_core(self, target, no_recursion, caller_info=''):
962
- util.debug("Entered resolve_target_core(%s)" % (target))
1136
+ def resolve_target_core( # pylint: disable=too-many-locals,too-many-branches
1137
+ self, target: str, no_recursion: bool, caller_info: str = ''
1138
+ ) -> bool:
1139
+ '''Returns True if target is found. recursive point for resolving path or DEPS markup
1140
+ target names.'''
1141
+
1142
+ util.debug(f"Entered resolve_target_core({target=})")
963
1143
  found_target = False
964
- util.debug("Starting to resolve target '%s'" % (target))
965
1144
  target_path, target_node = os.path.split(target)
966
1145
 
967
- deps = None
968
- data = None
1146
+ deps, data, deps_file = None, None, None
969
1147
  found_deps_file = False
970
1148
 
971
1149
  if self.config['deps_markup_supported']:
972
- deps = deps_helpers.DepsFile(
1150
+ deps = DepsFile(
973
1151
  command_design_ref=self, target_path=target_path, cache=self.cached_deps
974
1152
  )
975
1153
  deps_file = deps.deps_file
976
1154
  data = deps.data
977
- data_only_lines = deps.line_numbers
978
1155
 
979
1156
  # Continue if we have data, otherwise look for files other than DEPS.<yml|yaml>
980
1157
  if data is not None:
@@ -987,7 +1164,7 @@ class CommandDesign(Command):
987
1164
 
988
1165
  # For convenience, use an external class for this DEPS.yml table/dict
989
1166
  # This could be re-used for any markup DEPS.json, DEPS.toml, DEPS.py, etc.
990
- deps_processor = deps_helpers.DepsProcessor(
1167
+ deps_processor = DepsProcessor(
991
1168
  command_design_ref = self,
992
1169
  deps_entry = entry,
993
1170
  target = target,
@@ -1008,10 +1185,11 @@ class CommandDesign(Command):
1008
1185
  # Recurse on the returned deps (ordered list), if they haven't already been traversed.
1009
1186
  for x in deps_targets_to_resolve:
1010
1187
  caller_info = deps.gen_caller_info(target_node)
1011
- if x and type(x) is tuple:
1188
+ if x and isinstance(x, tuple):
1012
1189
  # if deps_processor.process_deps_entry() gave us a tuple, it's an
1013
1190
  # unprocessed 'command' that we kept in order until now. Append it.
1014
- assert len(x) == 2, f'command tuple {x=} must be len 2, {target_node=} {deps_file=}'
1191
+ assert len(x) == 2, \
1192
+ f'command tuple {x=} must be len 2, {target_node=} {deps_file=}'
1015
1193
  shell_commands_list, work_dir_add_srcs_list = x
1016
1194
  self.append_shell_commands( cmds=shell_commands_list )
1017
1195
  self.append_work_dir_add_srcs( add_srcs=work_dir_add_srcs_list,
@@ -1025,32 +1203,40 @@ class CommandDesign(Command):
1025
1203
  forced_extension=forced_extension)
1026
1204
  else:
1027
1205
  util.debug(f' ... Calling resolve_target_core({x=})')
1028
- found_target |= self.resolve_target_core( x, no_recursion, caller_info=caller_info)
1206
+ found_target |= self.resolve_target_core(
1207
+ x, no_recursion, caller_info=caller_info
1208
+ )
1029
1209
 
1030
1210
 
1031
1211
  # Done with DEPS.yml if it existed.
1032
1212
 
1033
1213
  if not found_target:
1034
- util.debug("Haven't been able to resolve %s via DEPS" % (target))
1214
+ util.debug(f"Haven't been able to resolve {target=} via DEPS")
1035
1215
  known_file_extensions_for_source = []
1036
- for x in ['verilog', 'systemverilog', 'vhdl', 'cpp']:
1037
- known_file_extensions_for_source += self.config.get('file_extensions', {}).get(x, [])
1216
+ for x in ('verilog', 'systemverilog', 'vhdl', 'cpp'):
1217
+ known_file_extensions_for_source += self.config.get(
1218
+ 'file_extensions', {}).get(x, [])
1038
1219
  for e in known_file_extensions_for_source:
1039
1220
  try_file = target + e
1040
- util.debug("Looking for %s" % (try_file))
1221
+ util.debug(f"Looking for file {try_file}")
1041
1222
  if os.path.exists(try_file):
1042
- self.add_file(try_file, caller_info=str('n/a::' + target + '::n/a'))
1223
+ self.add_file(try_file, caller_info=f'n/a::{target}::n/a')
1043
1224
  found_target = True
1044
1225
  break # move on to the next target
1045
1226
  if not found_target: # if STILL not found_this_target...
1046
- self.error("Unable to resolve target '%s'" % (target))
1227
+ self.error(f"Unable to resolve {target=}")
1047
1228
 
1048
1229
  # if we've found any target since being called, it means we found the one we were called for
1049
1230
  return found_target
1050
1231
 
1051
- def add_file(self, filename, use_abspath=True, add_to_non_sources=False,
1052
- caller_info:str='', forced_extension:str=''):
1053
- file_base, file_ext = os.path.splitext(filename)
1232
+ def add_file(
1233
+ self, filename: str, use_abspath: bool = True, add_to_non_sources: bool = False,
1234
+ caller_info: str = '', forced_extension: str = ''
1235
+ ) -> str:
1236
+ '''Given a filename, add it to one of self.files_sv or similar lists
1237
+
1238
+ based on file extension or prefix directive.'''
1239
+ _, file_ext = os.path.splitext(filename)
1054
1240
  if use_abspath:
1055
1241
  file_abspath = os.path.abspath(filename)
1056
1242
  else:
@@ -1058,8 +1244,8 @@ class CommandDesign(Command):
1058
1244
 
1059
1245
 
1060
1246
  if file_abspath in self.files:
1061
- util.debug("Not adding file %s, already have it" % (file_abspath))
1062
- return
1247
+ util.debug(f"Not adding file {file_abspath}, already have it")
1248
+ return ''
1063
1249
 
1064
1250
  known_file_ext_dict = self.config.get('file_extensions', {})
1065
1251
  v_file_ext_list = known_file_ext_dict.get('verilog', [])
@@ -1082,34 +1268,43 @@ class CommandDesign(Command):
1082
1268
 
1083
1269
  if add_to_non_sources:
1084
1270
  self.files_non_source.append(file_abspath)
1085
- util.debug("Added non-source file file %s as %s" % (filename, file_abspath))
1271
+ util.debug(f"Added non-source file file {filename} as {file_abspath}")
1086
1272
  elif file_ext in v_file_ext_list and not self.args['all-sv']:
1087
1273
  self.files_v.append(file_abspath)
1088
- util.debug("Added Verilog file %s as %s" % (filename, file_abspath))
1089
- elif file_ext in sv_file_ext_list or ((file_ext in v_file_ext_list) and self.args['all-sv']):
1274
+ util.debug(f"Added Verilog file {filename} as {file_abspath}")
1275
+ elif file_ext in sv_file_ext_list or \
1276
+ ((file_ext in v_file_ext_list) and self.args['all-sv']):
1090
1277
  self.files_sv.append(file_abspath)
1091
- util.debug("Added SystemVerilog file %s as %s" % (filename, file_abspath))
1278
+ util.debug(f"Added SystemVerilog file {filename} as {file_abspath}")
1092
1279
  elif file_ext in vhdl_file_ext_list:
1093
1280
  self.files_vhd.append(file_abspath)
1094
- util.debug("Added VHDL file %s as %s" % (filename, file_abspath))
1281
+ util.debug(f"Added VHDL file {filename} as {file_abspath}")
1095
1282
  elif file_ext in cpp_file_ext_list:
1096
1283
  self.files_cpp.append(file_abspath)
1097
- util.debug("Added C++ file %s as %s" % (filename, file_abspath))
1284
+ util.debug(f"Added C++ file {filename} as {file_abspath}")
1098
1285
  elif file_ext in sdc_file_ext_list:
1099
1286
  self.files_sdc.append(file_abspath)
1100
- util.debug("Added SDC file %s as %s" % (filename, file_abspath))
1287
+ util.debug(f"Added SDC file {filename} as {file_abspath}")
1101
1288
  else:
1102
1289
  # unknown file extension. In these cases we link the file to the working directory
1103
- # so it is available (for example, a .mem file that is expected to exist with relative path)
1290
+ # so it is available (for example, a .mem file that is expected to exist with relative
1291
+ # path)
1104
1292
  self.files_non_source.append(file_abspath)
1105
- util.debug("Added non-source file %s as %s" % (filename, file_abspath))
1293
+ util.debug(f"Added non-source file {filename} as {file_abspath}")
1106
1294
 
1107
1295
  self.files[file_abspath] = True
1108
1296
  self.files_caller_info[file_abspath] = caller_info
1109
1297
  return file_abspath
1110
1298
 
1111
- def process_tokens(self, tokens: list, process_all: bool = True,
1112
- pwd: str = os.getcwd()) -> list:
1299
+ def process_tokens( # pylint: disable=too-many-locals,too-many-branches,too-many-statements
1300
+ self, tokens: list, process_all: bool = True,
1301
+ pwd: str = os.getcwd()
1302
+ ) -> list:
1303
+ '''Returns a list of unparsed args (self.args did not have these keys present)
1304
+
1305
+ Entrypoint from eda.py calling a command handler obj.process_tokens(args).
1306
+ Dervied classes are expected to call this parent process_tokens to parse and handle
1307
+ their self.args dict'''
1113
1308
 
1114
1309
  util.debug(f'CommandDesign - process_tokens start - {tokens=}')
1115
1310
 
@@ -1123,19 +1318,21 @@ class CommandDesign(Command):
1123
1318
  # walk the list, remove all items after we're done.
1124
1319
  remove_list = []
1125
1320
  for token in unparsed:
1126
- # Since this is a raw argparser, we may have args that come from shlex.quote(token), such as:
1127
- # token = '\'+define+OC_ROOT="/foo/bar/opencos"\''
1128
- # So we have to check for strings that have been escaped for shell with extra single quotes.
1321
+ # Since this is a raw argparser, we may have args that come from shlex.quote(token),
1322
+ # such as:
1323
+ # token = '\'+define+OC_ROOT="/foo/bar/opencos"\''
1324
+ # So we have to check for strings that have been escaped for shell with extra single
1325
+ # quotes.
1129
1326
  m = re.match(r"^\'?\+\w+", token)
1130
1327
  if m:
1131
1328
  # Copy and strip all outer ' or " on the plusarg:
1132
- plusarg = util.strip_outer_quotes(token)
1329
+ plusarg = strip_outer_quotes(token)
1133
1330
  self.process_plusarg(plusarg, pwd=pwd)
1134
1331
  remove_list.append(token)
1135
1332
  for x in remove_list:
1136
1333
  unparsed.remove(x)
1137
1334
 
1138
- if len(unparsed) == 0 and self.error_on_no_files_or_targets:
1335
+ if not unparsed and self.error_on_no_files_or_targets:
1139
1336
  # derived classes can set error_on_no_files_or_targets=True
1140
1337
  # For example: CommandSim will error (requires files/targets),
1141
1338
  # CommandWaves does not (files/targets not required)
@@ -1143,11 +1340,11 @@ class CommandDesign(Command):
1143
1340
  # check the DEPS markup file for a single target, and if so run it
1144
1341
  # on the one and only target.
1145
1342
  if self.config['deps_markup_supported']:
1146
- deps = deps_helpers.DepsFile(
1343
+ deps = DepsFile(
1147
1344
  command_design_ref=self, target_path=os.getcwd(), cache=self.cached_deps
1148
1345
  )
1149
1346
  if deps.deps_file and deps.data:
1150
- all_targets = deps_helpers.deps_data_get_all_targets(deps.data)
1347
+ all_targets = deps_data_get_all_targets(deps.data)
1151
1348
  if all_targets:
1152
1349
  target = all_targets[-1]
1153
1350
  unparsed.append(target)
@@ -1169,7 +1366,7 @@ class CommandDesign(Command):
1169
1366
  file_exists, fpath, forced_extension = files.get_source_file(token)
1170
1367
  if file_exists:
1171
1368
  file_abspath = os.path.abspath(fpath)
1172
- file_base, file_ext = os.path.splitext(file_abspath)
1369
+ _, file_ext = os.path.splitext(file_abspath)
1173
1370
  if not forced_extension and not file_ext:
1174
1371
  # This probably isn't a file we want to use
1175
1372
  util.warning(f'looking for deps {token=}, found {file_abspath=}' \
@@ -1189,11 +1386,12 @@ class CommandDesign(Command):
1189
1386
  remove_list.append(token)
1190
1387
  continue # done with token, consume it, we added the file.
1191
1388
 
1192
- # we appear to be dealing with a target name which needs to be resolved (usually recursively)
1389
+ # we appear to be dealing with a target name which needs to be resolved (usually
1390
+ # recursively)
1193
1391
  if token.startswith(os.sep):
1194
1392
  target_name = token # if it's absolute path, don't prepend anything
1195
1393
  else:
1196
- target_name = os.path.join(".", token) # prepend ./so that we always have a <path>/<file>
1394
+ target_name = os.path.join(".", token) # prepend ./ to make it like: <path>/<file>
1197
1395
 
1198
1396
  util.debug(f'Calling self.resolve_target on {target_name=} ({token=})')
1199
1397
  if self.resolve_target(target_name, caller_info=caller_info):
@@ -1242,49 +1440,81 @@ class CommandDesign(Command):
1242
1440
  self.target = self.args['top']
1243
1441
 
1244
1442
  if self.error_on_missing_top and not self.args.get('top', ''):
1245
- self.error(f"Did not get a --top or DEPS top, required to run command",
1443
+ self.error("Did not get a --top or DEPS top, required to run command",
1246
1444
  f"'{self.command_name}' for tool={self.args.get('tool', None)}")
1247
1445
 
1248
1446
  return unparsed
1249
1447
 
1250
1448
 
1251
- def get_command_line_args(self, remove_args:list=[], remove_args_startswith:list=[]) -> list:
1449
+ def get_command_line_args( # pylint: disable=dangerous-default-value
1450
+ self, remove_args: list = [], remove_args_startswith: list = []
1451
+ ) -> list:
1252
1452
  '''Returns a list of all the args if you wanted to re-run this command
1253
1453
  (excludes eda, command, target).'''
1254
1454
 
1255
1455
  # This will not set bool's that are False, does not add --no-<somearg>
1256
- # nor --<somearg>=False
1257
- # This will not set str's that are empty.
1258
- # this will not set ints that are 0
1456
+ # nor --<somearg>=False (it's unclear which are 'store_true' v/ bool-action-kwargs)
1457
+ # This will not set str's that are empty
1458
+ # this will not set ints that are 0 (unless modified)
1459
+ # TODO(drew): we may want to revisit this if we tracked default arg values,
1460
+ # or perhaps if they have been "modified".
1259
1461
  ret = []
1260
1462
  for k,v in self.args.items():
1261
1463
 
1262
1464
  # Some args cannot be extracted and work, so omit these:
1263
1465
  if k in ['top-path'] + remove_args:
1264
1466
  continue
1265
- if any([k.startswith(x) for x in remove_args_startswith]):
1467
+ if any(k.startswith(x) for x in remove_args_startswith):
1266
1468
  continue
1267
1469
 
1268
- if type(v) is bool and v:
1470
+ is_modified = self.modified_args.get(k, False)
1471
+
1472
+ if isinstance(v, bool) and v:
1269
1473
  ret.append(f'--{k}')
1270
- elif type(v) is int and bool(v):
1474
+ elif isinstance(v, int) and (bool(v) or is_modified):
1271
1475
  ret.append(f'--{k}={v}')
1272
- elif type(v) is str and v:
1476
+ elif isinstance(v, str) and v:
1273
1477
  ret.append(f'--{k}={v}')
1274
- elif type(v) is list:
1478
+ elif isinstance(v, list):
1275
1479
  for item in v:
1276
- if item or type(item) not in [bool, str]:
1480
+ if item or isinstance(item, (bool, str)):
1277
1481
  # don't print bool/str that are blank.
1278
1482
  ret.append(f'--{k}={item}') # lists append
1279
1483
 
1280
1484
  return ret
1281
1485
 
1282
1486
 
1283
- _threads_start = 0
1284
- _threads_done = 0
1487
+ #_THREADS_START = 0
1488
+ #_THREADS_DONE = 0
1489
+
1490
+ class ThreadStats:
1491
+ '''To avoid globals for two ints, keep a class holder so CommandParallel and
1492
+ CommandParallelWorker can share values'''
1493
+
1494
+ done = 0
1495
+ started = 0
1496
+
1497
+ def all_done(self) -> bool:
1498
+ '''Returns True if all started jobs are done'''
1499
+ return self.started == self.done
1500
+
1501
+ def get_remaining(self) -> int:
1502
+ '''Returns remaining number of jobs'''
1503
+ return max(self.started - self.done, 0)
1504
+
1505
+ def __repr__(self) -> str:
1506
+ '''Pretty print DONE/STARTED'''
1507
+ return f'{self.done}/{self.started}'
1508
+
1285
1509
 
1286
1510
  class CommandParallelWorker(threading.Thread):
1287
- def __init__(self, n, work_queue, done_queue):
1511
+ '''Helper class for a single threaded job, often run by a CommandParallel command
1512
+ handler class'''
1513
+
1514
+ def __init__(
1515
+ self, n, work_queue: queue.Queue, done_queue: queue.Queue,
1516
+ threads_stats: ThreadStats
1517
+ ):
1288
1518
  threading.Thread.__init__(self)
1289
1519
  self.n = n
1290
1520
  self.work_queue = work_queue
@@ -1294,54 +1524,46 @@ class CommandParallelWorker(threading.Thread):
1294
1524
  self.proc = None
1295
1525
  self.pid = None
1296
1526
  self.last_timer_debug = 0
1527
+ self.threads_stats = threads_stats # ref to shared object
1297
1528
  util.debug(f"WORKER_{n}: START")
1298
1529
 
1299
1530
  def run(self):
1300
- global _threads_start
1301
- global _threads_done
1531
+ '''Runs this single job via threading. This is typically created and called by
1532
+
1533
+ a CommandParallel class handler
1534
+ '''
1535
+
1302
1536
  while True:
1303
1537
  # Get the work from the queue and expand the tuple
1304
- i, command_list, job_name, work_dir = self.work_queue.get()
1538
+ i, command_list, job_name, _ = self.work_queue.get()
1305
1539
  self.job_name = job_name
1306
1540
  try:
1307
1541
  util.debug(f"WORKER_{self.n}: Running job {i}: {job_name}")
1308
- PIPE=subprocess.PIPE
1309
- STDOUT=subprocess.STDOUT
1310
1542
  util.debug(f"WORKER_{self.n}: Calling Popen")
1311
- proc = subprocess.Popen(command_list, stdout=PIPE, stderr=STDOUT)
1543
+ proc = subprocess.Popen( # pylint: disable=consider-using-with
1544
+ command_list, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
1545
+ )
1312
1546
  self.proc = proc
1313
1547
  util.debug(f"WORKER_{self.n}: Opened process, PID={proc.pid}")
1314
1548
  self.pid = proc.pid
1315
- _threads_start += 1
1316
- while proc.returncode == None:
1549
+ self.threads_stats.started += 1
1550
+ while proc.returncode is None:
1317
1551
  try:
1318
1552
  if (time.time() - self.last_timer_debug) > 10:
1319
1553
  util.debug(f"WORKER_{self.n}: Calling proc.communicate")
1320
1554
  stdout, stderr = proc.communicate(timeout=0.5)
1321
- util.debug(f"WORKER_{self.n}: got: \n*** stdout:\n{stdout}\n*** stderr:{stderr}")
1555
+ util.debug(f"WORKER_{self.n}: got: \n*** stdout:\n{stdout}\n***",
1556
+ f"stderr:{stderr}")
1322
1557
  except subprocess.TimeoutExpired:
1323
1558
  if (time.time() - self.last_timer_debug) > 10:
1324
- util.debug(f"WORKER_{self.n}: Timer expired, stop_request={self.stop_request}")
1559
+ util.debug(f"WORKER_{self.n}: Timer expired, stop_request=",
1560
+ f"{self.stop_request}")
1325
1561
  self.last_timer_debug = time.time()
1326
- pass
1327
1562
  if self.stop_request:
1328
1563
  util.debug(f"WORKER_{self.n}: got stop request, issuing SIGINT")
1329
1564
  proc.send_signal(signal.SIGINT)
1330
1565
  util.debug(f"WORKER_{self.n}: got stop request, calling proc.wait")
1331
1566
  proc.wait()
1332
- if False and self.stop_request:
1333
- util.debug(f"WORKER_{self.n}: got stop request, issuing proc.terminate")
1334
- proc.terminate()
1335
- util.debug(f"WORKER_{self.n}: proc poll returns is now {proc.poll()}")
1336
- try:
1337
- util.debug(f"WORKER_{self.n}: Calling proc.communicate")
1338
- stdout, stderr = proc.communicate(timeout=0.2) # for completeness, in case we ever pipe/search stdout/stderr
1339
- util.debug(f"WORKER_{self.n}: got: \n*** stdout:\n{stdout}\n*** stderr:{stderr}")
1340
- except subprocess.TimeoutExpired:
1341
- util.debug(f"WORKER_{self.n}: timeout waiting for comminicate after terminate")
1342
- except:
1343
- pass
1344
- util.debug(f"WORKER_{self.n}: proc poll returns is now {proc.poll()}")
1345
1567
 
1346
1568
  util.debug(f"WORKER_{self.n}: -- out of while loop")
1347
1569
  self.pid = None
@@ -1350,99 +1572,113 @@ class CommandParallelWorker(threading.Thread):
1350
1572
  util.debug(f"WORKER_{self.n}: proc poll returns is now {proc.poll()}")
1351
1573
  try:
1352
1574
  util.debug(f"WORKER_{self.n}: Calling proc.communicate one last time")
1353
- stdout, stderr = proc.communicate(timeout=0.1) # for completeness, in case we ever pipe/search stdout/stderr
1354
- util.debug(f"WORKER_{self.n}: got: \n*** stdout:\n{stdout}\n*** stderr:{stderr}")
1575
+ stdout, stderr = proc.communicate(timeout=0.1)
1576
+ util.debug(f"WORKER_{self.n}: got: \n*** stdout:\n{stdout}\n***",
1577
+ f"stderr:{stderr}")
1355
1578
  except subprocess.TimeoutExpired:
1356
1579
  util.debug(f"WORKER_{self.n}: timeout waiting for communicate after loop?")
1357
- except:
1358
- pass
1580
+ except Exception as e:
1581
+ util.debug(f"WORKER_{self.n}: timeout with exception {e=}")
1582
+
1359
1583
  return_code = proc.poll()
1360
- util.debug(f"WORKER_{self.n}: Finished job {i}: {job_name} with return code {return_code}")
1584
+ util.debug(f"WORKER_{self.n}: Finished job {i}: {job_name} with return code",
1585
+ f"{return_code}")
1361
1586
  self.done_queue.put((i, job_name, return_code))
1362
1587
  finally:
1363
1588
  util.debug(f"WORKER_{self.n}: -- in finally block")
1364
1589
  self.work_queue.task_done()
1365
- _threads_done += 1
1590
+ self.threads_stats.done += 1
1366
1591
 
1367
1592
 
1368
1593
  class CommandParallel(Command):
1594
+ '''Base class command handler for running multiple eda jobs, such as for child classes
1595
+
1596
+ in commands.multi or commands.sweep
1597
+ '''
1369
1598
  def __init__(self, config: dict):
1370
1599
  Command.__init__(self, config=config)
1371
1600
  self.jobs = []
1372
1601
  self.jobs_status = []
1373
1602
  self.args['parallel'] = 1
1374
1603
  self.worker_threads = []
1604
+ self.threads_stats = ThreadStats()
1375
1605
 
1376
- def __del__(self):
1377
- util.debug(f"In Command.__del__, threads done/started: {_threads_done}/{_threads_start}")
1378
- if _threads_start == _threads_done:
1606
+ def __del__(self): # pylint: disable=too-many-branches
1607
+ util.debug(f"In Command.__del__, threads done/started: {self.threads_stats}")
1608
+ if self.threads_stats.all_done():
1379
1609
  return
1380
- util.warning(f"Need to shut down {_threads_start-_threads_done} worker threads...")
1610
+ util.warning(f"Need to shut down {self.threads_stats.get_remaining()} worker threads...")
1381
1611
  for w in self.worker_threads:
1382
1612
  if w.proc:
1383
1613
  util.warning(f"Requesting stop of PID {w.pid}: {w.job_name}")
1384
1614
  w.stop_request = True
1385
- for i in range(10):
1386
- util.debug(f"Threads done/started: {_threads_done}/{_threads_start}")
1387
- if _threads_start == _threads_done:
1388
- util.info(f"All threads done")
1615
+ for _ in range(10):
1616
+ util.debug(f"Threads done/started: {self.threads_stats}")
1617
+ if self.threads_stats.started == self.threads_stats.done:
1618
+ util.info("All threads done")
1389
1619
  return
1390
1620
  time.sleep(1)
1391
- subprocess.Popen(['stty', 'sane']).wait()
1392
- util.debug(f"Scanning workers again")
1621
+ subprocess.Popen(['stty', 'sane']).wait() # pylint: disable=consider-using-with
1622
+ util.debug("Scanning workers again")
1393
1623
  for w in self.worker_threads:
1394
1624
  if w.proc:
1395
1625
  util.info(f"need to SIGINT WORKER_{w.n}, may need manual cleanup, check 'ps'")
1396
1626
  if w.pid:
1397
1627
  os.kill(w.pid, signal.SIGINT)
1398
- for i in range(5):
1399
- util.debug(f"Threads done/started: {_threads_done}/{_threads_start}")
1400
- if _threads_start == _threads_done:
1401
- util.info(f"All threads done")
1628
+ for _ in range(5):
1629
+ util.debug(f"Threads done/started: {self.threads_stats}")
1630
+ if self.threads_stats.all_done():
1631
+ util.info("All threads done")
1402
1632
  return
1403
1633
  time.sleep(1)
1404
- subprocess.Popen(['stty', 'sane']).wait()
1405
- util.debug(f"Scanning workers again")
1634
+ subprocess.Popen(['stty', 'sane']).wait() # pylint: disable=consider-using-with
1635
+ util.debug("Scanning workers again")
1406
1636
  for w in self.worker_threads:
1407
1637
  if w.proc:
1408
1638
  util.info(f"need to TERM WORKER_{w.n}, probably needs manual cleanup, check 'ps'")
1409
1639
  if w.pid:
1410
1640
  os.kill(w.pid, signal.SIGTERM)
1411
- for i in range(5):
1412
- util.debug(f"Threads done/started: {_threads_done}/{_threads_start}")
1413
- if _threads_start == _threads_done:
1414
- util.info(f"All threads done")
1641
+ for _ in range(5):
1642
+ util.debug(f"Threads done/started: {self.threads_stats}")
1643
+ if self.threads_stats.all_done():
1644
+ util.info("All threads done")
1415
1645
  return
1416
1646
  time.sleep(1)
1417
- subprocess.Popen(['stty', 'sane']).wait()
1418
- util.debug(f"Scanning workers again")
1647
+ subprocess.Popen(['stty', 'sane']).wait() # pylint: disable=consider-using-with
1648
+ util.debug("Scanning workers again")
1419
1649
  for w in self.worker_threads:
1420
1650
  if w.proc:
1421
1651
  util.info(f"need to KILL WORKER_{w.n}, probably needs manual cleanup, check 'ps'")
1422
1652
  if w.pid:
1423
1653
  os.kill(w.pid, signal.SIGKILL)
1424
1654
  util.stop_log()
1425
- subprocess.Popen(['stty', 'sane']).wait()
1655
+ subprocess.Popen(['stty', 'sane']).wait() # pylint: disable=consider-using-with
1656
+
1657
+ def run_jobs( # pylint: disable=too-many-locals,too-many-branches,too-many-statements
1658
+ self, command: str
1659
+ ) -> None:
1660
+ '''Runs all "jobs" in self.jobs list, either serially or in parallel'''
1426
1661
 
1427
- def run_jobs(self, command):
1428
- # this is where we actually run the jobs. it's a messy piece of code and prob could use refactoring
1429
- # but the goal was to share as much as possible (job start, end, pass/fail judgement, etc) while
1430
- # supporting various mode combinations (parallel mode, verbose mode, fancy mode, etc) and keeping the
1431
- # UI output functional and awesome sauce
1662
+ # this is where we actually run the jobs. it's a messy piece of code and prob could use
1663
+ # refactoring but the goal was to share as much as possible (job start, end, pass/fail
1664
+ # judgement, etc) while supporting various mode combinations (parallel mode, verbose mode,
1665
+ # fancy mode, etc) and keeping the UI output functional and awesome sauce
1432
1666
 
1433
1667
  # walk targets to find the longest name, for display reasons
1434
1668
  longest_job_name = 0
1435
1669
  total_jobs = len(self.jobs)
1436
1670
  self.jobs_status = [None] * total_jobs
1437
1671
  for i in range(total_jobs):
1438
- l = len(self.jobs[i]['name'])
1439
- if l>longest_job_name: longest_job_name = l
1672
+ longest_job_name = max(longest_job_name, len(self.jobs[i]['name']))
1440
1673
 
1441
1674
  run_parallel = self.args['parallel'] > 1
1442
1675
 
1443
1676
  # figure out the width to print various numbers
1444
1677
  jobs_digits = len(f"{total_jobs}")
1445
- jobs_fmt = "%%%dd" % jobs_digits # ugh, for printing out a number with N digits
1678
+
1679
+ job_done_return_code = None
1680
+ job_done_run_time = 0
1681
+ job_done_name = ''
1446
1682
 
1447
1683
  # run the jobs!
1448
1684
  running_jobs = {}
@@ -1453,9 +1689,10 @@ class CommandParallel(Command):
1453
1689
  jobs_launched = 0
1454
1690
  num_parallel = min(len(self.jobs), self.args['parallel'])
1455
1691
  # 16 should really be the size of window or ?
1456
- (columns,lines) = shutil.get_terminal_size()
1457
- # we will enter fancy mode if we are parallel and we can leave 6 lines of regular scrolling output
1458
- fancy_mode = util.args['fancy'] and (num_parallel > 1) and (num_parallel <= (lines-6))
1692
+ _, lines = shutil.get_terminal_size()
1693
+ # we will enter fancy mode if we are parallel and we can leave 6 lines of regular scrolling
1694
+ # output
1695
+ fancy_mode = all([util.args['fancy'], num_parallel > 1, num_parallel <= (lines-6)])
1459
1696
  multi_cwd = util.getcwd() + os.sep
1460
1697
 
1461
1698
  self.patch_jobs_for_duplicate_target_names()
@@ -1466,8 +1703,12 @@ class CommandParallel(Command):
1466
1703
  work_queue = queue.Queue()
1467
1704
  done_queue = queue.Queue()
1468
1705
  for x in range(num_parallel):
1469
- worker = CommandParallelWorker(x, work_queue, done_queue)
1470
- # Setting daemon to True will let the main thread exit even though the workers are blocking
1706
+ worker = CommandParallelWorker(
1707
+ n=x, work_queue=work_queue, done_queue=done_queue,
1708
+ threads_stats=self.threads_stats
1709
+ )
1710
+ # Setting daemon to True will let the main thread exit even though the workers are
1711
+ # blocking
1471
1712
  worker.daemon = True
1472
1713
  worker.start()
1473
1714
  self.worker_threads.append(worker)
@@ -1478,22 +1719,30 @@ class CommandParallel(Command):
1478
1719
  for x in range(num_parallel):
1479
1720
  util.fancy_print(f"Starting worker {x}", x)
1480
1721
 
1481
- while len(self.jobs) or len(running_jobs.items()):
1722
+ while self.jobs or running_jobs:
1482
1723
  job_done = False
1483
1724
  job_done_quiet = False
1484
1725
  anything_done = False
1485
1726
 
1486
1727
  def sprint_job_line(job_number=0, job_name="", final=False, hide_stats=False):
1487
- return (f"INFO: [EDA] " +
1488
- util.string_or_space(f"[job {jobs_fmt%job_number}/{jobs_fmt%total_jobs} ", final) +
1489
- util.string_or_space(f"| pass ", hide_stats or final) +
1490
- util.string_or_space(f"{jobs_fmt%len(passed_jobs)}/{jobs_fmt%jobs_complete} ", hide_stats) +
1491
- util.string_or_space(f"@ {(100*(jobs_complete))/total_jobs:5.1f}%", hide_stats or final) +
1492
- util.string_or_space(f"] ", final) +
1493
- f"{command} {(job_name+' ').ljust(longest_job_name+3,'.')}")
1494
-
1495
- # for any kind of run (parallel or not, fancy or not, verbose or not) ... can we launch a job?
1496
- if len(self.jobs) and (len(running_jobs.items()) < num_parallel):
1728
+ return (
1729
+ "INFO: [EDA] " +
1730
+ string_or_space(
1731
+ f"[job {job_number:0{jobs_digits}d}/{total_jobs:0{jobs_digits}d} ",
1732
+ final) +
1733
+ string_or_space("| pass ", hide_stats or final) +
1734
+ string_or_space(
1735
+ f"{len(passed_jobs):0{jobs_digits}d}/{jobs_complete:0{jobs_digits}d} ",
1736
+ hide_stats) +
1737
+ string_or_space(f"@ {(100 * (jobs_complete)) / total_jobs : 5.1f}%",
1738
+ hide_stats or final) +
1739
+ string_or_space("] ", final) +
1740
+ f"{command} {(job_name+' ').ljust(longest_job_name+3,'.')}"
1741
+ )
1742
+
1743
+ # for any kind of run (parallel or not, fancy or not, verbose or not) ...
1744
+ # can we launch a job?
1745
+ if self.jobs and (len(running_jobs) < num_parallel):
1497
1746
  # we are launching a job
1498
1747
  jobs_launched += 1
1499
1748
  anything_done = True
@@ -1501,15 +1750,17 @@ class CommandParallel(Command):
1501
1750
  # TODO(drew): it might be nice to pass more items on 'job' dict, like
1502
1751
  # logfile or job-name, so CommandSweep or CommandMulti don't have to set
1503
1752
  # via args. on their command_list.
1504
- if job['name'].startswith(multi_cwd): job['name'] = job['name'][len(multi_cwd):]
1505
- # in all but fancy mode, we will print this text at the launch of a job. It may get a newline below
1753
+ if job['name'].startswith(multi_cwd):
1754
+ job['name'] = job['name'][len(multi_cwd):]
1755
+ # in all but fancy mode, we will print this text at the launch of a job. It may
1756
+ # get a newline below
1506
1757
  job_text = sprint_job_line(jobs_launched, job['name'], hide_stats=run_parallel)
1507
1758
  command_list = job['command_list']
1508
1759
  cwd = util.getcwd()
1509
1760
 
1510
1761
  if run_parallel:
1511
1762
  # multithreaded job launch: add to queue
1512
- worker = workers.pop(0) # we don't actually know which thread will pick up, but GUI will be consistent
1763
+ worker = workers.pop(0)
1513
1764
  running_jobs[str(jobs_launched)] = { 'name' : job['name'],
1514
1765
  'number' : jobs_launched,
1515
1766
  'worker' : worker,
@@ -1519,15 +1770,20 @@ class CommandParallel(Command):
1519
1770
  suffix = "<START>"
1520
1771
  if fancy_mode:
1521
1772
  util.fancy_print(job_text+suffix, worker)
1773
+ elif failed_jobs:
1774
+ # if we aren't in fancy mode, we will print a START line, periodic RUNNING
1775
+ # lines, and PASS/FAIL line per-job
1776
+ util.print_orange(job_text + Colors.yellow + suffix)
1522
1777
  else:
1523
- # if we aren't in fancy mode, we will print a START line, periodic RUNNING lines, and PASS/FAIL line per-job
1524
- if len(failed_jobs): util.print_orange(job_text + util.string_yellow + suffix)
1525
- else: util.print_yellow(job_text + util.string_yellow + suffix)
1778
+ util.print_yellow(job_text + Colors.yellow + suffix)
1526
1779
  else:
1527
- # single-threaded job launch, we are going to print out job info as we start each job... no newline
1528
- # since non-verbose silences the job and prints only <PASS>/<FAIL> after the trailing "..." we leave here
1529
- if len(failed_jobs): util.print_orange(job_text, end="")
1530
- else: util.print_yellow(job_text, end="")
1780
+ # single-threaded job launch, we are going to print out job info as we start
1781
+ # each job... no newline. since non-verbose silences the job and prints only
1782
+ # <PASS>/<FAIL> after the trailing "..." we leave here
1783
+ if failed_jobs:
1784
+ util.print_orange(job_text, end="")
1785
+ else:
1786
+ util.print_yellow(job_text, end="")
1531
1787
  job_done_number = jobs_launched
1532
1788
  job_done_name = job['name']
1533
1789
  job_start_time = time.time()
@@ -1539,13 +1795,16 @@ class CommandParallel(Command):
1539
1795
  _, _, job_done_return_code = self.exec(
1540
1796
  cwd, command_list, background=False, stop_on_error=False, quiet=False
1541
1797
  )
1542
- # reprint the job text previously printed before running job(and given "\n" after the trailing "...")
1798
+ # reprint the job text previously printed before running job (and given
1799
+ # "\n" after the trailing "...")
1543
1800
  else:
1544
1801
  # run job, swallowing output (hope you have a logfile)
1545
1802
  _, _, job_done_return_code = self.exec(
1546
1803
  cwd, command_list, background=True, stop_on_error=False, quiet=True
1547
1804
  )
1548
- job_done_quiet = True # in this case, we have the job start text (trailing "...", no newline) printed
1805
+ # in this case, we have the job start text (trailing "...", no newline)
1806
+ # printed
1807
+ job_done_quiet = True
1549
1808
  job_done = True
1550
1809
  job_done_run_time = time.time() - job_start_time
1551
1810
  # Since we consumed the job, use the job['index'] to track the per-job status:
@@ -1553,15 +1812,16 @@ class CommandParallel(Command):
1553
1812
  if run_parallel:
1554
1813
  # parallel run, check for completed job
1555
1814
  if done_queue.qsize():
1556
- # we're collecting a finished job from a worker thread. note we will only reap one job per iter of the big
1557
- # loop, so as to share job completion code at the bottom
1815
+ # we're collecting a finished job from a worker thread. note we will only
1816
+ # reap one job per iter of the big loop, so as to share job completion code
1817
+ # at the bottom
1558
1818
  anything_done = True
1559
1819
  job_done = True
1560
1820
  job_done_number, job_done_name, job_done_return_code = done_queue.get()
1561
1821
  t = running_jobs[str(job_done_number)]
1562
1822
  # in fancy mode, we need to clear the worker line related to this job.
1563
1823
  if fancy_mode:
1564
- util.fancy_print(f"INFO: [EDA] Parallel: Worker Idle ...", t['worker'])
1824
+ util.fancy_print("INFO: [EDA] Parallel: Worker Idle ...", t['worker'])
1565
1825
  job_done_run_time = time.time() - t['start_time']
1566
1826
  util.debug(f"removing job #{job_done_number} from running jobs")
1567
1827
  del running_jobs[str(job_done_number)]
@@ -1573,33 +1833,40 @@ class CommandParallel(Command):
1573
1833
  if (fancy_mode or (time.time() - t['update_time']) > 30):
1574
1834
  t['update_time'] = time.time()
1575
1835
  job_text = sprint_job_line(t['number'], t['name'], hide_stats=True)
1576
- suffix = f"<RUNNING: {util.sprint_time(time.time() - t['start_time'])}>"
1836
+ suffix = f"<RUNNING: {sprint_time(time.time() - t['start_time'])}>"
1577
1837
  if fancy_mode:
1578
1838
  util.fancy_print(f"{job_text}{suffix}", t['worker'])
1839
+ elif failed_jobs:
1840
+ util.print_orange(job_text + Colors.yellow + suffix)
1579
1841
  else:
1580
- if len(failed_jobs): util.print_orange(job_text+util.string_yellow+suffix)
1581
- else: util.print_yellow(job_text+util.string_yellow+suffix)
1842
+ util.print_yellow(job_text + Colors.yellow + suffix)
1582
1843
 
1583
1844
  # shared job completion code
1584
- # single or multi-threaded, we can arrive here to harvest <= 1 jobs, and need {job, return_code} valid, and
1585
- # we expect the start of a status line to have been printed, ready for pass/fail
1845
+ # single or multi-threaded, we can arrive here to harvest <= 1 jobs, and need
1846
+ # {job, return_code} valid, and we expect the start of a status line to have been
1847
+ # printed, ready for pass/fail
1586
1848
  if job_done:
1587
1849
  jobs_complete += 1
1588
1850
  if job_done_return_code is None or job_done_return_code:
1589
- # embed the color code, to change color of pass/fail during the util.print_orange/yellow below
1851
+ # embed the color code, to change color of pass/fail during the
1852
+ # util.print_orange/yellow below
1590
1853
  if job_done_return_code == 124:
1591
- # bash uses 124 for bash timeout errors, if that was preprended to the command list.
1592
- suffix = f"{util.string_red}<TOUT: {util.sprint_time(job_done_run_time)}>"
1854
+ # bash uses 124 for bash timeout errors, if that was preprended to the
1855
+ # command list.
1856
+ suffix = f"{Colors.red}<TOUT: {sprint_time(job_done_run_time)}>"
1593
1857
  else:
1594
- suffix = f"{util.string_red}<FAIL: {util.sprint_time(job_done_run_time)}>"
1858
+ suffix = f"{Colors.red}<FAIL: {sprint_time(job_done_run_time)}>"
1595
1859
  failed_jobs.append(job_done_name)
1596
1860
  else:
1597
- suffix = f"{util.string_green}<PASS: {util.sprint_time(job_done_run_time)}>"
1861
+ suffix = f"{Colors.green}<PASS: {sprint_time(job_done_run_time)}>"
1598
1862
  passed_jobs.append(job_done_name)
1599
1863
  # we want to print in one shot, because in fancy modes that's all that we're allowed
1600
- job_done_text = "" if job_done_quiet else sprint_job_line(job_done_number, job_done_name)
1601
- if len(failed_jobs): util.print_orange(f"{job_done_text}{suffix}")
1602
- else: util.print_yellow(f"{job_done_text}{suffix}")
1864
+ job_done_text = "" if job_done_quiet else sprint_job_line(job_done_number,
1865
+ job_done_name)
1866
+ if failed_jobs:
1867
+ util.print_orange(f"{job_done_text}{suffix}")
1868
+ else:
1869
+ util.print_yellow(f"{job_done_text}{suffix}")
1603
1870
  self.jobs_status[job_done_number-1] = job_done_return_code
1604
1871
 
1605
1872
  if not anything_done:
@@ -1607,23 +1874,26 @@ class CommandParallel(Command):
1607
1874
 
1608
1875
  if total_jobs:
1609
1876
  emoji = "< :) >" if (len(passed_jobs) == total_jobs) else "< :( >"
1610
- util.info(sprint_job_line(final=True,job_name="jobs passed")+emoji, start="")
1877
+ util.info(sprint_job_line(final=True, job_name="jobs passed") + emoji, start="")
1611
1878
  else:
1612
- util.info(f"Parallel: <No jobs found>")
1879
+ util.info("Parallel: <No jobs found>")
1613
1880
  # Make sure all jobs have a set status:
1614
- for i,rc in enumerate(self.jobs_status):
1615
- if rc is None or type(rc) != int:
1881
+ for i, rc in enumerate(self.jobs_status):
1882
+ if rc is None or not isinstance(rc, int):
1616
1883
  self.error(f'job {i=} {rc=} did not return a proper return code')
1617
- jobs_status[i] = 1
1884
+ self.jobs_status[i] = 2
1618
1885
 
1619
1886
  # if self.status > 0, then keep it non-zero, else set it if we still have running jobs.
1620
1887
  if self.status == 0:
1621
- self.status = 0 if len(self.jobs_status) == 0 else max(self.jobs_status)
1888
+ if self.jobs_status:
1889
+ self.status = max(self.jobs_status)
1890
+ # else keep at 0, empty list.
1622
1891
  util.fancy_stop()
1623
1892
 
1624
1893
  @staticmethod
1625
1894
  def get_name_from_target(target: str) -> str:
1626
- return target.replace('../', '').lstrip('./')
1895
+ '''Given a target path, strip leftmost path separators to get a shorter string name'''
1896
+ return target.replace('../', '').lstrip('./').lstrip(os.path.sep)
1627
1897
 
1628
1898
 
1629
1899
  def update_args_list(self, args: list, tool: str) -> None:
@@ -1667,7 +1937,7 @@ class CommandParallel(Command):
1667
1937
  tokens=tokens.copy(),
1668
1938
  apply_parsed_args=False,
1669
1939
  )
1670
- util.debug(f'{self.command_name}: {single_cmd_unparsed=}')
1940
+ util.debug(f'{self.command_name}: {single_cmd_parsed=}, {single_cmd_unparsed=}')
1671
1941
 
1672
1942
  # There should not be any single_cmd_unparsed args starting with '-'
1673
1943
  bad_remaining_args = [x for x in single_cmd_unparsed if x.startswith('-')]
@@ -1694,7 +1964,7 @@ class CommandParallel(Command):
1694
1964
  if item.startswith(f'--{arg_name}='):
1695
1965
  _, name = item.split(f'--{arg_name}=')
1696
1966
  return name
1697
- elif item == f'--{arg_name}':
1967
+ if item == f'--{arg_name}':
1698
1968
  return job_dict['command_list'][i + 1]
1699
1969
  return ''
1700
1970
 
@@ -1704,7 +1974,7 @@ class CommandParallel(Command):
1704
1974
  if item.startswith(f'--{arg_name}='):
1705
1975
  job_dict['command_list'][i] = f'--{arg_name}=' + new_value
1706
1976
  return True
1707
- elif item == f'--{arg_name}':
1977
+ if item == f'--{arg_name}':
1708
1978
  job_dict['command_list'][i + 1] = new_value
1709
1979
  return True
1710
1980
  return False
@@ -1729,7 +1999,6 @@ class CommandParallel(Command):
1729
1999
  tpath, _ = os.path.split(job_dict['target'])
1730
2000
 
1731
2001
  # prepend path information to job-name:
1732
- patched_sub_work_dir = False
1733
2002
  patched_target_path = os.path.relpath(tpath).replace(os.sep, '_')
1734
2003
  new_job_name = f'{patched_target_path}.{key}'
1735
2004
  replace_job_arg(job_dict, arg_name='job-name', new_value=new_job_name)