opencos-eda 0.2.48__py3-none-any.whl → 0.2.49__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. opencos/__init__.py +4 -2
  2. opencos/_version.py +10 -7
  3. opencos/commands/flist.py +8 -7
  4. opencos/commands/multi.py +13 -14
  5. opencos/commands/sweep.py +3 -2
  6. opencos/deps/__init__.py +0 -0
  7. opencos/deps/defaults.py +69 -0
  8. opencos/deps/deps_commands.py +419 -0
  9. opencos/deps/deps_file.py +326 -0
  10. opencos/deps/deps_processor.py +670 -0
  11. opencos/deps_schema.py +7 -8
  12. opencos/eda.py +84 -64
  13. opencos/eda_base.py +572 -316
  14. opencos/eda_config.py +80 -14
  15. opencos/eda_extract_targets.py +22 -14
  16. opencos/eda_tool_helper.py +33 -7
  17. opencos/export_helper.py +166 -86
  18. opencos/export_json_convert.py +31 -23
  19. opencos/files.py +2 -1
  20. opencos/hw/__init__.py +0 -0
  21. opencos/{oc_cli.py → hw/oc_cli.py} +9 -4
  22. opencos/names.py +0 -4
  23. opencos/peakrdl_cleanup.py +13 -7
  24. opencos/seed.py +19 -11
  25. opencos/tests/helpers.py +3 -2
  26. opencos/tests/test_deps_helpers.py +35 -32
  27. opencos/tests/test_eda.py +36 -29
  28. opencos/tests/test_eda_elab.py +5 -3
  29. opencos/tests/test_eda_synth.py +1 -1
  30. opencos/tests/test_oc_cli.py +1 -1
  31. opencos/tests/test_tools.py +3 -2
  32. opencos/tools/iverilog.py +2 -2
  33. opencos/tools/modelsim_ase.py +2 -2
  34. opencos/tools/riviera.py +1 -1
  35. opencos/tools/slang.py +1 -1
  36. opencos/tools/surelog.py +1 -1
  37. opencos/tools/verilator.py +1 -1
  38. opencos/tools/vivado.py +1 -1
  39. opencos/tools/yosys.py +4 -3
  40. opencos/util.py +374 -468
  41. opencos/utils/__init__.py +0 -0
  42. opencos/utils/markup_helpers.py +98 -0
  43. opencos/utils/str_helpers.py +111 -0
  44. opencos/utils/subprocess_helpers.py +108 -0
  45. {opencos_eda-0.2.48.dist-info → opencos_eda-0.2.49.dist-info}/METADATA +1 -1
  46. opencos_eda-0.2.49.dist-info/RECORD +88 -0
  47. {opencos_eda-0.2.48.dist-info → opencos_eda-0.2.49.dist-info}/entry_points.txt +1 -1
  48. opencos/deps_helpers.py +0 -1346
  49. opencos_eda-0.2.48.dist-info/RECORD +0 -79
  50. /opencos/{pcie.py → hw/pcie.py} +0 -0
  51. {opencos_eda-0.2.48.dist-info → opencos_eda-0.2.49.dist-info}/WHEEL +0 -0
  52. {opencos_eda-0.2.48.dist-info → opencos_eda-0.2.49.dist-info}/licenses/LICENSE +0 -0
  53. {opencos_eda-0.2.48.dist-info → opencos_eda-0.2.49.dist-info}/licenses/LICENSE.spdx +0 -0
  54. {opencos_eda-0.2.48.dist-info → opencos_eda-0.2.49.dist-info}/top_level.txt +0 -0
opencos/eda_base.py CHANGED
@@ -1,22 +1,36 @@
1
+ '''opencos.eda_base holds base classes for opencos.tools and opencos.commands
2
+
3
+ Note that opencos.Command.process_tokens (on a derived class) is the main entrypoint
4
+ from eda.py, it will find a command handler class for a command + tool, and call
5
+ its obj.process_tokens(args)
6
+ '''
7
+
8
+ # TODO(drew): Clean up the following pylint
9
+ # pylint: disable=too-many-lines
10
+
1
11
  import argparse
2
12
  import copy
3
- import glob
4
13
  import subprocess
5
14
  import os
6
15
  import queue
7
16
  import re
8
17
  import shutil
9
- import shlex
10
18
  import signal
11
19
  import sys
12
20
  import threading
13
21
  import time
14
22
  from pathlib import Path
15
23
 
16
- import opencos
17
- from opencos import seed, deps_helpers, util, files
24
+ from opencos import seed, util, files
18
25
  from opencos import eda_config
19
- from opencos.deps_helpers import get_deps_markup_file, deps_markup_safe_load
26
+
27
+ from opencos.util import Colors
28
+ from opencos.utils.str_helpers import sprint_time, strip_outer_quotes, string_or_space, \
29
+ indent_wrap_long_text
30
+ from opencos.utils.subprocess_helpers import subprocess_run_background
31
+
32
+ from opencos.deps.deps_file import DepsFile, deps_data_get_all_targets
33
+ from opencos.deps.deps_processor import DepsProcessor
20
34
 
21
35
 
22
36
  def print_base_help() -> None:
@@ -28,9 +42,12 @@ def print_base_help() -> None:
28
42
 
29
43
 
30
44
  def get_argparser() -> argparse.ArgumentParser:
45
+ '''Returns the ArgumentParser for general eda CLI'''
31
46
  parser = argparse.ArgumentParser(prog='eda options', add_help=False, allow_abbrev=False)
32
47
  parser.add_argument('-q', '--quit', action='store_true',
33
- help='For interactive mode (eda called with no options, command, or targets)')
48
+ help=(
49
+ 'For interactive mode (eda called with no options, command, or'
50
+ ' targets)'))
34
51
  parser.add_argument('--exit', action='store_true', help='same as --quit')
35
52
  parser.add_argument('-h', '--help', action='store_true')
36
53
 
@@ -40,22 +57,25 @@ def get_argparser() -> argparse.ArgumentParser:
40
57
  help='Tool to use for this command, such as: modelsim_ase, verilator,' \
41
58
  + ' modelsim_ase=/path/to/bin/vsim, verilator=/path/to/bin/verilator')
42
59
  parser.add_argument('--eda-safe', action='store_true',
43
- help='disable all DEPS file deps shell commands, overrides values from --config-yml')
60
+ help=('disable all DEPS file deps shell commands, overrides values from'
61
+ ' --config-yml'))
44
62
  return parser
45
63
 
46
64
 
47
65
  def get_argparser_short_help() -> str:
66
+ '''Returns str for the eda_base argparser (--help, --tool, etc)'''
48
67
  return util.get_argparser_short_help(parser=get_argparser())
49
68
 
50
69
 
51
- def get_eda_exec(command:str=''):
70
+ def get_eda_exec(command: str = '') -> str:
71
+ '''Returns the full path of `eda` executable to be used for a given eda <command>'''
52
72
  # NOTE(drew): This is kind of flaky. 'eda multi' reinvokes 'eda'. But the executable for 'eda'
53
73
  # is one of:
54
74
  # 1. pip3 install opencos-eda
55
75
  # -- script 'eda', installed from PyPi
56
76
  # 2. pip3 uninstall .; python3 -m build; pip3 install
57
77
  # -- script 'eda' but installed from local.
58
- # 2. (opencos repo)/bin/eda - a python wrapper to link to (opencos repo)/opencos/eda.py (package)
78
+ # 2. (opencos repo)/bin/eda - a python wrapper to link to (opencos repo)/opencos/eda.py, but
59
79
  # packages cannot be run standalone, they need to be called as: python3 -m opencos.eda,
60
80
  # and do not work with relative paths. This only works if env OC_ROOT is set or can be found.
61
81
  # 3. If you ran 'source bin/addpath' then you are always using the local (opencos repo)/bin/eda
@@ -64,11 +84,13 @@ def get_eda_exec(command:str=''):
64
84
  # Can we run from OC_ROOT/bin/eda?
65
85
  oc_root = util.get_oc_root()
66
86
  if not oc_root:
67
- util.error(f"Need 'eda' in our path to run 'eda {command}', could not find env OC_ROOT, {eda_path=}, {oc_root=}")
87
+ util.error(f"Need 'eda' in our path to run 'eda {command}', could not find env",
88
+ f"OC_ROOT, {eda_path=}, {oc_root=}")
68
89
  else:
69
90
  bin_eda = os.path.join(oc_root, 'bin', 'eda')
70
91
  if not os.path.exists(bin_eda):
71
- util.error(f"Need 'eda' in our path to run 'eda {command}', cound not find bin/, {eda_path=}, {oc_root=}, {bin_eda=}")
92
+ util.error(f"Need 'eda' in our path to run 'eda {command}', cound not find",
93
+ f"bin/, {eda_path=}, {oc_root=}, {bin_eda=}")
72
94
  else:
73
95
  util.info(f"'eda' not in path, using {bin_eda=} for 'eda' {command} executable")
74
96
  eda_path = os.path.abspath(bin_eda)
@@ -76,9 +98,10 @@ def get_eda_exec(command:str=''):
76
98
  return eda_path
77
99
 
78
100
 
79
- def which_tool(command, config):
80
- '''Returns which tool will be used for a command, given the command_handlers in config dict.'''
81
- from opencos import eda_tool_helper
101
+ def which_tool(command: str, config: dict) -> str:
102
+ '''Returns which tool (str) will be used for a command, given the command_handlers in
103
+
104
+ config dict.'''
82
105
  if config is None:
83
106
  util.error(f'which_tool({command=}) called w/out config')
84
107
  if not command in config.get('command_handler', {}):
@@ -136,9 +159,10 @@ class Tool:
136
159
  self.set_tool_config_from_config()
137
160
 
138
161
  def set_exe(self, config: dict) -> None:
162
+ '''Sets self._EXE based on config'''
139
163
  if self._TOOL and self._TOOL in config.get('auto_tools_order', [{}])[0]:
140
164
  exe = config.get('auto_tools_order', [{}])[0][self._TOOL].get('exe', '')
141
- if exe and type(exe) is list:
165
+ if exe and isinstance(exe, list):
142
166
  exe = exe[0] # pick first
143
167
  if exe and exe != self._EXE:
144
168
  util.info(f'Override for {self._TOOL} using exe {exe}')
@@ -155,9 +179,16 @@ class Tool:
155
179
  return self._VERSION
156
180
 
157
181
  def set_tool_defines(self) -> None:
158
- pass
182
+ '''Derived classes may override, sets any additional defines based on tool.'''
183
+ return
184
+
159
185
 
160
186
  class Command:
187
+ '''Base class for all: eda <command>
188
+
189
+ The Command class should be used when you don't require files, otherwise consider
190
+ CommandDesign.
191
+ '''
161
192
 
162
193
  command_name: str = ''
163
194
 
@@ -170,20 +201,20 @@ class Command:
170
201
  "keep" : False,
171
202
  "force" : False,
172
203
  "fake" : False,
173
- "stop-before-compile": False, # Usually in the self.do_it() method, stop prior to compile/elaborate/simulate
204
+ "stop-before-compile": False,
174
205
  "stop-after-compile": False,
175
- "stop-after-elaborate": False, # Set to True to only run compile + elaboration (aka compile + lint)
206
+ "stop-after-elaborate": False,
176
207
  "lint": False, # Same as stop-after-elaborate
177
208
  "eda-dir" : "eda.work", # all eda jobs go in here
178
209
  "job-name" : "", # this is used to create a certain dir under "eda_dir"
179
- "work-dir" : "", # this can be used to run the job in a certain dir, else it will be <eda-dir>/<job-name> else <eda-dir>/<target>_<command>
180
- "sub-work-dir" : "", # this can be used to name the dir built under <eda-dir>, which seems to be same function as job-name??
210
+ "work-dir" : "", # default is usually <eda-dir>/<job-name> or <eda-dir>/<target>.<cmd>
211
+ "sub-work-dir" : "", # this can be used to name the dir built under <eda-dir>
181
212
  "work-dir-use-target-dir": False,
182
213
  "suffix" : "",
183
214
  "design" : "", # not sure how this relates to top
184
215
  'export': False,
185
- 'export-run': False, # run from the exported location if possible, if not possible run the command in usual place.
186
- 'export-json': False, # generates an export.json suitable for a testrunner, if possible for self.command.
216
+ 'export-run': False, # run from the exported location if possible
217
+ 'export-json': False, # generates an export.json suitable for a testrunner
187
218
  'enable-tags': [],
188
219
  'disable-tags': [],
189
220
  'test-mode': False,
@@ -214,6 +245,7 @@ class Command:
214
245
  self.status = 0
215
246
  self.errors_log_f = None
216
247
 
248
+
217
249
  def error(self, *args, **kwargs) -> None:
218
250
  '''Returns None, child classes can call self.error(..) instead of util.error,
219
251
 
@@ -225,10 +257,11 @@ class Command:
225
257
  if self.args['work-dir']:
226
258
  if not self.errors_log_f:
227
259
  try:
228
- self.errors_log_f = open(
229
- os.path.join(self.args['work-dir'], 'eda.errors.log'), 'w'
260
+ self.errors_log_f = open( # pylint: disable=consider-using-with
261
+ os.path.join(self.args['work-dir'], 'eda.errors.log'), 'w',
262
+ encoding='utf-8'
230
263
  )
231
- except:
264
+ except FileNotFoundError:
232
265
  pass
233
266
  if self.errors_log_f:
234
267
  print(
@@ -245,17 +278,29 @@ class Command:
245
278
  util.error(f"command '{self.command_name}' has previous errors")
246
279
  return self.status > 0
247
280
 
248
- def which_tool(self, command:str):
281
+ def which_tool(self, command:str) -> str:
282
+ '''Returns a str for the tool name used for the requested command'''
249
283
  return which_tool(command, config=self.config)
250
284
 
251
- def create_work_dir(self):
285
+ def create_work_dir( # pylint: disable=too-many-branches,too-many-statements
286
+ self
287
+ ) -> str:
288
+ '''Creates the working directory and populates self.args['work-dir']
289
+
290
+ Generally uses ./ self.args['eda-dir'] / <target-name>.<command> /
291
+ however, self.args['job-name'] or ['sub-work-dir'] can override that.
292
+
293
+ Additionally, the work-dir is attempted to be deleted if it already exists
294
+ AND it is beneath the caller's original working directory. We also will
295
+ not delete 'work-dir' that is ./ or $PWD.
296
+ '''
252
297
  util.debug(f"create_work_dir: {self.args['eda-dir']=} {self.args['work-dir']=}")
253
298
  if self.args['work-dir-use-target-dir']:
254
299
  if not self.target_path:
255
300
  self.target_path = '.'
256
- util.info(f"create_work_dir: --work-dir-use-target-dir: using:",
257
- f"{os.path.abspath(self.target_path)}",
258
- f'target={self.target}')
301
+ util.info("create_work_dir: --work-dir-use-target-dir: using:",
302
+ f"{os.path.abspath(self.target_path)}",
303
+ f'target={self.target}')
259
304
  self.args['work-dir'] = self.target_path
260
305
  self.args['sub-work-dir'] = ''
261
306
  return self.args['work-dir']
@@ -263,28 +308,37 @@ class Command:
263
308
  if not os.path.exists(self.args['eda-dir']):
264
309
  util.safe_mkdir(self.args['eda-dir'])
265
310
  util.info(f"create_work_dir: created {self.args['eda-dir']}")
311
+
266
312
  if self.args['design'] == "":
267
313
  if ('top' in self.args) and (self.args['top'] != ""):
268
314
  self.args['design'] = self.args['top']
269
- util.debug(f"create_work_dir: set {self.args['design']=} from {self.args['top']=}, since it was empty")
315
+ util.debug(f"create_work_dir: set {self.args['design']=} from",
316
+ f"{self.args['top']=}, since it was empty")
270
317
  else:
271
318
  self.args['design'] = "design" # generic, i.e. to create work dir "design_upload"
272
- util.debug(f"create_work_dir: set {self.args['design']=} to 'design', since it was empty and we have no top")
319
+ util.debug(f"create_work_dir: set {self.args['design']=} to 'design', since it",
320
+ "was empty and we have no top")
321
+
273
322
  if self.target == "":
274
323
  self.target = self.args['design']
275
324
  util.debug(f"create_work_dir: set {self.target=} from design name, since it was empty")
325
+
276
326
  if self.args['work-dir'] == '':
277
327
  if self.args['sub-work-dir'] == '':
278
328
  if self.args['job-name'] != '':
279
329
  self.args['sub-work-dir'] = self.args['job-name']
280
- util.debug(f"create_work_dir: set {self.args['sub-work-dir']=} from {self.args['job-name']=}, since it was empty")
330
+ util.debug(f"create_work_dir: set {self.args['sub-work-dir']=} from",
331
+ f"{self.args['job-name']=}, since it was empty")
281
332
  else:
282
333
  self.args['sub-work-dir'] = f'{self.target}.{self.command_name}'
283
- util.debug(f"create_work_dir: set {self.args['sub-work-dir']=} from {self.target=} and {self.command_name=}, since it was empty and we have no job-name")
334
+ util.debug(f"create_work_dir: set {self.args['sub-work-dir']=} from",
335
+ f"{self.target=} and {self.command_name=}, since it was empty and",
336
+ "we have no job-name")
284
337
  self.args['work-dir'] = os.path.join(self.args['eda-dir'], self.args['sub-work-dir'])
285
338
  util.debug(f"create_work_dir: set {self.args['work-dir']=}")
339
+
286
340
  keep_file = os.path.join(self.args['work-dir'], "eda.keep")
287
- if (os.path.exists(self.args['work-dir'])):
341
+ if os.path.exists(self.args['work-dir']):
288
342
  if os.path.exists(keep_file) and not self.args['force']:
289
343
  self.error(f"Cannot remove old work dir due to '{keep_file}'")
290
344
  elif os.path.abspath(self.args['work-dir']) in os.getcwd():
@@ -293,16 +347,19 @@ class Command:
293
347
  # --work-dir=$PWD
294
348
  # --work-dir=/some/path/almost/here
295
349
  # Allow it, but preserve the existing directory, we don't want to blow away
296
- # files up-heir from us.
350
+ # files up-hier from us.
297
351
  # Enables support for --work-dir=.
298
- util.info(f"Not removing existing work-dir: '{self.args['work-dir']}' is within {os.getcwd()=}")
352
+ util.info(f"Not removing existing work-dir: '{self.args['work-dir']}' is within",
353
+ f"{os.getcwd()=}")
299
354
  elif str(Path(self.args['work-dir'])).startswith(str(Path('/'))):
300
355
  # Do not allow other absolute path work dirs if it already exists.
301
356
  # This prevents you from --work-dir=~ and eda wipes out your home dir.
302
- self.error(f'Cannot use work-dir={self.args["work-dir"]} starting with absolute path "/"')
357
+ self.error(f'Cannot use work-dir={self.args["work-dir"]} starting with',
358
+ 'fabsolute path "/"')
303
359
  elif str(Path('..')) in str(Path(self.args['work-dir'])):
304
360
  # Do not allow other ../ work dirs if it already exists.
305
- self.error(f'Cannot use work-dir={self.args["work-dir"]} with up-hierarchy ../ paths')
361
+ self.error(f'Cannot use work-dir={self.args["work-dir"]} with up-hierarchy'
362
+ '../ paths')
306
363
  else:
307
364
  # If we made it this far, on a directory that exists, that appears safe
308
365
  # to delete and re-create:
@@ -313,22 +370,32 @@ class Command:
313
370
  else:
314
371
  util.safe_mkdir(self.args['work-dir'])
315
372
  util.debug(f'create_work_dir: created {self.args["work-dir"]}')
316
- if (self.args['keep']):
317
- open(keep_file, 'w').close()
373
+
374
+ if self.args['keep']:
375
+ open(keep_file, 'w', encoding='utf-8').close() # pylint: disable=consider-using-with
318
376
  util.debug(f'create_work_dir: created {keep_file=}')
319
377
  return self.args['work-dir']
320
378
 
321
- def exec(self, work_dir, command_list, background=False, stop_on_error=True,
322
- quiet=False, tee_fpath=None, shell=False):
379
+
380
+ def exec(self, work_dir: str, command_list: list, background: bool = False,
381
+ stop_on_error: bool = True, quiet: bool = False, tee_fpath: str = '',
382
+ shell: bool = False) -> (str, str, int):
383
+ '''Runs a command via subprocess and returns a tuple of (stderr, stdout, rc)
384
+
385
+ - work_dir: is passed to the subprocess call, so os.chdir(..) is not invoked.
386
+ - command_list: should be a list of individual strings w/out spaces.
387
+ - background: if False, does not print to stdout (but will print to any
388
+ tee_fpath or the global log
389
+ - shell: arg passed to subprocess, defaults to False.
390
+ '''
391
+
323
392
  if not tee_fpath and getattr(command_list, 'tee_fpath', None):
324
393
  tee_fpath = getattr(command_list, 'tee_fpath', '')
325
394
  if not quiet:
326
395
  util.info(f"exec: {' '.join(command_list)} (in {work_dir}, {tee_fpath=})")
327
- original_cwd = util.getcwd()
328
- os.chdir(work_dir)
329
396
 
330
- stdout, stderr, return_code = util.subprocess_run_background(
331
- work_dir=None, # we've already called os.chdir(work_dir).
397
+ stdout, stderr, return_code = subprocess_run_background(
398
+ work_dir=work_dir,
332
399
  command_list=command_list,
333
400
  background=background,
334
401
  fake=self.args.get('fake', False),
@@ -336,16 +403,23 @@ class Command:
336
403
  shell=shell
337
404
  )
338
405
 
339
- os.chdir(original_cwd)
340
406
  if return_code:
341
407
  self.status += return_code
342
- if stop_on_error: self.error(f"exec: returned with error (return code: {return_code})")
343
- else : util.debug(f"exec: returned with error (return code: {return_code})")
408
+ if stop_on_error:
409
+ self.error(f"exec: returned with error (return code: {return_code})")
410
+ else:
411
+ util.debug(f"exec: returned with error (return code: {return_code})")
344
412
  else:
345
413
  util.debug(f"exec: returned without error (return code: {return_code})")
346
414
  return stderr, stdout, return_code
347
415
 
348
- def set_arg(self, key, value):
416
+ def set_arg( # pylint: disable=too-many-branches
417
+ self, key, value
418
+ ) -> None:
419
+ '''Sets self.args[key] with value, and self.modified_args[key]=True
420
+
421
+ Does type checking, handles list type append
422
+ '''
349
423
 
350
424
  # Do some minimal type handling, preserving the type(self.args[key])
351
425
  if key not in self.args:
@@ -378,14 +452,14 @@ class Command:
378
452
  else:
379
453
  self.args[key] = True
380
454
  else:
381
- raise Exception(f'set_arg({key=}, {value=}) bool, {type(cur_value)=} {type(value)=}')
455
+ assert False, f'set_arg({key=}, {value=}) bool {type(cur_value)=} {type(value)=}'
382
456
 
383
457
  elif isinstance(cur_value, int):
384
458
  # if int, attempt to convert string or bool --> int
385
459
  if isinstance(value, (bool, int, str)):
386
460
  self.args[key] = int(value)
387
461
  else:
388
- raise Exception(f'set_arg({key=}, {value=}) int, {type(cur_value)=} {type(value)=}')
462
+ assert False, f'set_arg({key=}, {value=}) int {type(cur_value)=} {type(value)=}'
389
463
 
390
464
 
391
465
  else:
@@ -396,7 +470,9 @@ class Command:
396
470
  util.debug(f'Set arg["{key}"]="{self.args[key]}"')
397
471
 
398
472
 
399
- def get_argparser(self, parser_arg_list=None) -> argparse.ArgumentParser:
473
+ def get_argparser( # pylint: disable=too-many-branches
474
+ self, parser_arg_list=None
475
+ ) -> argparse.ArgumentParser:
400
476
  ''' Returns an argparse.ArgumentParser() based on self.args (dict)
401
477
 
402
478
  If parser_arg_list is not None, the ArgumentParser() is created using only the keys in
@@ -404,8 +480,8 @@ class Command:
404
480
  '''
405
481
 
406
482
  # Preference is --args-with-dashes, which then become parsed.args_with_dashes, b/c
407
- # parsed.args-with-dashes is not legal python. Some of self.args.keys() still have - or _, so
408
- # this will handle both.
483
+ # parsed.args-with-dashes is not legal python. Some of self.args.keys() still have - or _,
484
+ # so this will handle both.
409
485
  # Also, preference is for self.args.keys(), to be str with - dashes
410
486
  parser = argparse.ArgumentParser(prog='eda', add_help=False, allow_abbrev=False)
411
487
  bool_action_kwargs = util.get_argparse_bool_action_kwargs()
@@ -439,19 +515,19 @@ class Command:
439
515
  help_kwargs = {'help': f'{type(value).__name__} default={value}'}
440
516
 
441
517
 
442
- # It's important to set the default=None on these, except for list types where default is []
443
- # If the parsed Namespace has values set to None or [], we do not update. This means that as deps
444
- # are processed that have args set, they cannot override the top level args that were already set.
445
- # nor be overriden by defaults.
446
- if type(value) is bool:
447
- # For bool, support --key and --no-key with this action=argparse.BooleanOptionalAction.
448
- # Note, this means you cannot use --some-bool=True, or --some-bool=False, has to be --some-bool
449
- # or --no-some-bool.
518
+ # It's important to set the default=None on these, except for list types where default
519
+ # is []. If the parsed Namespace has values set to None or [], we do not update. This
520
+ # means that as deps are processed that have args set, they cannot override the top
521
+ # level args that were already set, nor be overriden by defaults.
522
+ if isinstance(value, bool):
523
+ # For bool, support --key and --no-key with action=argparse.BooleanOptionalAction.
524
+ # Note, this means you cannot use --some-bool=True, or --some-bool=False, has to
525
+ # be --some-bool or --no-some-bool.
450
526
  parser.add_argument(
451
527
  *arguments, default=None, **bool_action_kwargs, **help_kwargs)
452
- elif type(value) is list:
528
+ elif isinstance(value, list):
453
529
  parser.add_argument(*arguments, default=value, action='append', **help_kwargs)
454
- elif type(value) in [int, str]:
530
+ elif isinstance(value, (int, str)):
455
531
  parser.add_argument(*arguments, default=value, type=type(value), **help_kwargs)
456
532
  elif value is None:
457
533
  parser.add_argument(*arguments, default=None, **help_kwargs)
@@ -462,10 +538,10 @@ class Command:
462
538
 
463
539
 
464
540
  def run_argparser_on_list(
465
- self, tokens: list, parser_arg_list=None, apply_parsed_args:bool=True
541
+ self, tokens: list, parser_arg_list=None, apply_parsed_args: bool = True
466
542
  ) -> (dict, list):
467
- ''' Creates an argparse.ArgumentParser() for all the keys in self.args, and attempts to parse
468
- from the provided list. Parsed args are applied to self.args.
543
+ ''' Creates an argparse.ArgumentParser() for all the keys in self.args, and attempts to
544
+ parse from the provided list. Parsed args are applied to self.args.
469
545
 
470
546
  Returns a tuple of (parsed argparse.Namespace obj, list of unparsed args)
471
547
 
@@ -479,17 +555,16 @@ class Command:
479
555
  parsed, unparsed = parser.parse_known_args(tokens + [''])
480
556
  unparsed = list(filter(None, unparsed))
481
557
  except argparse.ArgumentError:
482
- self.error(f'problem {command_name=} attempting to parse_known_args for {tokens=}')
558
+ self.error(f'problem {self.command_name=} attempting to parse_known_args for {tokens=}')
483
559
 
484
560
  parsed_as_dict = vars(parsed)
485
561
 
486
562
  args_to_be_applied = {}
487
563
 
488
- for key,value in parsed_as_dict.items():
489
- # key should have _ instead of POSIX dashes, but we still support dashes like self.args['build-file'],
490
- # etc.
564
+ for key, value in parsed_as_dict.items():
565
+ # key currently has _ instead of POSIX dashes, so we convert to dashes, for
566
+ # most self.args, like self.args['build-file'], etc.
491
567
  if key not in self.args and '_' in key:
492
- # try with dashes instead of _
493
568
  key = key.replace('_', '-')
494
569
  assert key in self.args, f'{key=} not in {self.args=}'
495
570
 
@@ -501,7 +576,8 @@ class Command:
501
576
  return parsed, unparsed
502
577
 
503
578
 
504
- def apply_args_from_dict(self, args_to_be_applied:dict) -> list:
579
+ def apply_args_from_dict(self, args_to_be_applied: dict) -> list:
580
+ '''Given a dict of key/value for self.args, apply them to self.args'''
505
581
  util.debug('apply_args_from_dict() -- called by:',
506
582
  f'{self.command_name=}, {self.__class__.__name__=},',
507
583
  f'{args_to_be_applied=}')
@@ -523,7 +599,7 @@ class Command:
523
599
  f"with different cur value (cur value={self.args.get(key, None)})")
524
600
  continue
525
601
  if self.args[key] != value:
526
- util.debug(f"Command.run_argparser_on_list - setting set_arg b/c",
602
+ util.debug("Command.run_argparser_on_list - setting set_arg b/c",
527
603
  f" argparse -- {key=} {value=} (cur value={self.args[key]})")
528
604
  self.set_arg(key, value) # Note this has special handling for lists already.
529
605
  self.modified_args[key] = True
@@ -541,7 +617,7 @@ class Command:
541
617
  _, unparsed = self.run_argparser_on_list(tokens)
542
618
  if process_all and len(unparsed) > 0:
543
619
  self.error(f"Didn't understand argument: '{unparsed=}' in",
544
- f" {self.command_name=} context")
620
+ f" {self.command_name=} context, {pwd=}")
545
621
 
546
622
  return unparsed
547
623
 
@@ -574,27 +650,33 @@ class Command:
574
650
 
575
651
  Allows command handlers to make thier own customizations from --config-yaml=YAML.
576
652
  '''
577
- pass
653
+ return
578
654
 
579
655
  def update_tool_config(self) -> None:
580
656
  '''Returns None. Hook for classes like CommandSim to make tool specific overrides.'''
581
- pass
657
+ return
582
658
 
583
659
  def write_eda_config_and_args(self):
660
+ '''Attempts to write eda_output_config.yml to our work-dir'''
584
661
  if not self.args.get('work-dir', None):
585
662
  util.warning(f'Ouput work-dir not set, saving ouput eda_config to {os.getcwd()}')
586
- util.write_eda_config_and_args(dirpath=self.args.get('work-dir', os.getcwd()),
587
- command_obj_ref=self)
663
+ eda_config.write_eda_config_and_args(
664
+ dirpath=self.args.get('work-dir', os.getcwd()), command_obj_ref=self
665
+ )
588
666
 
589
667
  def is_export_enabled(self) -> bool:
590
- # check if any self.args['export'] is set in any way (but not set to False
591
- # or empty list)
592
- return any([arg.startswith('export') and v for arg,v in self.args.items()])
668
+ '''Returns True if any self.args['export'] is set in any way (but not set to False
669
+ or empty list)'''
670
+ return any(arg.startswith('export') and v for arg,v in self.args.items())
593
671
 
594
672
  def run(self) -> None:
673
+ '''Alias for do_it(self)'''
595
674
  self.do_it()
596
675
 
597
676
  def do_it(self) -> None:
677
+ '''Main entrypoint of running a command, usually called by process_tokens.
678
+
679
+ process_tokens(..) is the starting entrypoint from eda.py'''
598
680
  self.write_eda_config_and_args()
599
681
  self.error(f"No tool bound to command '{self.command_name}', you",
600
682
  " probably need to setup tool, or use '--tool <name>'")
@@ -610,7 +692,9 @@ class Command:
610
692
  self.set_tool_defines()
611
693
 
612
694
 
613
- def help(self, tokens: list = []) -> None:
695
+ def help( # pylint: disable=dangerous-default-value,too-many-branches
696
+ self, tokens: list = []
697
+ ) -> None:
614
698
  '''Since we don't quite follow standard argparger help()/usage(), we'll format our own
615
699
 
616
700
  if self.args_help has additional help information.
@@ -618,7 +702,7 @@ class Command:
618
702
 
619
703
  # Indent long lines (>100) to indent=56 (this is where we leave off w/ {vstr:12} below.
620
704
  def indent_me(text:str):
621
- return util.indent_wrap_long_text(text, width=100, indent=56)
705
+ return indent_wrap_long_text(text, width=100, indent=56)
622
706
 
623
707
  util.info('Help:')
624
708
  # using bare 'print' here, since help was requested, avoids --color and --quiet
@@ -637,14 +721,14 @@ class Command:
637
721
  lines.append(f"Generic help for command='{self.command_name}'"
638
722
  f" (using '{self.__class__.__name__}')")
639
723
  else:
640
- lines.append(f"Generic help (from class Command):")
724
+ lines.append("Generic help (from class Command):")
641
725
 
642
726
  # Attempt to run argparser on args, but don't error if it fails.
643
727
  unparsed = []
644
728
  if tokens:
645
729
  try:
646
730
  _, unparsed = self.run_argparser_on_list(tokens=tokens)
647
- except:
731
+ except Exception:
648
732
  pass
649
733
 
650
734
  for k in sorted(self.args.keys()):
@@ -653,13 +737,13 @@ class Command:
653
737
  khelp = self.args_help.get(k, '')
654
738
  if khelp:
655
739
  khelp = f' - {khelp}'
656
- if type(v) == bool :
740
+ if isinstance(v, bool):
657
741
  lines.append(indent_me(f" --{k:20} : boolean : {vstr:12}{khelp}"))
658
- elif type(v) == int:
742
+ elif isinstance(v, int):
659
743
  lines.append(indent_me(f" --{k:20} : integer : {vstr:12}{khelp}"))
660
- elif type(v) == list:
744
+ elif isinstance(v, list):
661
745
  lines.append(indent_me(f" --{k:20} : list : {vstr:12}{khelp}"))
662
- elif type(v) == str:
746
+ elif isinstance(v, str):
663
747
  vstr = "'" + v + "'"
664
748
  lines.append(indent_me(f" --{k:20} : string : {vstr:12}{khelp}"))
665
749
  else:
@@ -672,7 +756,10 @@ class Command:
672
756
  print(f'Unparsed args: {unparsed}')
673
757
 
674
758
 
675
- class CommandDesign(Command):
759
+ class CommandDesign(Command): # pylint: disable=too-many-instance-attributes
760
+ '''CommandDesign is the eda base class for command handlers that need to track files.
761
+
762
+ This is the base class for CommandSim, CommandSynth, and others.'''
676
763
 
677
764
  # Used by for DEPS work_dir_add_srcs@ commands, by class methods:
678
765
  # update_file_lists_for_work_dir(..), and resolve_target(..)
@@ -693,11 +780,16 @@ class CommandDesign(Command):
693
780
  self.args_help.update({
694
781
  'seed': 'design seed, default is 31-bit non-zero urandom',
695
782
  'top': 'TOP level verilog/SV module or VHDL entity for this target',
696
- 'all-sv': ('Maintain .sv and .v in single file list.'
697
- ' False: .sv flist separate from .v flist and separate compile(s)'
698
- ' True: .sv and .v files compiled together if possible'),
699
- 'unprocessed-plusargs': 'Args that began with +, but were not +define+ or +incdir+, +<name>, ' \
700
- + ' or +<name>=<value>. These become tool dependent, for example "sim" commands will treat as sim-plusargs',
783
+ 'all-sv': (
784
+ 'Maintain .sv and .v in single file list.'
785
+ ' False: .sv flist separate from .v flist and separate compile(s)'
786
+ ' True: .sv and .v files compiled together if possible'
787
+ ),
788
+ 'unprocessed-plusargs': (
789
+ 'Args that began with +, but were not +define+ or +incdir+, +<name>, '
790
+ ' or +<name>=<value>. These become tool dependent, for example "sim" commands will'
791
+ ' treat as sim-plusargs'
792
+ ),
701
793
  })
702
794
  self.defines = {}
703
795
  self.incdirs = []
@@ -715,12 +807,18 @@ class CommandDesign(Command):
715
807
  for (d,v) in self.config.get('defines', {}).items():
716
808
  self.defines[d] = v
717
809
 
718
- self.cached_deps = {} # key = abspath of DEPS markup file, has sub-dicts for 'data' and 'line_numbers'.
810
+ # cached_deps: key = abspath of DEPS markup file, value is a dict with
811
+ # keys 'data' and 'line_numbers'
812
+ self.cached_deps = {}
719
813
  self.targets_dict = {} # key = targets that we've already processed in DEPS files
720
814
  self.last_added_source_file_inferred_top = ''
721
815
 
722
- def run_dep_commands(self):
723
- # Run any shell@ commands from DEPS files
816
+
817
+ def run_dep_commands(self) -> None:
818
+ '''Run shell/peakrdl style commands from DEPS files
819
+
820
+ These are deferred to maintain the deps ordering, and run in that order.
821
+ '''
724
822
  self.run_dep_shell_commands()
725
823
  # Update any work_dir_add_srcs@ in our self.files, self.files_v, etc, b/c
726
824
  # self.args['work-dir'] now exists.
@@ -728,7 +826,10 @@ class CommandDesign(Command):
728
826
  # Link any non-sources to our work-dir:
729
827
  self.update_non_source_files_in_work_dir()
730
828
 
731
- def run_dep_shell_commands(self):
829
+
830
+ def run_dep_shell_commands(self) -> None:
831
+ '''Specifically runs shell command from DEPS files'''
832
+
732
833
  # Runs from self.args['work-dir']
733
834
  all_cmds_lists = []
734
835
 
@@ -747,7 +848,8 @@ class CommandDesign(Command):
747
848
  log_fnames_count.update({target_node: lognum + 1})
748
849
  all_cmds_lists += [
749
850
  [], # blank line
750
- # comment, where it came from, log to {node}__shell_{lognum}.log (or tee name from DEPS.yml)
851
+ # comment, where it came from, log to {node}__shell_{lognum}.log
852
+ # (or tee name from DEPS.yml)
751
853
  [f'# command {i}: target: {d["target_path"]} : {target_node} --> {log}'],
752
854
  ]
753
855
  if not run_from_work_dir:
@@ -761,10 +863,12 @@ class CommandDesign(Command):
761
863
 
762
864
  d['exec_list'] = clist # update to tee_fpath is set.
763
865
 
764
- util.write_shell_command_file(dirpath=self.args['work-dir'], filename='pre_compile_dep_shell_commands.sh',
765
- command_lists=all_cmds_lists)
866
+ util.write_shell_command_file(
867
+ dirpath=self.args['work-dir'], filename='pre_compile_dep_shell_commands.sh',
868
+ command_lists=all_cmds_lists
869
+ )
766
870
 
767
- for i,d in enumerate(self.dep_shell_commands):
871
+ for i, d in enumerate(self.dep_shell_commands):
768
872
  util.info(f'run_dep_shell_commands {i=}: {d=}')
769
873
  clist = util.ShellCommandList(d['exec_list'])
770
874
  tee_fpath=clist.tee_fpath
@@ -782,16 +886,23 @@ class CommandDesign(Command):
782
886
  self.exec(run_from_dir, clist, tee_fpath=tee_fpath,
783
887
  shell=self.config.get('deps_subprocess_shell', False))
784
888
 
785
- def update_file_lists_for_work_dir(self):
786
- if len(self.dep_work_dir_add_srcs) == 0:
889
+
890
+ def update_file_lists_for_work_dir(self) -> None:
891
+ '''Handles any source files that were creating by "shell" style commands in the
892
+
893
+ work-dir, these need to be added via self.add_file(..) in the correct order
894
+ of the dependencies. Basically, any files that were left with @EDA-WORK-DIR@ prefix
895
+ need to be patched and added.
896
+ '''
897
+ if not self.dep_work_dir_add_srcs:
787
898
  return
788
899
 
789
- # If we encounter any @EDA-WORK_DIR@some_file.v in self.files, self.files_v, etc, then replace it with:
790
- # self.args['work-dir'] / some_file.v:
900
+ # If we encounter any @EDA-WORK_DIR@some_file.v in self.files, self.files_v, etc,
901
+ # then replace it with: self.args['work-dir'] / some_file.v:
791
902
  _work_dir_add_srcs_path_string_len = len(self._work_dir_add_srcs_path_string)
792
903
  work_dir_abspath = os.path.abspath(self.args['work-dir'])
793
- for key in list(self.files.keys()): # list so it's not an iterator, we're updating self.files.
794
- if type(key) is str and key.startswith(self._work_dir_add_srcs_path_string):
904
+ for key in list(self.files.keys()): # list so it's not an iterator, updates self.files.
905
+ if isinstance(key, str) and key.startswith(self._work_dir_add_srcs_path_string):
795
906
  new_key = os.path.join(work_dir_abspath, key[_work_dir_add_srcs_path_string_len :])
796
907
  self.files.pop(key)
797
908
  self.files[new_key] = True
@@ -800,12 +911,23 @@ class CommandDesign(Command):
800
911
  self.files_sdc]
801
912
  for my_file_list in my_file_lists_list:
802
913
  for i,value in enumerate(my_file_list):
803
- if value and type(value) is str and value.startswith(self._work_dir_add_srcs_path_string):
804
- new_value = os.path.join(work_dir_abspath, value[_work_dir_add_srcs_path_string_len :])
914
+ if value and isinstance(value, str) and \
915
+ value.startswith(self._work_dir_add_srcs_path_string):
916
+ new_value = os.path.join(
917
+ work_dir_abspath, value[_work_dir_add_srcs_path_string_len :]
918
+ )
805
919
  my_file_list[i] = new_value
806
920
  util.debug(f"file lists: replaced {value} with {new_value}")
807
921
 
808
- def update_non_source_files_in_work_dir(self):
922
+
923
+ def update_non_source_files_in_work_dir(self) -> None:
924
+ '''For non-source files (that are tracked as 'reqs' in DEPS markup files) these
925
+
926
+ need to be copied or linked to the work-dir. For example, if some SV assumes it
927
+ can $readmemh('file_that_is_here.txt') but we're running out of work-dir. Linking
928
+ is the easy work-around vs trying to run-in-place of all SV files.
929
+ '''
930
+
809
931
  for fname in self.files_non_source:
810
932
  _, leaf_fname = os.path.split(fname)
811
933
  destfile = os.path.join(self.args['work-dir'], leaf_fname)
@@ -815,16 +937,25 @@ class CommandDesign(Command):
815
937
  util.info(f'{fname=} {self.files_caller_info=}')
816
938
  self.error(f'Non-source file (reqs?) {relfname=} does not exist from {caller_info}')
817
939
  elif not os.path.exists(destfile):
818
- util.debug(f'updating non-source file to work-dir: Linked {fname=} to {destfile=}, from {caller_info}')
940
+ util.debug(f'updating non-source file to work-dir: Linked {fname=} to {destfile=},',
941
+ f'from {caller_info}')
819
942
  if sys.platform == "win32":
820
943
  shutil.copyfile(fname, destfile) # On Windows, fall back to copying
821
944
  else:
822
945
  os.symlink(src=fname, dst=destfile)
823
946
 
824
- def get_top_name(self, name):
947
+ @staticmethod
948
+ def get_top_name(name: str) -> str:
949
+ '''Attempt to get the 'top' module name from a file, such as path/to/mine.sv will
950
+
951
+ return "mine"'''
952
+ # TODO(drew): Use the helper method in util for this instead to peek in file contents?
825
953
  return os.path.splitext(os.path.basename(name))[0]
826
954
 
827
- def process_plusarg(self, plusarg: str, pwd: str = os.getcwd()):
955
+
956
+ def process_plusarg( # pylint: disable=too-many-branches
957
+ self, plusarg: str, pwd: str = os.getcwd()
958
+ ) -> None:
828
959
  '''Retuns str, parses a +define+, +incdir+, +key=value str; adds to internal.
829
960
 
830
961
  Adds to self.defines, self.incdirs,
@@ -836,12 +967,12 @@ class CommandDesign(Command):
836
967
  # args that come from shlex.quote(token), such as:
837
968
  # token = '\'+define+OC_ROOT="/foo/bar/opencos"\''
838
969
  # So we strip all outer ' or " on the plusarg:
839
- plusarg = util.strip_outer_quotes(plusarg)
970
+ plusarg = strip_outer_quotes(plusarg)
840
971
  if not pwd:
841
972
  pwd = ''
842
973
 
843
974
  if plusarg.startswith('+define+'):
844
- plusarg = plusarg.lstrip('+define+')
975
+ plusarg = plusarg[len('+define+'):]
845
976
  m = re.match(r'^(\w+)$', plusarg)
846
977
  if m:
847
978
  k = m.group(1)
@@ -849,11 +980,12 @@ class CommandDesign(Command):
849
980
  util.debug(f"Defined {k}")
850
981
  return None
851
982
  m = re.match(r'^(\w+)\=(\S+)$', plusarg)
852
- if not m: m = re.match(r'^(\w+)\=(\"[^\"]*\")$', plusarg)
983
+ if not m:
984
+ m = re.match(r'^(\w+)\=(\"[^\"]*\")$', plusarg)
853
985
  if m:
854
986
  k = m.group(1)
855
987
  v = m.group(2)
856
- if v and type(v) is str:
988
+ if v and isinstance(v, str):
857
989
  if v.startswith('%PWD%/'):
858
990
  v = v.replace('%PWD%', os.path.abspath(pwd))
859
991
  if v.startswith('%SEED%'):
@@ -865,7 +997,7 @@ class CommandDesign(Command):
865
997
  return None
866
998
 
867
999
  if plusarg.startswith('+incdir+'):
868
- plusarg = plusarg.lstrip('+incdir+')
1000
+ plusarg = plusarg[len('+incdir+'):]
869
1001
  m = re.match(r'^(\S+)$', plusarg)
870
1002
  if m:
871
1003
  incdir = m.group(1)
@@ -881,40 +1013,59 @@ class CommandDesign(Command):
881
1013
  if not self.config.get('bare_plusarg_supported', False):
882
1014
  self.error(f"bare plusarg(s) are not supported: {plusarg}'")
883
1015
  return None
884
- elif plusarg not in self.args['unprocessed-plusargs']:
1016
+ if plusarg not in self.args['unprocessed-plusargs']:
885
1017
  self.args['unprocessed-plusargs'].append(plusarg)
886
1018
  # For anything added to unprocessed-plusarg, we have to return it, to let
887
1019
  # derived classes have the option to handle it
888
1020
  return plusarg
889
- else:
890
- self.error(f"Didn't understand +plusarg: '{plusarg}'")
1021
+
1022
+ self.error(f"Didn't understand +plusarg: '{plusarg}'")
891
1023
  return None
892
1024
 
893
1025
 
894
- def append_shell_commands(self, cmds : list):
895
- # Each entry in cmds (list) should be a dict with keys ['target_node', 'target_path', 'exec_list']
1026
+ def append_shell_commands(self, cmds : list) -> None:
1027
+ ''' Given a cmds (list), where each item is a dict with:
1028
+
1029
+ { 'target_node': str,
1030
+ 'target_path': str,
1031
+ 'exec_list': list
1032
+ }
1033
+ add to this class's dep_shell_commands for deferred processing
1034
+ '''
1035
+
896
1036
  for entry in cmds:
897
- if entry is None or type(entry) is not dict:
1037
+ if entry is None or not isinstance(entry, dict):
898
1038
  continue
899
1039
  if entry in self.dep_shell_commands:
900
- # we've already run this exact command (target node, target path, exec list), don't run it
901
- # again
1040
+ # we've already run this exact command (target node, target path, exec list),
1041
+ # don't run it again
902
1042
  continue
903
1043
 
904
1044
  assert 'exec_list' in entry, f'{entry=}'
905
1045
  util.debug(f'adding - dep_shell_command: {entry=}')
906
1046
  self.dep_shell_commands.append(entry)
907
1047
 
908
- def append_work_dir_add_srcs(self, add_srcs: list, caller_info: str):
909
- # Each entry in add_srcs (list) should be a dict with keys ['target_node', 'target_path', 'file_list']
1048
+
1049
+ def append_work_dir_add_srcs(self, add_srcs: list, caller_info: str) -> None:
1050
+ '''Given add_srcs (list), where each item is a dict with:
1051
+
1052
+ { 'target_node': str,
1053
+ 'target_path': str,
1054
+ 'file_list': list
1055
+ }
1056
+
1057
+ adds files to set self.dep_work_dir_add_src, and call add_file on it, basically
1058
+ resolving adding source files that were generated and residing in the work-dir.
1059
+ '''
910
1060
  for entry in add_srcs:
911
- if entry is None or type(entry) is not dict:
1061
+ if entry is None or not isinstance(entry, dict):
912
1062
  continue
913
1063
 
914
1064
  work_dir_files = entry['file_list']
915
1065
  for filename in work_dir_files:
916
- # Unfortunately, self.args['work-dir'] doesn't exist yet and hasn't been set, so we'll add these
917
- # files as '@EDA-WORK_DIR@' + filename, and have to replace the EDA-WORK_DIR@ string later in our flow.
1066
+ # Unfortunately, self.args['work-dir'] doesn't exist yet and hasn't been set,
1067
+ # so we'll add these files as '@EDA-WORK_DIR@' + filename, and have to replace
1068
+ # the EDA-WORK_DIR@ string later in our flow.
918
1069
  filename_use = self._work_dir_add_srcs_path_string + filename
919
1070
  dep_key_tuple = (
920
1071
  entry['target_path'],
@@ -929,14 +1080,25 @@ class CommandDesign(Command):
929
1080
  self.dep_work_dir_add_srcs.add(dep_key_tuple) # add to set()
930
1081
  elif dep_key_tuple not in self.dep_work_dir_add_srcs:
931
1082
  # we've already added the file so this dep was skipped for this one file.
932
- util.warning(f'work_dir_add_srcs@ {dep_key_tuple=} but {filename_use=}' \
933
- + 'is already in self.files (duplicate dependency on generated file?)')
1083
+ util.warning(f'work_dir_add_srcs@ {dep_key_tuple=} but {filename_use=}',
1084
+ 'is already in self.files (duplicate dependency on generated',
1085
+ 'file?)')
1086
+
934
1087
 
1088
+ def resolve_target(
1089
+ self, target: str, no_recursion: bool = False, caller_info: str = ''
1090
+ ) -> bool:
1091
+ '''Returns True if target is found. Entry point for resolving a CLI path based target name.
935
1092
 
936
- def resolve_target(self, target, no_recursion=False, caller_info=''):
937
- util.debug("Entered resolve_target(%s)" % (target))
938
- # self.target is a name we grab for the job (i.e. for naming work dir etc). we don't want the path prefix.
939
- # TODO: too messy -- there's also a self.target, args['job-name'], args['work-dir'], args['design'], args['top'], args['sub-work-dir'] ...
1093
+ Will recursively call resolve_target_core, looking for a filename existing
1094
+ or DEPS markup file in that path and a target (key) in the markup table.
1095
+
1096
+ This will populate all source file dependencies, commands, and other DEPS markup
1097
+ features as the target is resolved.
1098
+ '''
1099
+ util.debug(f"Entered resolve_target({target})")
1100
+ # self.target is a name we grab for the job (i.e. for naming work dir etc). we don't want
1101
+ # the path prefix.
940
1102
 
941
1103
  self.target_path, self.target = os.path.split(target)
942
1104
 
@@ -950,7 +1112,7 @@ class CommandDesign(Command):
950
1112
  # If the target is a file (we're at the root here processing CLI arg tokens)
951
1113
  # and that file exists and has an extension, then there's no reason to go looking
952
1114
  # in DEPS files, add the file and return True.
953
- file_base, file_ext = os.path.splitext(fpath)
1115
+ _, file_ext = os.path.splitext(fpath)
954
1116
  if forced_extension or file_ext:
955
1117
  self.add_file(fpath, caller_info=caller_info,
956
1118
  forced_extension=forced_extension)
@@ -958,23 +1120,25 @@ class CommandDesign(Command):
958
1120
 
959
1121
  return self.resolve_target_core(target, no_recursion, caller_info)
960
1122
 
961
- def resolve_target_core(self, target, no_recursion, caller_info=''):
962
- util.debug("Entered resolve_target_core(%s)" % (target))
1123
+ def resolve_target_core( # pylint: disable=too-many-locals,too-many-branches
1124
+ self, target: str, no_recursion: bool, caller_info: str = ''
1125
+ ) -> bool:
1126
+ '''Returns True if target is found. recursive point for resolving path or DEPS markup
1127
+ target names.'''
1128
+
1129
+ util.debug(f"Entered resolve_target_core({target=})")
963
1130
  found_target = False
964
- util.debug("Starting to resolve target '%s'" % (target))
965
1131
  target_path, target_node = os.path.split(target)
966
1132
 
967
- deps = None
968
- data = None
1133
+ deps, data, deps_file = None, None, None
969
1134
  found_deps_file = False
970
1135
 
971
1136
  if self.config['deps_markup_supported']:
972
- deps = deps_helpers.DepsFile(
1137
+ deps = DepsFile(
973
1138
  command_design_ref=self, target_path=target_path, cache=self.cached_deps
974
1139
  )
975
1140
  deps_file = deps.deps_file
976
1141
  data = deps.data
977
- data_only_lines = deps.line_numbers
978
1142
 
979
1143
  # Continue if we have data, otherwise look for files other than DEPS.<yml|yaml>
980
1144
  if data is not None:
@@ -987,7 +1151,7 @@ class CommandDesign(Command):
987
1151
 
988
1152
  # For convenience, use an external class for this DEPS.yml table/dict
989
1153
  # This could be re-used for any markup DEPS.json, DEPS.toml, DEPS.py, etc.
990
- deps_processor = deps_helpers.DepsProcessor(
1154
+ deps_processor = DepsProcessor(
991
1155
  command_design_ref = self,
992
1156
  deps_entry = entry,
993
1157
  target = target,
@@ -1008,10 +1172,11 @@ class CommandDesign(Command):
1008
1172
  # Recurse on the returned deps (ordered list), if they haven't already been traversed.
1009
1173
  for x in deps_targets_to_resolve:
1010
1174
  caller_info = deps.gen_caller_info(target_node)
1011
- if x and type(x) is tuple:
1175
+ if x and isinstance(x, tuple):
1012
1176
  # if deps_processor.process_deps_entry() gave us a tuple, it's an
1013
1177
  # unprocessed 'command' that we kept in order until now. Append it.
1014
- assert len(x) == 2, f'command tuple {x=} must be len 2, {target_node=} {deps_file=}'
1178
+ assert len(x) == 2, \
1179
+ f'command tuple {x=} must be len 2, {target_node=} {deps_file=}'
1015
1180
  shell_commands_list, work_dir_add_srcs_list = x
1016
1181
  self.append_shell_commands( cmds=shell_commands_list )
1017
1182
  self.append_work_dir_add_srcs( add_srcs=work_dir_add_srcs_list,
@@ -1025,32 +1190,40 @@ class CommandDesign(Command):
1025
1190
  forced_extension=forced_extension)
1026
1191
  else:
1027
1192
  util.debug(f' ... Calling resolve_target_core({x=})')
1028
- found_target |= self.resolve_target_core( x, no_recursion, caller_info=caller_info)
1193
+ found_target |= self.resolve_target_core(
1194
+ x, no_recursion, caller_info=caller_info
1195
+ )
1029
1196
 
1030
1197
 
1031
1198
  # Done with DEPS.yml if it existed.
1032
1199
 
1033
1200
  if not found_target:
1034
- util.debug("Haven't been able to resolve %s via DEPS" % (target))
1201
+ util.debug(f"Haven't been able to resolve {target=} via DEPS")
1035
1202
  known_file_extensions_for_source = []
1036
- for x in ['verilog', 'systemverilog', 'vhdl', 'cpp']:
1037
- known_file_extensions_for_source += self.config.get('file_extensions', {}).get(x, [])
1203
+ for x in ('verilog', 'systemverilog', 'vhdl', 'cpp'):
1204
+ known_file_extensions_for_source += self.config.get(
1205
+ 'file_extensions', {}).get(x, [])
1038
1206
  for e in known_file_extensions_for_source:
1039
1207
  try_file = target + e
1040
- util.debug("Looking for %s" % (try_file))
1208
+ util.debug(f"Looking for file {try_file}")
1041
1209
  if os.path.exists(try_file):
1042
- self.add_file(try_file, caller_info=str('n/a::' + target + '::n/a'))
1210
+ self.add_file(try_file, caller_info=f'n/a::{target}::n/a')
1043
1211
  found_target = True
1044
1212
  break # move on to the next target
1045
1213
  if not found_target: # if STILL not found_this_target...
1046
- self.error("Unable to resolve target '%s'" % (target))
1214
+ self.error(f"Unable to resolve {target=}")
1047
1215
 
1048
1216
  # if we've found any target since being called, it means we found the one we were called for
1049
1217
  return found_target
1050
1218
 
1051
- def add_file(self, filename, use_abspath=True, add_to_non_sources=False,
1052
- caller_info:str='', forced_extension:str=''):
1053
- file_base, file_ext = os.path.splitext(filename)
1219
+ def add_file(
1220
+ self, filename: str, use_abspath: bool = True, add_to_non_sources: bool = False,
1221
+ caller_info: str = '', forced_extension: str = ''
1222
+ ) -> str:
1223
+ '''Given a filename, add it to one of self.files_sv or similar lists
1224
+
1225
+ based on file extension or prefix directive.'''
1226
+ _, file_ext = os.path.splitext(filename)
1054
1227
  if use_abspath:
1055
1228
  file_abspath = os.path.abspath(filename)
1056
1229
  else:
@@ -1058,8 +1231,8 @@ class CommandDesign(Command):
1058
1231
 
1059
1232
 
1060
1233
  if file_abspath in self.files:
1061
- util.debug("Not adding file %s, already have it" % (file_abspath))
1062
- return
1234
+ util.debug(f"Not adding file {file_abspath}, already have it")
1235
+ return ''
1063
1236
 
1064
1237
  known_file_ext_dict = self.config.get('file_extensions', {})
1065
1238
  v_file_ext_list = known_file_ext_dict.get('verilog', [])
@@ -1082,34 +1255,43 @@ class CommandDesign(Command):
1082
1255
 
1083
1256
  if add_to_non_sources:
1084
1257
  self.files_non_source.append(file_abspath)
1085
- util.debug("Added non-source file file %s as %s" % (filename, file_abspath))
1258
+ util.debug(f"Added non-source file file {filename} as {file_abspath}")
1086
1259
  elif file_ext in v_file_ext_list and not self.args['all-sv']:
1087
1260
  self.files_v.append(file_abspath)
1088
- util.debug("Added Verilog file %s as %s" % (filename, file_abspath))
1089
- elif file_ext in sv_file_ext_list or ((file_ext in v_file_ext_list) and self.args['all-sv']):
1261
+ util.debug(f"Added Verilog file {filename} as {file_abspath}")
1262
+ elif file_ext in sv_file_ext_list or \
1263
+ ((file_ext in v_file_ext_list) and self.args['all-sv']):
1090
1264
  self.files_sv.append(file_abspath)
1091
- util.debug("Added SystemVerilog file %s as %s" % (filename, file_abspath))
1265
+ util.debug(f"Added SystemVerilog file {filename} as {file_abspath}")
1092
1266
  elif file_ext in vhdl_file_ext_list:
1093
1267
  self.files_vhd.append(file_abspath)
1094
- util.debug("Added VHDL file %s as %s" % (filename, file_abspath))
1268
+ util.debug(f"Added VHDL file {filename} as {file_abspath}")
1095
1269
  elif file_ext in cpp_file_ext_list:
1096
1270
  self.files_cpp.append(file_abspath)
1097
- util.debug("Added C++ file %s as %s" % (filename, file_abspath))
1271
+ util.debug(f"Added C++ file {filename} as {file_abspath}")
1098
1272
  elif file_ext in sdc_file_ext_list:
1099
1273
  self.files_sdc.append(file_abspath)
1100
- util.debug("Added SDC file %s as %s" % (filename, file_abspath))
1274
+ util.debug(f"Added SDC file {filename} as {file_abspath}")
1101
1275
  else:
1102
1276
  # unknown file extension. In these cases we link the file to the working directory
1103
- # so it is available (for example, a .mem file that is expected to exist with relative path)
1277
+ # so it is available (for example, a .mem file that is expected to exist with relative
1278
+ # path)
1104
1279
  self.files_non_source.append(file_abspath)
1105
- util.debug("Added non-source file %s as %s" % (filename, file_abspath))
1280
+ util.debug(f"Added non-source file {filename} as {file_abspath}")
1106
1281
 
1107
1282
  self.files[file_abspath] = True
1108
1283
  self.files_caller_info[file_abspath] = caller_info
1109
1284
  return file_abspath
1110
1285
 
1111
- def process_tokens(self, tokens: list, process_all: bool = True,
1112
- pwd: str = os.getcwd()) -> list:
1286
+ def process_tokens( # pylint: disable=too-many-locals,too-many-branches,too-many-statements
1287
+ self, tokens: list, process_all: bool = True,
1288
+ pwd: str = os.getcwd()
1289
+ ) -> list:
1290
+ '''Returns a list of unparsed args (self.args did not have these keys present)
1291
+
1292
+ Entrypoint from eda.py calling a command handler obj.process_tokens(args).
1293
+ Dervied classes are expected to call this parent process_tokens to parse and handle
1294
+ their self.args dict'''
1113
1295
 
1114
1296
  util.debug(f'CommandDesign - process_tokens start - {tokens=}')
1115
1297
 
@@ -1123,19 +1305,21 @@ class CommandDesign(Command):
1123
1305
  # walk the list, remove all items after we're done.
1124
1306
  remove_list = []
1125
1307
  for token in unparsed:
1126
- # Since this is a raw argparser, we may have args that come from shlex.quote(token), such as:
1127
- # token = '\'+define+OC_ROOT="/foo/bar/opencos"\''
1128
- # So we have to check for strings that have been escaped for shell with extra single quotes.
1308
+ # Since this is a raw argparser, we may have args that come from shlex.quote(token),
1309
+ # such as:
1310
+ # token = '\'+define+OC_ROOT="/foo/bar/opencos"\''
1311
+ # So we have to check for strings that have been escaped for shell with extra single
1312
+ # quotes.
1129
1313
  m = re.match(r"^\'?\+\w+", token)
1130
1314
  if m:
1131
1315
  # Copy and strip all outer ' or " on the plusarg:
1132
- plusarg = util.strip_outer_quotes(token)
1316
+ plusarg = strip_outer_quotes(token)
1133
1317
  self.process_plusarg(plusarg, pwd=pwd)
1134
1318
  remove_list.append(token)
1135
1319
  for x in remove_list:
1136
1320
  unparsed.remove(x)
1137
1321
 
1138
- if len(unparsed) == 0 and self.error_on_no_files_or_targets:
1322
+ if not unparsed and self.error_on_no_files_or_targets:
1139
1323
  # derived classes can set error_on_no_files_or_targets=True
1140
1324
  # For example: CommandSim will error (requires files/targets),
1141
1325
  # CommandWaves does not (files/targets not required)
@@ -1143,11 +1327,11 @@ class CommandDesign(Command):
1143
1327
  # check the DEPS markup file for a single target, and if so run it
1144
1328
  # on the one and only target.
1145
1329
  if self.config['deps_markup_supported']:
1146
- deps = deps_helpers.DepsFile(
1330
+ deps = DepsFile(
1147
1331
  command_design_ref=self, target_path=os.getcwd(), cache=self.cached_deps
1148
1332
  )
1149
1333
  if deps.deps_file and deps.data:
1150
- all_targets = deps_helpers.deps_data_get_all_targets(deps.data)
1334
+ all_targets = deps_data_get_all_targets(deps.data)
1151
1335
  if all_targets:
1152
1336
  target = all_targets[-1]
1153
1337
  unparsed.append(target)
@@ -1169,7 +1353,7 @@ class CommandDesign(Command):
1169
1353
  file_exists, fpath, forced_extension = files.get_source_file(token)
1170
1354
  if file_exists:
1171
1355
  file_abspath = os.path.abspath(fpath)
1172
- file_base, file_ext = os.path.splitext(file_abspath)
1356
+ _, file_ext = os.path.splitext(file_abspath)
1173
1357
  if not forced_extension and not file_ext:
1174
1358
  # This probably isn't a file we want to use
1175
1359
  util.warning(f'looking for deps {token=}, found {file_abspath=}' \
@@ -1189,11 +1373,12 @@ class CommandDesign(Command):
1189
1373
  remove_list.append(token)
1190
1374
  continue # done with token, consume it, we added the file.
1191
1375
 
1192
- # we appear to be dealing with a target name which needs to be resolved (usually recursively)
1376
+ # we appear to be dealing with a target name which needs to be resolved (usually
1377
+ # recursively)
1193
1378
  if token.startswith(os.sep):
1194
1379
  target_name = token # if it's absolute path, don't prepend anything
1195
1380
  else:
1196
- target_name = os.path.join(".", token) # prepend ./so that we always have a <path>/<file>
1381
+ target_name = os.path.join(".", token) # prepend ./ to make it like: <path>/<file>
1197
1382
 
1198
1383
  util.debug(f'Calling self.resolve_target on {target_name=} ({token=})')
1199
1384
  if self.resolve_target(target_name, caller_info=caller_info):
@@ -1242,49 +1427,81 @@ class CommandDesign(Command):
1242
1427
  self.target = self.args['top']
1243
1428
 
1244
1429
  if self.error_on_missing_top and not self.args.get('top', ''):
1245
- self.error(f"Did not get a --top or DEPS top, required to run command",
1430
+ self.error("Did not get a --top or DEPS top, required to run command",
1246
1431
  f"'{self.command_name}' for tool={self.args.get('tool', None)}")
1247
1432
 
1248
1433
  return unparsed
1249
1434
 
1250
1435
 
1251
- def get_command_line_args(self, remove_args:list=[], remove_args_startswith:list=[]) -> list:
1436
+ def get_command_line_args( # pylint: disable=dangerous-default-value
1437
+ self, remove_args: list = [], remove_args_startswith: list = []
1438
+ ) -> list:
1252
1439
  '''Returns a list of all the args if you wanted to re-run this command
1253
1440
  (excludes eda, command, target).'''
1254
1441
 
1255
1442
  # This will not set bool's that are False, does not add --no-<somearg>
1256
- # nor --<somearg>=False
1257
- # This will not set str's that are empty.
1258
- # this will not set ints that are 0
1443
+ # nor --<somearg>=False (it's unclear which are 'store_true' v/ bool-action-kwargs)
1444
+ # This will not set str's that are empty
1445
+ # this will not set ints that are 0 (unless modified)
1446
+ # TODO(drew): we may want to revisit this if we tracked default arg values,
1447
+ # or perhaps if they have been "modified".
1259
1448
  ret = []
1260
1449
  for k,v in self.args.items():
1261
1450
 
1262
1451
  # Some args cannot be extracted and work, so omit these:
1263
1452
  if k in ['top-path'] + remove_args:
1264
1453
  continue
1265
- if any([k.startswith(x) for x in remove_args_startswith]):
1454
+ if any(k.startswith(x) for x in remove_args_startswith):
1266
1455
  continue
1267
1456
 
1268
- if type(v) is bool and v:
1457
+ is_modified = self.modified_args.get(k, False)
1458
+
1459
+ if isinstance(v, bool) and v:
1269
1460
  ret.append(f'--{k}')
1270
- elif type(v) is int and bool(v):
1461
+ elif isinstance(v, int) and (bool(v) or is_modified):
1271
1462
  ret.append(f'--{k}={v}')
1272
- elif type(v) is str and v:
1463
+ elif isinstance(v, str) and v:
1273
1464
  ret.append(f'--{k}={v}')
1274
- elif type(v) is list:
1465
+ elif isinstance(v, list):
1275
1466
  for item in v:
1276
- if item or type(item) not in [bool, str]:
1467
+ if item or isinstance(item, (bool, str)):
1277
1468
  # don't print bool/str that are blank.
1278
1469
  ret.append(f'--{k}={item}') # lists append
1279
1470
 
1280
1471
  return ret
1281
1472
 
1282
1473
 
1283
- _threads_start = 0
1284
- _threads_done = 0
1474
+ #_THREADS_START = 0
1475
+ #_THREADS_DONE = 0
1476
+
1477
+ class ThreadStats:
1478
+ '''To avoid globals for two ints, keep a class holder so CommandParallel and
1479
+ CommandParallelWorker can share values'''
1480
+
1481
+ done = 0
1482
+ started = 0
1483
+
1484
+ def all_done(self) -> bool:
1485
+ '''Returns True if all started jobs are done'''
1486
+ return self.started == self.done
1487
+
1488
+ def get_remaining(self) -> int:
1489
+ '''Returns remaining number of jobs'''
1490
+ return max(self.started - self.done, 0)
1491
+
1492
+ def __repr__(self) -> str:
1493
+ '''Pretty print DONE/STARTED'''
1494
+ return f'{self.done}/{self.started}'
1495
+
1285
1496
 
1286
1497
  class CommandParallelWorker(threading.Thread):
1287
- def __init__(self, n, work_queue, done_queue):
1498
+ '''Helper class for a single threaded job, often run by a CommandParallel command
1499
+ handler class'''
1500
+
1501
+ def __init__(
1502
+ self, n, work_queue: queue.Queue, done_queue: queue.Queue,
1503
+ threads_stats: ThreadStats
1504
+ ):
1288
1505
  threading.Thread.__init__(self)
1289
1506
  self.n = n
1290
1507
  self.work_queue = work_queue
@@ -1294,54 +1511,46 @@ class CommandParallelWorker(threading.Thread):
1294
1511
  self.proc = None
1295
1512
  self.pid = None
1296
1513
  self.last_timer_debug = 0
1514
+ self.threads_stats = threads_stats # ref to shared object
1297
1515
  util.debug(f"WORKER_{n}: START")
1298
1516
 
1299
1517
  def run(self):
1300
- global _threads_start
1301
- global _threads_done
1518
+ '''Runs this single job via threading. This is typically created and called by
1519
+
1520
+ a CommandParallel class handler
1521
+ '''
1522
+
1302
1523
  while True:
1303
1524
  # Get the work from the queue and expand the tuple
1304
- i, command_list, job_name, work_dir = self.work_queue.get()
1525
+ i, command_list, job_name, _ = self.work_queue.get()
1305
1526
  self.job_name = job_name
1306
1527
  try:
1307
1528
  util.debug(f"WORKER_{self.n}: Running job {i}: {job_name}")
1308
- PIPE=subprocess.PIPE
1309
- STDOUT=subprocess.STDOUT
1310
1529
  util.debug(f"WORKER_{self.n}: Calling Popen")
1311
- proc = subprocess.Popen(command_list, stdout=PIPE, stderr=STDOUT)
1530
+ proc = subprocess.Popen( # pylint: disable=consider-using-with
1531
+ command_list, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
1532
+ )
1312
1533
  self.proc = proc
1313
1534
  util.debug(f"WORKER_{self.n}: Opened process, PID={proc.pid}")
1314
1535
  self.pid = proc.pid
1315
- _threads_start += 1
1316
- while proc.returncode == None:
1536
+ self.threads_stats.started += 1
1537
+ while proc.returncode is None:
1317
1538
  try:
1318
1539
  if (time.time() - self.last_timer_debug) > 10:
1319
1540
  util.debug(f"WORKER_{self.n}: Calling proc.communicate")
1320
1541
  stdout, stderr = proc.communicate(timeout=0.5)
1321
- util.debug(f"WORKER_{self.n}: got: \n*** stdout:\n{stdout}\n*** stderr:{stderr}")
1542
+ util.debug(f"WORKER_{self.n}: got: \n*** stdout:\n{stdout}\n***",
1543
+ f"stderr:{stderr}")
1322
1544
  except subprocess.TimeoutExpired:
1323
1545
  if (time.time() - self.last_timer_debug) > 10:
1324
- util.debug(f"WORKER_{self.n}: Timer expired, stop_request={self.stop_request}")
1546
+ util.debug(f"WORKER_{self.n}: Timer expired, stop_request=",
1547
+ f"{self.stop_request}")
1325
1548
  self.last_timer_debug = time.time()
1326
- pass
1327
1549
  if self.stop_request:
1328
1550
  util.debug(f"WORKER_{self.n}: got stop request, issuing SIGINT")
1329
1551
  proc.send_signal(signal.SIGINT)
1330
1552
  util.debug(f"WORKER_{self.n}: got stop request, calling proc.wait")
1331
1553
  proc.wait()
1332
- if False and self.stop_request:
1333
- util.debug(f"WORKER_{self.n}: got stop request, issuing proc.terminate")
1334
- proc.terminate()
1335
- util.debug(f"WORKER_{self.n}: proc poll returns is now {proc.poll()}")
1336
- try:
1337
- util.debug(f"WORKER_{self.n}: Calling proc.communicate")
1338
- stdout, stderr = proc.communicate(timeout=0.2) # for completeness, in case we ever pipe/search stdout/stderr
1339
- util.debug(f"WORKER_{self.n}: got: \n*** stdout:\n{stdout}\n*** stderr:{stderr}")
1340
- except subprocess.TimeoutExpired:
1341
- util.debug(f"WORKER_{self.n}: timeout waiting for comminicate after terminate")
1342
- except:
1343
- pass
1344
- util.debug(f"WORKER_{self.n}: proc poll returns is now {proc.poll()}")
1345
1554
 
1346
1555
  util.debug(f"WORKER_{self.n}: -- out of while loop")
1347
1556
  self.pid = None
@@ -1350,99 +1559,113 @@ class CommandParallelWorker(threading.Thread):
1350
1559
  util.debug(f"WORKER_{self.n}: proc poll returns is now {proc.poll()}")
1351
1560
  try:
1352
1561
  util.debug(f"WORKER_{self.n}: Calling proc.communicate one last time")
1353
- stdout, stderr = proc.communicate(timeout=0.1) # for completeness, in case we ever pipe/search stdout/stderr
1354
- util.debug(f"WORKER_{self.n}: got: \n*** stdout:\n{stdout}\n*** stderr:{stderr}")
1562
+ stdout, stderr = proc.communicate(timeout=0.1)
1563
+ util.debug(f"WORKER_{self.n}: got: \n*** stdout:\n{stdout}\n***",
1564
+ f"stderr:{stderr}")
1355
1565
  except subprocess.TimeoutExpired:
1356
1566
  util.debug(f"WORKER_{self.n}: timeout waiting for communicate after loop?")
1357
- except:
1358
- pass
1567
+ except Exception as e:
1568
+ util.debug(f"WORKER_{self.n}: timeout with exception {e=}")
1569
+
1359
1570
  return_code = proc.poll()
1360
- util.debug(f"WORKER_{self.n}: Finished job {i}: {job_name} with return code {return_code}")
1571
+ util.debug(f"WORKER_{self.n}: Finished job {i}: {job_name} with return code",
1572
+ f"{return_code}")
1361
1573
  self.done_queue.put((i, job_name, return_code))
1362
1574
  finally:
1363
1575
  util.debug(f"WORKER_{self.n}: -- in finally block")
1364
1576
  self.work_queue.task_done()
1365
- _threads_done += 1
1577
+ self.threads_stats.done += 1
1366
1578
 
1367
1579
 
1368
1580
  class CommandParallel(Command):
1581
+ '''Base class command handler for running multiple eda jobs, such as for child classes
1582
+
1583
+ in commands.multi or commands.sweep
1584
+ '''
1369
1585
  def __init__(self, config: dict):
1370
1586
  Command.__init__(self, config=config)
1371
1587
  self.jobs = []
1372
1588
  self.jobs_status = []
1373
1589
  self.args['parallel'] = 1
1374
1590
  self.worker_threads = []
1591
+ self.threads_stats = ThreadStats()
1375
1592
 
1376
- def __del__(self):
1377
- util.debug(f"In Command.__del__, threads done/started: {_threads_done}/{_threads_start}")
1378
- if _threads_start == _threads_done:
1593
+ def __del__(self): # pylint: disable=too-many-branches
1594
+ util.debug(f"In Command.__del__, threads done/started: {self.threads_stats}")
1595
+ if self.threads_stats.all_done():
1379
1596
  return
1380
- util.warning(f"Need to shut down {_threads_start-_threads_done} worker threads...")
1597
+ util.warning(f"Need to shut down {self.threads_stats.get_remaining()} worker threads...")
1381
1598
  for w in self.worker_threads:
1382
1599
  if w.proc:
1383
1600
  util.warning(f"Requesting stop of PID {w.pid}: {w.job_name}")
1384
1601
  w.stop_request = True
1385
- for i in range(10):
1386
- util.debug(f"Threads done/started: {_threads_done}/{_threads_start}")
1387
- if _threads_start == _threads_done:
1388
- util.info(f"All threads done")
1602
+ for _ in range(10):
1603
+ util.debug(f"Threads done/started: {self.threads_stats}")
1604
+ if self.threads_stats.started == self.threads_stats.done:
1605
+ util.info("All threads done")
1389
1606
  return
1390
1607
  time.sleep(1)
1391
- subprocess.Popen(['stty', 'sane']).wait()
1392
- util.debug(f"Scanning workers again")
1608
+ subprocess.Popen(['stty', 'sane']).wait() # pylint: disable=consider-using-with
1609
+ util.debug("Scanning workers again")
1393
1610
  for w in self.worker_threads:
1394
1611
  if w.proc:
1395
1612
  util.info(f"need to SIGINT WORKER_{w.n}, may need manual cleanup, check 'ps'")
1396
1613
  if w.pid:
1397
1614
  os.kill(w.pid, signal.SIGINT)
1398
- for i in range(5):
1399
- util.debug(f"Threads done/started: {_threads_done}/{_threads_start}")
1400
- if _threads_start == _threads_done:
1401
- util.info(f"All threads done")
1615
+ for _ in range(5):
1616
+ util.debug(f"Threads done/started: {self.threads_stats}")
1617
+ if self.threads_stats.all_done():
1618
+ util.info("All threads done")
1402
1619
  return
1403
1620
  time.sleep(1)
1404
- subprocess.Popen(['stty', 'sane']).wait()
1405
- util.debug(f"Scanning workers again")
1621
+ subprocess.Popen(['stty', 'sane']).wait() # pylint: disable=consider-using-with
1622
+ util.debug("Scanning workers again")
1406
1623
  for w in self.worker_threads:
1407
1624
  if w.proc:
1408
1625
  util.info(f"need to TERM WORKER_{w.n}, probably needs manual cleanup, check 'ps'")
1409
1626
  if w.pid:
1410
1627
  os.kill(w.pid, signal.SIGTERM)
1411
- for i in range(5):
1412
- util.debug(f"Threads done/started: {_threads_done}/{_threads_start}")
1413
- if _threads_start == _threads_done:
1414
- util.info(f"All threads done")
1628
+ for _ in range(5):
1629
+ util.debug(f"Threads done/started: {self.threads_stats}")
1630
+ if self.threads_stats.all_done():
1631
+ util.info("All threads done")
1415
1632
  return
1416
1633
  time.sleep(1)
1417
- subprocess.Popen(['stty', 'sane']).wait()
1418
- util.debug(f"Scanning workers again")
1634
+ subprocess.Popen(['stty', 'sane']).wait() # pylint: disable=consider-using-with
1635
+ util.debug("Scanning workers again")
1419
1636
  for w in self.worker_threads:
1420
1637
  if w.proc:
1421
1638
  util.info(f"need to KILL WORKER_{w.n}, probably needs manual cleanup, check 'ps'")
1422
1639
  if w.pid:
1423
1640
  os.kill(w.pid, signal.SIGKILL)
1424
1641
  util.stop_log()
1425
- subprocess.Popen(['stty', 'sane']).wait()
1642
+ subprocess.Popen(['stty', 'sane']).wait() # pylint: disable=consider-using-with
1643
+
1644
+ def run_jobs( # pylint: disable=too-many-locals,too-many-branches,too-many-statements
1645
+ self, command: str
1646
+ ) -> None:
1647
+ '''Runs all "jobs" in self.jobs list, either serially or in parallel'''
1426
1648
 
1427
- def run_jobs(self, command):
1428
- # this is where we actually run the jobs. it's a messy piece of code and prob could use refactoring
1429
- # but the goal was to share as much as possible (job start, end, pass/fail judgement, etc) while
1430
- # supporting various mode combinations (parallel mode, verbose mode, fancy mode, etc) and keeping the
1431
- # UI output functional and awesome sauce
1649
+ # this is where we actually run the jobs. it's a messy piece of code and prob could use
1650
+ # refactoring but the goal was to share as much as possible (job start, end, pass/fail
1651
+ # judgement, etc) while supporting various mode combinations (parallel mode, verbose mode,
1652
+ # fancy mode, etc) and keeping the UI output functional and awesome sauce
1432
1653
 
1433
1654
  # walk targets to find the longest name, for display reasons
1434
1655
  longest_job_name = 0
1435
1656
  total_jobs = len(self.jobs)
1436
1657
  self.jobs_status = [None] * total_jobs
1437
1658
  for i in range(total_jobs):
1438
- l = len(self.jobs[i]['name'])
1439
- if l>longest_job_name: longest_job_name = l
1659
+ longest_job_name = max(longest_job_name, len(self.jobs[i]['name']))
1440
1660
 
1441
1661
  run_parallel = self.args['parallel'] > 1
1442
1662
 
1443
1663
  # figure out the width to print various numbers
1444
1664
  jobs_digits = len(f"{total_jobs}")
1445
- jobs_fmt = "%%%dd" % jobs_digits # ugh, for printing out a number with N digits
1665
+
1666
+ job_done_return_code = None
1667
+ job_done_run_time = 0
1668
+ job_done_name = ''
1446
1669
 
1447
1670
  # run the jobs!
1448
1671
  running_jobs = {}
@@ -1453,9 +1676,10 @@ class CommandParallel(Command):
1453
1676
  jobs_launched = 0
1454
1677
  num_parallel = min(len(self.jobs), self.args['parallel'])
1455
1678
  # 16 should really be the size of window or ?
1456
- (columns,lines) = shutil.get_terminal_size()
1457
- # we will enter fancy mode if we are parallel and we can leave 6 lines of regular scrolling output
1458
- fancy_mode = util.args['fancy'] and (num_parallel > 1) and (num_parallel <= (lines-6))
1679
+ _, lines = shutil.get_terminal_size()
1680
+ # we will enter fancy mode if we are parallel and we can leave 6 lines of regular scrolling
1681
+ # output
1682
+ fancy_mode = all([util.args['fancy'], num_parallel > 1, num_parallel <= (lines-6)])
1459
1683
  multi_cwd = util.getcwd() + os.sep
1460
1684
 
1461
1685
  self.patch_jobs_for_duplicate_target_names()
@@ -1466,8 +1690,12 @@ class CommandParallel(Command):
1466
1690
  work_queue = queue.Queue()
1467
1691
  done_queue = queue.Queue()
1468
1692
  for x in range(num_parallel):
1469
- worker = CommandParallelWorker(x, work_queue, done_queue)
1470
- # Setting daemon to True will let the main thread exit even though the workers are blocking
1693
+ worker = CommandParallelWorker(
1694
+ n=x, work_queue=work_queue, done_queue=done_queue,
1695
+ threads_stats=self.threads_stats
1696
+ )
1697
+ # Setting daemon to True will let the main thread exit even though the workers are
1698
+ # blocking
1471
1699
  worker.daemon = True
1472
1700
  worker.start()
1473
1701
  self.worker_threads.append(worker)
@@ -1478,22 +1706,30 @@ class CommandParallel(Command):
1478
1706
  for x in range(num_parallel):
1479
1707
  util.fancy_print(f"Starting worker {x}", x)
1480
1708
 
1481
- while len(self.jobs) or len(running_jobs.items()):
1709
+ while self.jobs or running_jobs:
1482
1710
  job_done = False
1483
1711
  job_done_quiet = False
1484
1712
  anything_done = False
1485
1713
 
1486
1714
  def sprint_job_line(job_number=0, job_name="", final=False, hide_stats=False):
1487
- return (f"INFO: [EDA] " +
1488
- util.string_or_space(f"[job {jobs_fmt%job_number}/{jobs_fmt%total_jobs} ", final) +
1489
- util.string_or_space(f"| pass ", hide_stats or final) +
1490
- util.string_or_space(f"{jobs_fmt%len(passed_jobs)}/{jobs_fmt%jobs_complete} ", hide_stats) +
1491
- util.string_or_space(f"@ {(100*(jobs_complete))/total_jobs:5.1f}%", hide_stats or final) +
1492
- util.string_or_space(f"] ", final) +
1493
- f"{command} {(job_name+' ').ljust(longest_job_name+3,'.')}")
1494
-
1495
- # for any kind of run (parallel or not, fancy or not, verbose or not) ... can we launch a job?
1496
- if len(self.jobs) and (len(running_jobs.items()) < num_parallel):
1715
+ return (
1716
+ "INFO: [EDA] " +
1717
+ string_or_space(
1718
+ f"[job {job_number:0{jobs_digits}d}/{total_jobs:0{jobs_digits}d} ",
1719
+ final) +
1720
+ string_or_space("| pass ", hide_stats or final) +
1721
+ string_or_space(
1722
+ f"{len(passed_jobs):0{jobs_digits}d}/{jobs_complete:0{jobs_digits}d} ",
1723
+ hide_stats) +
1724
+ string_or_space(f"@ {(100 * (jobs_complete)) / total_jobs : 5.1f}%",
1725
+ hide_stats or final) +
1726
+ string_or_space("] ", final) +
1727
+ f"{command} {(job_name+' ').ljust(longest_job_name+3,'.')}"
1728
+ )
1729
+
1730
+ # for any kind of run (parallel or not, fancy or not, verbose or not) ...
1731
+ # can we launch a job?
1732
+ if self.jobs and (len(running_jobs) < num_parallel):
1497
1733
  # we are launching a job
1498
1734
  jobs_launched += 1
1499
1735
  anything_done = True
@@ -1501,15 +1737,17 @@ class CommandParallel(Command):
1501
1737
  # TODO(drew): it might be nice to pass more items on 'job' dict, like
1502
1738
  # logfile or job-name, so CommandSweep or CommandMulti don't have to set
1503
1739
  # via args. on their command_list.
1504
- if job['name'].startswith(multi_cwd): job['name'] = job['name'][len(multi_cwd):]
1505
- # in all but fancy mode, we will print this text at the launch of a job. It may get a newline below
1740
+ if job['name'].startswith(multi_cwd):
1741
+ job['name'] = job['name'][len(multi_cwd):]
1742
+ # in all but fancy mode, we will print this text at the launch of a job. It may
1743
+ # get a newline below
1506
1744
  job_text = sprint_job_line(jobs_launched, job['name'], hide_stats=run_parallel)
1507
1745
  command_list = job['command_list']
1508
1746
  cwd = util.getcwd()
1509
1747
 
1510
1748
  if run_parallel:
1511
1749
  # multithreaded job launch: add to queue
1512
- worker = workers.pop(0) # we don't actually know which thread will pick up, but GUI will be consistent
1750
+ worker = workers.pop(0)
1513
1751
  running_jobs[str(jobs_launched)] = { 'name' : job['name'],
1514
1752
  'number' : jobs_launched,
1515
1753
  'worker' : worker,
@@ -1519,15 +1757,20 @@ class CommandParallel(Command):
1519
1757
  suffix = "<START>"
1520
1758
  if fancy_mode:
1521
1759
  util.fancy_print(job_text+suffix, worker)
1760
+ elif failed_jobs:
1761
+ # if we aren't in fancy mode, we will print a START line, periodic RUNNING
1762
+ # lines, and PASS/FAIL line per-job
1763
+ util.print_orange(job_text + Colors.yellow + suffix)
1522
1764
  else:
1523
- # if we aren't in fancy mode, we will print a START line, periodic RUNNING lines, and PASS/FAIL line per-job
1524
- if len(failed_jobs): util.print_orange(job_text + util.string_yellow + suffix)
1525
- else: util.print_yellow(job_text + util.string_yellow + suffix)
1765
+ util.print_yellow(job_text + Colors.yellow + suffix)
1526
1766
  else:
1527
- # single-threaded job launch, we are going to print out job info as we start each job... no newline
1528
- # since non-verbose silences the job and prints only <PASS>/<FAIL> after the trailing "..." we leave here
1529
- if len(failed_jobs): util.print_orange(job_text, end="")
1530
- else: util.print_yellow(job_text, end="")
1767
+ # single-threaded job launch, we are going to print out job info as we start
1768
+ # each job... no newline. since non-verbose silences the job and prints only
1769
+ # <PASS>/<FAIL> after the trailing "..." we leave here
1770
+ if failed_jobs:
1771
+ util.print_orange(job_text, end="")
1772
+ else:
1773
+ util.print_yellow(job_text, end="")
1531
1774
  job_done_number = jobs_launched
1532
1775
  job_done_name = job['name']
1533
1776
  job_start_time = time.time()
@@ -1539,13 +1782,16 @@ class CommandParallel(Command):
1539
1782
  _, _, job_done_return_code = self.exec(
1540
1783
  cwd, command_list, background=False, stop_on_error=False, quiet=False
1541
1784
  )
1542
- # reprint the job text previously printed before running job(and given "\n" after the trailing "...")
1785
+ # reprint the job text previously printed before running job (and given
1786
+ # "\n" after the trailing "...")
1543
1787
  else:
1544
1788
  # run job, swallowing output (hope you have a logfile)
1545
1789
  _, _, job_done_return_code = self.exec(
1546
1790
  cwd, command_list, background=True, stop_on_error=False, quiet=True
1547
1791
  )
1548
- job_done_quiet = True # in this case, we have the job start text (trailing "...", no newline) printed
1792
+ # in this case, we have the job start text (trailing "...", no newline)
1793
+ # printed
1794
+ job_done_quiet = True
1549
1795
  job_done = True
1550
1796
  job_done_run_time = time.time() - job_start_time
1551
1797
  # Since we consumed the job, use the job['index'] to track the per-job status:
@@ -1553,15 +1799,16 @@ class CommandParallel(Command):
1553
1799
  if run_parallel:
1554
1800
  # parallel run, check for completed job
1555
1801
  if done_queue.qsize():
1556
- # we're collecting a finished job from a worker thread. note we will only reap one job per iter of the big
1557
- # loop, so as to share job completion code at the bottom
1802
+ # we're collecting a finished job from a worker thread. note we will only
1803
+ # reap one job per iter of the big loop, so as to share job completion code
1804
+ # at the bottom
1558
1805
  anything_done = True
1559
1806
  job_done = True
1560
1807
  job_done_number, job_done_name, job_done_return_code = done_queue.get()
1561
1808
  t = running_jobs[str(job_done_number)]
1562
1809
  # in fancy mode, we need to clear the worker line related to this job.
1563
1810
  if fancy_mode:
1564
- util.fancy_print(f"INFO: [EDA] Parallel: Worker Idle ...", t['worker'])
1811
+ util.fancy_print("INFO: [EDA] Parallel: Worker Idle ...", t['worker'])
1565
1812
  job_done_run_time = time.time() - t['start_time']
1566
1813
  util.debug(f"removing job #{job_done_number} from running jobs")
1567
1814
  del running_jobs[str(job_done_number)]
@@ -1573,33 +1820,40 @@ class CommandParallel(Command):
1573
1820
  if (fancy_mode or (time.time() - t['update_time']) > 30):
1574
1821
  t['update_time'] = time.time()
1575
1822
  job_text = sprint_job_line(t['number'], t['name'], hide_stats=True)
1576
- suffix = f"<RUNNING: {util.sprint_time(time.time() - t['start_time'])}>"
1823
+ suffix = f"<RUNNING: {sprint_time(time.time() - t['start_time'])}>"
1577
1824
  if fancy_mode:
1578
1825
  util.fancy_print(f"{job_text}{suffix}", t['worker'])
1826
+ elif failed_jobs:
1827
+ util.print_orange(job_text + Colors.yellow + suffix)
1579
1828
  else:
1580
- if len(failed_jobs): util.print_orange(job_text+util.string_yellow+suffix)
1581
- else: util.print_yellow(job_text+util.string_yellow+suffix)
1829
+ util.print_yellow(job_text + Colors.yellow + suffix)
1582
1830
 
1583
1831
  # shared job completion code
1584
- # single or multi-threaded, we can arrive here to harvest <= 1 jobs, and need {job, return_code} valid, and
1585
- # we expect the start of a status line to have been printed, ready for pass/fail
1832
+ # single or multi-threaded, we can arrive here to harvest <= 1 jobs, and need
1833
+ # {job, return_code} valid, and we expect the start of a status line to have been
1834
+ # printed, ready for pass/fail
1586
1835
  if job_done:
1587
1836
  jobs_complete += 1
1588
1837
  if job_done_return_code is None or job_done_return_code:
1589
- # embed the color code, to change color of pass/fail during the util.print_orange/yellow below
1838
+ # embed the color code, to change color of pass/fail during the
1839
+ # util.print_orange/yellow below
1590
1840
  if job_done_return_code == 124:
1591
- # bash uses 124 for bash timeout errors, if that was preprended to the command list.
1592
- suffix = f"{util.string_red}<TOUT: {util.sprint_time(job_done_run_time)}>"
1841
+ # bash uses 124 for bash timeout errors, if that was preprended to the
1842
+ # command list.
1843
+ suffix = f"{Colors.red}<TOUT: {sprint_time(job_done_run_time)}>"
1593
1844
  else:
1594
- suffix = f"{util.string_red}<FAIL: {util.sprint_time(job_done_run_time)}>"
1845
+ suffix = f"{Colors.red}<FAIL: {sprint_time(job_done_run_time)}>"
1595
1846
  failed_jobs.append(job_done_name)
1596
1847
  else:
1597
- suffix = f"{util.string_green}<PASS: {util.sprint_time(job_done_run_time)}>"
1848
+ suffix = f"{Colors.green}<PASS: {sprint_time(job_done_run_time)}>"
1598
1849
  passed_jobs.append(job_done_name)
1599
1850
  # we want to print in one shot, because in fancy modes that's all that we're allowed
1600
- job_done_text = "" if job_done_quiet else sprint_job_line(job_done_number, job_done_name)
1601
- if len(failed_jobs): util.print_orange(f"{job_done_text}{suffix}")
1602
- else: util.print_yellow(f"{job_done_text}{suffix}")
1851
+ job_done_text = "" if job_done_quiet else sprint_job_line(job_done_number,
1852
+ job_done_name)
1853
+ if failed_jobs:
1854
+ util.print_orange(f"{job_done_text}{suffix}")
1855
+ else:
1856
+ util.print_yellow(f"{job_done_text}{suffix}")
1603
1857
  self.jobs_status[job_done_number-1] = job_done_return_code
1604
1858
 
1605
1859
  if not anything_done:
@@ -1607,23 +1861,26 @@ class CommandParallel(Command):
1607
1861
 
1608
1862
  if total_jobs:
1609
1863
  emoji = "< :) >" if (len(passed_jobs) == total_jobs) else "< :( >"
1610
- util.info(sprint_job_line(final=True,job_name="jobs passed")+emoji, start="")
1864
+ util.info(sprint_job_line(final=True, job_name="jobs passed") + emoji, start="")
1611
1865
  else:
1612
- util.info(f"Parallel: <No jobs found>")
1866
+ util.info("Parallel: <No jobs found>")
1613
1867
  # Make sure all jobs have a set status:
1614
- for i,rc in enumerate(self.jobs_status):
1615
- if rc is None or type(rc) != int:
1868
+ for i, rc in enumerate(self.jobs_status):
1869
+ if rc is None or not isinstance(rc, int):
1616
1870
  self.error(f'job {i=} {rc=} did not return a proper return code')
1617
- jobs_status[i] = 1
1871
+ self.jobs_status[i] = 2
1618
1872
 
1619
1873
  # if self.status > 0, then keep it non-zero, else set it if we still have running jobs.
1620
1874
  if self.status == 0:
1621
- self.status = 0 if len(self.jobs_status) == 0 else max(self.jobs_status)
1875
+ if self.jobs_status:
1876
+ self.status = max(self.jobs_status)
1877
+ # else keep at 0, empty list.
1622
1878
  util.fancy_stop()
1623
1879
 
1624
1880
  @staticmethod
1625
1881
  def get_name_from_target(target: str) -> str:
1626
- return target.replace('../', '').lstrip('./')
1882
+ '''Given a target path, strip leftmost path separators to get a shorter string name'''
1883
+ return target.replace('../', '').lstrip('./').lstrip(os.path.sep)
1627
1884
 
1628
1885
 
1629
1886
  def update_args_list(self, args: list, tool: str) -> None:
@@ -1667,7 +1924,7 @@ class CommandParallel(Command):
1667
1924
  tokens=tokens.copy(),
1668
1925
  apply_parsed_args=False,
1669
1926
  )
1670
- util.debug(f'{self.command_name}: {single_cmd_unparsed=}')
1927
+ util.debug(f'{self.command_name}: {single_cmd_parsed=}, {single_cmd_unparsed=}')
1671
1928
 
1672
1929
  # There should not be any single_cmd_unparsed args starting with '-'
1673
1930
  bad_remaining_args = [x for x in single_cmd_unparsed if x.startswith('-')]
@@ -1694,7 +1951,7 @@ class CommandParallel(Command):
1694
1951
  if item.startswith(f'--{arg_name}='):
1695
1952
  _, name = item.split(f'--{arg_name}=')
1696
1953
  return name
1697
- elif item == f'--{arg_name}':
1954
+ if item == f'--{arg_name}':
1698
1955
  return job_dict['command_list'][i + 1]
1699
1956
  return ''
1700
1957
 
@@ -1704,7 +1961,7 @@ class CommandParallel(Command):
1704
1961
  if item.startswith(f'--{arg_name}='):
1705
1962
  job_dict['command_list'][i] = f'--{arg_name}=' + new_value
1706
1963
  return True
1707
- elif item == f'--{arg_name}':
1964
+ if item == f'--{arg_name}':
1708
1965
  job_dict['command_list'][i + 1] = new_value
1709
1966
  return True
1710
1967
  return False
@@ -1729,7 +1986,6 @@ class CommandParallel(Command):
1729
1986
  tpath, _ = os.path.split(job_dict['target'])
1730
1987
 
1731
1988
  # prepend path information to job-name:
1732
- patched_sub_work_dir = False
1733
1989
  patched_target_path = os.path.relpath(tpath).replace(os.sep, '_')
1734
1990
  new_job_name = f'{patched_target_path}.{key}'
1735
1991
  replace_job_arg(job_dict, arg_name='job-name', new_value=new_job_name)