opencos-eda 0.2.47__py3-none-any.whl → 0.2.49__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. opencos/__init__.py +4 -2
  2. opencos/_version.py +10 -7
  3. opencos/commands/flist.py +8 -7
  4. opencos/commands/multi.py +35 -18
  5. opencos/commands/sweep.py +9 -4
  6. opencos/commands/waves.py +1 -1
  7. opencos/deps/__init__.py +0 -0
  8. opencos/deps/defaults.py +69 -0
  9. opencos/deps/deps_commands.py +419 -0
  10. opencos/deps/deps_file.py +326 -0
  11. opencos/deps/deps_processor.py +670 -0
  12. opencos/deps_schema.py +7 -8
  13. opencos/eda.py +92 -67
  14. opencos/eda_base.py +625 -332
  15. opencos/eda_config.py +80 -14
  16. opencos/eda_extract_targets.py +22 -14
  17. opencos/eda_tool_helper.py +33 -7
  18. opencos/export_helper.py +166 -86
  19. opencos/export_json_convert.py +31 -23
  20. opencos/files.py +2 -1
  21. opencos/hw/__init__.py +0 -0
  22. opencos/{oc_cli.py → hw/oc_cli.py} +9 -4
  23. opencos/names.py +0 -4
  24. opencos/peakrdl_cleanup.py +13 -7
  25. opencos/seed.py +19 -11
  26. opencos/tests/helpers.py +27 -14
  27. opencos/tests/test_deps_helpers.py +35 -32
  28. opencos/tests/test_eda.py +47 -41
  29. opencos/tests/test_eda_elab.py +5 -3
  30. opencos/tests/test_eda_synth.py +1 -1
  31. opencos/tests/test_oc_cli.py +1 -1
  32. opencos/tests/test_tools.py +3 -2
  33. opencos/tools/iverilog.py +2 -2
  34. opencos/tools/modelsim_ase.py +2 -2
  35. opencos/tools/riviera.py +1 -1
  36. opencos/tools/slang.py +1 -1
  37. opencos/tools/surelog.py +1 -1
  38. opencos/tools/verilator.py +1 -1
  39. opencos/tools/vivado.py +1 -1
  40. opencos/tools/yosys.py +4 -3
  41. opencos/util.py +440 -483
  42. opencos/utils/__init__.py +0 -0
  43. opencos/utils/markup_helpers.py +98 -0
  44. opencos/utils/str_helpers.py +111 -0
  45. opencos/utils/subprocess_helpers.py +108 -0
  46. {opencos_eda-0.2.47.dist-info → opencos_eda-0.2.49.dist-info}/METADATA +1 -1
  47. opencos_eda-0.2.49.dist-info/RECORD +88 -0
  48. {opencos_eda-0.2.47.dist-info → opencos_eda-0.2.49.dist-info}/entry_points.txt +1 -1
  49. opencos/deps_helpers.py +0 -1346
  50. opencos_eda-0.2.47.dist-info/RECORD +0 -79
  51. /opencos/{pcie.py → hw/pcie.py} +0 -0
  52. {opencos_eda-0.2.47.dist-info → opencos_eda-0.2.49.dist-info}/WHEEL +0 -0
  53. {opencos_eda-0.2.47.dist-info → opencos_eda-0.2.49.dist-info}/licenses/LICENSE +0 -0
  54. {opencos_eda-0.2.47.dist-info → opencos_eda-0.2.49.dist-info}/licenses/LICENSE.spdx +0 -0
  55. {opencos_eda-0.2.47.dist-info → opencos_eda-0.2.49.dist-info}/top_level.txt +0 -0
opencos/eda_base.py CHANGED
@@ -1,22 +1,36 @@
1
+ '''opencos.eda_base holds base classes for opencos.tools and opencos.commands
2
+
3
+ Note that opencos.Command.process_tokens (on a derived class) is the main entrypoint
4
+ from eda.py, it will find a command handler class for a command + tool, and call
5
+ its obj.process_tokens(args)
6
+ '''
7
+
8
+ # TODO(drew): Clean up the following pylint
9
+ # pylint: disable=too-many-lines
10
+
1
11
  import argparse
2
12
  import copy
3
- import glob
4
13
  import subprocess
5
14
  import os
6
15
  import queue
7
16
  import re
8
17
  import shutil
9
- import shlex
10
18
  import signal
11
19
  import sys
12
20
  import threading
13
21
  import time
14
22
  from pathlib import Path
15
23
 
16
- import opencos
17
- from opencos import seed, deps_helpers, util, files
24
+ from opencos import seed, util, files
18
25
  from opencos import eda_config
19
- from opencos.deps_helpers import get_deps_markup_file, deps_markup_safe_load
26
+
27
+ from opencos.util import Colors
28
+ from opencos.utils.str_helpers import sprint_time, strip_outer_quotes, string_or_space, \
29
+ indent_wrap_long_text
30
+ from opencos.utils.subprocess_helpers import subprocess_run_background
31
+
32
+ from opencos.deps.deps_file import DepsFile, deps_data_get_all_targets
33
+ from opencos.deps.deps_processor import DepsProcessor
20
34
 
21
35
 
22
36
  def print_base_help() -> None:
@@ -28,9 +42,12 @@ def print_base_help() -> None:
28
42
 
29
43
 
30
44
  def get_argparser() -> argparse.ArgumentParser:
45
+ '''Returns the ArgumentParser for general eda CLI'''
31
46
  parser = argparse.ArgumentParser(prog='eda options', add_help=False, allow_abbrev=False)
32
47
  parser.add_argument('-q', '--quit', action='store_true',
33
- help='For interactive mode (eda called with no options, command, or targets)')
48
+ help=(
49
+ 'For interactive mode (eda called with no options, command, or'
50
+ ' targets)'))
34
51
  parser.add_argument('--exit', action='store_true', help='same as --quit')
35
52
  parser.add_argument('-h', '--help', action='store_true')
36
53
 
@@ -40,22 +57,25 @@ def get_argparser() -> argparse.ArgumentParser:
40
57
  help='Tool to use for this command, such as: modelsim_ase, verilator,' \
41
58
  + ' modelsim_ase=/path/to/bin/vsim, verilator=/path/to/bin/verilator')
42
59
  parser.add_argument('--eda-safe', action='store_true',
43
- help='disable all DEPS file deps shell commands, overrides values from --config-yml')
60
+ help=('disable all DEPS file deps shell commands, overrides values from'
61
+ ' --config-yml'))
44
62
  return parser
45
63
 
46
64
 
47
65
  def get_argparser_short_help() -> str:
66
+ '''Returns str for the eda_base argparser (--help, --tool, etc)'''
48
67
  return util.get_argparser_short_help(parser=get_argparser())
49
68
 
50
69
 
51
- def get_eda_exec(command:str=''):
70
+ def get_eda_exec(command: str = '') -> str:
71
+ '''Returns the full path of `eda` executable to be used for a given eda <command>'''
52
72
  # NOTE(drew): This is kind of flaky. 'eda multi' reinvokes 'eda'. But the executable for 'eda'
53
73
  # is one of:
54
74
  # 1. pip3 install opencos-eda
55
75
  # -- script 'eda', installed from PyPi
56
76
  # 2. pip3 uninstall .; python3 -m build; pip3 install
57
77
  # -- script 'eda' but installed from local.
58
- # 2. (opencos repo)/bin/eda - a python wrapper to link to (opencos repo)/opencos/eda.py (package)
78
+ # 2. (opencos repo)/bin/eda - a python wrapper to link to (opencos repo)/opencos/eda.py, but
59
79
  # packages cannot be run standalone, they need to be called as: python3 -m opencos.eda,
60
80
  # and do not work with relative paths. This only works if env OC_ROOT is set or can be found.
61
81
  # 3. If you ran 'source bin/addpath' then you are always using the local (opencos repo)/bin/eda
@@ -64,11 +84,13 @@ def get_eda_exec(command:str=''):
64
84
  # Can we run from OC_ROOT/bin/eda?
65
85
  oc_root = util.get_oc_root()
66
86
  if not oc_root:
67
- util.error(f"Need 'eda' in our path to run 'eda {command}', could not find env OC_ROOT, {eda_path=}, {oc_root=}")
87
+ util.error(f"Need 'eda' in our path to run 'eda {command}', could not find env",
88
+ f"OC_ROOT, {eda_path=}, {oc_root=}")
68
89
  else:
69
90
  bin_eda = os.path.join(oc_root, 'bin', 'eda')
70
91
  if not os.path.exists(bin_eda):
71
- util.error(f"Need 'eda' in our path to run 'eda {command}', cound not find bin/, {eda_path=}, {oc_root=}, {bin_eda=}")
92
+ util.error(f"Need 'eda' in our path to run 'eda {command}', cound not find",
93
+ f"bin/, {eda_path=}, {oc_root=}, {bin_eda=}")
72
94
  else:
73
95
  util.info(f"'eda' not in path, using {bin_eda=} for 'eda' {command} executable")
74
96
  eda_path = os.path.abspath(bin_eda)
@@ -76,9 +98,10 @@ def get_eda_exec(command:str=''):
76
98
  return eda_path
77
99
 
78
100
 
79
- def which_tool(command, config):
80
- '''Returns which tool will be used for a command, given the command_handlers in config dict.'''
81
- from opencos import eda_tool_helper
101
+ def which_tool(command: str, config: dict) -> str:
102
+ '''Returns which tool (str) will be used for a command, given the command_handlers in
103
+
104
+ config dict.'''
82
105
  if config is None:
83
106
  util.error(f'which_tool({command=}) called w/out config')
84
107
  if not command in config.get('command_handler', {}):
@@ -136,9 +159,10 @@ class Tool:
136
159
  self.set_tool_config_from_config()
137
160
 
138
161
  def set_exe(self, config: dict) -> None:
162
+ '''Sets self._EXE based on config'''
139
163
  if self._TOOL and self._TOOL in config.get('auto_tools_order', [{}])[0]:
140
164
  exe = config.get('auto_tools_order', [{}])[0][self._TOOL].get('exe', '')
141
- if exe and type(exe) is list:
165
+ if exe and isinstance(exe, list):
142
166
  exe = exe[0] # pick first
143
167
  if exe and exe != self._EXE:
144
168
  util.info(f'Override for {self._TOOL} using exe {exe}')
@@ -155,9 +179,16 @@ class Tool:
155
179
  return self._VERSION
156
180
 
157
181
  def set_tool_defines(self) -> None:
158
- pass
182
+ '''Derived classes may override, sets any additional defines based on tool.'''
183
+ return
184
+
159
185
 
160
186
  class Command:
187
+ '''Base class for all: eda <command>
188
+
189
+ The Command class should be used when you don't require files, otherwise consider
190
+ CommandDesign.
191
+ '''
161
192
 
162
193
  command_name: str = ''
163
194
 
@@ -170,20 +201,20 @@ class Command:
170
201
  "keep" : False,
171
202
  "force" : False,
172
203
  "fake" : False,
173
- "stop-before-compile": False, # Usually in the self.do_it() method, stop prior to compile/elaborate/simulate
204
+ "stop-before-compile": False,
174
205
  "stop-after-compile": False,
175
- "stop-after-elaborate": False, # Set to True to only run compile + elaboration (aka compile + lint)
206
+ "stop-after-elaborate": False,
176
207
  "lint": False, # Same as stop-after-elaborate
177
208
  "eda-dir" : "eda.work", # all eda jobs go in here
178
209
  "job-name" : "", # this is used to create a certain dir under "eda_dir"
179
- "work-dir" : "", # this can be used to run the job in a certain dir, else it will be <eda-dir>/<job-name> else <eda-dir>/<target>_<command>
180
- "sub-work-dir" : "", # this can be used to name the dir built under <eda-dir>, which seems to be same function as job-name??
210
+ "work-dir" : "", # default is usually <eda-dir>/<job-name> or <eda-dir>/<target>.<cmd>
211
+ "sub-work-dir" : "", # this can be used to name the dir built under <eda-dir>
181
212
  "work-dir-use-target-dir": False,
182
213
  "suffix" : "",
183
214
  "design" : "", # not sure how this relates to top
184
215
  'export': False,
185
- 'export-run': False, # run from the exported location if possible, if not possible run the command in usual place.
186
- 'export-json': False, # generates an export.json suitable for a testrunner, if possible for self.command.
216
+ 'export-run': False, # run from the exported location if possible
217
+ 'export-json': False, # generates an export.json suitable for a testrunner
187
218
  'enable-tags': [],
188
219
  'disable-tags': [],
189
220
  'test-mode': False,
@@ -212,12 +243,32 @@ class Command:
212
243
  self.target = ""
213
244
  self.target_path = ""
214
245
  self.status = 0
246
+ self.errors_log_f = None
247
+
215
248
 
216
- def error(self, *args, **kwargs):
217
- '''Returns None, child classes can call self.error(..) instead of util.error, which updates their self.status.
249
+ def error(self, *args, **kwargs) -> None:
250
+ '''Returns None, child classes can call self.error(..) instead of util.error,
218
251
 
219
- Please consider using Command.error(..) (or self.error(..)) in place of util.error so self.status is updated.
252
+ which updates their self.status. Also will write out to eda.errors.log if the work-dir
253
+ exists. Please consider using Command.error(..) (or self.error(..)) in place of util.error
254
+ so self.status is updated.
220
255
  '''
256
+
257
+ if self.args['work-dir']:
258
+ if not self.errors_log_f:
259
+ try:
260
+ self.errors_log_f = open( # pylint: disable=consider-using-with
261
+ os.path.join(self.args['work-dir'], 'eda.errors.log'), 'w',
262
+ encoding='utf-8'
263
+ )
264
+ except FileNotFoundError:
265
+ pass
266
+ if self.errors_log_f:
267
+ print(
268
+ f'ERROR: [eda] ({self.command_name}) {" ".join(list(args))}',
269
+ file=self.errors_log_f
270
+ )
271
+
221
272
  self.status = util.error(*args, **kwargs)
222
273
 
223
274
  def status_any_error(self, report=True) -> bool:
@@ -227,17 +278,29 @@ class Command:
227
278
  util.error(f"command '{self.command_name}' has previous errors")
228
279
  return self.status > 0
229
280
 
230
- def which_tool(self, command:str):
281
+ def which_tool(self, command:str) -> str:
282
+ '''Returns a str for the tool name used for the requested command'''
231
283
  return which_tool(command, config=self.config)
232
284
 
233
- def create_work_dir(self):
285
+ def create_work_dir( # pylint: disable=too-many-branches,too-many-statements
286
+ self
287
+ ) -> str:
288
+ '''Creates the working directory and populates self.args['work-dir']
289
+
290
+ Generally uses ./ self.args['eda-dir'] / <target-name>.<command> /
291
+ however, self.args['job-name'] or ['sub-work-dir'] can override that.
292
+
293
+ Additionally, the work-dir is attempted to be deleted if it already exists
294
+ AND it is beneath the caller's original working directory. We also will
295
+ not delete 'work-dir' that is ./ or $PWD.
296
+ '''
234
297
  util.debug(f"create_work_dir: {self.args['eda-dir']=} {self.args['work-dir']=}")
235
298
  if self.args['work-dir-use-target-dir']:
236
299
  if not self.target_path:
237
300
  self.target_path = '.'
238
- util.info(f"create_work_dir: --work-dir-use-target-dir: using:",
239
- f"{os.path.abspath(self.target_path)}",
240
- f'target={self.target}')
301
+ util.info("create_work_dir: --work-dir-use-target-dir: using:",
302
+ f"{os.path.abspath(self.target_path)}",
303
+ f'target={self.target}')
241
304
  self.args['work-dir'] = self.target_path
242
305
  self.args['sub-work-dir'] = ''
243
306
  return self.args['work-dir']
@@ -245,28 +308,37 @@ class Command:
245
308
  if not os.path.exists(self.args['eda-dir']):
246
309
  util.safe_mkdir(self.args['eda-dir'])
247
310
  util.info(f"create_work_dir: created {self.args['eda-dir']}")
311
+
248
312
  if self.args['design'] == "":
249
313
  if ('top' in self.args) and (self.args['top'] != ""):
250
314
  self.args['design'] = self.args['top']
251
- util.debug(f"create_work_dir: set {self.args['design']=} from {self.args['top']=}, since it was empty")
315
+ util.debug(f"create_work_dir: set {self.args['design']=} from",
316
+ f"{self.args['top']=}, since it was empty")
252
317
  else:
253
318
  self.args['design'] = "design" # generic, i.e. to create work dir "design_upload"
254
- util.debug(f"create_work_dir: set {self.args['design']=} to 'design', since it was empty and we have no top")
319
+ util.debug(f"create_work_dir: set {self.args['design']=} to 'design', since it",
320
+ "was empty and we have no top")
321
+
255
322
  if self.target == "":
256
323
  self.target = self.args['design']
257
324
  util.debug(f"create_work_dir: set {self.target=} from design name, since it was empty")
325
+
258
326
  if self.args['work-dir'] == '':
259
327
  if self.args['sub-work-dir'] == '':
260
328
  if self.args['job-name'] != '':
261
329
  self.args['sub-work-dir'] = self.args['job-name']
262
- util.debug(f"create_work_dir: set {self.args['sub-work-dir']=} from {self.args['job-name']=}, since it was empty")
330
+ util.debug(f"create_work_dir: set {self.args['sub-work-dir']=} from",
331
+ f"{self.args['job-name']=}, since it was empty")
263
332
  else:
264
333
  self.args['sub-work-dir'] = f'{self.target}.{self.command_name}'
265
- util.debug(f"create_work_dir: set {self.args['sub-work-dir']=} from {self.target=} and {self.command_name=}, since it was empty and we have no job-name")
334
+ util.debug(f"create_work_dir: set {self.args['sub-work-dir']=} from",
335
+ f"{self.target=} and {self.command_name=}, since it was empty and",
336
+ "we have no job-name")
266
337
  self.args['work-dir'] = os.path.join(self.args['eda-dir'], self.args['sub-work-dir'])
267
338
  util.debug(f"create_work_dir: set {self.args['work-dir']=}")
339
+
268
340
  keep_file = os.path.join(self.args['work-dir'], "eda.keep")
269
- if (os.path.exists(self.args['work-dir'])):
341
+ if os.path.exists(self.args['work-dir']):
270
342
  if os.path.exists(keep_file) and not self.args['force']:
271
343
  self.error(f"Cannot remove old work dir due to '{keep_file}'")
272
344
  elif os.path.abspath(self.args['work-dir']) in os.getcwd():
@@ -275,16 +347,19 @@ class Command:
275
347
  # --work-dir=$PWD
276
348
  # --work-dir=/some/path/almost/here
277
349
  # Allow it, but preserve the existing directory, we don't want to blow away
278
- # files up-heir from us.
350
+ # files up-hier from us.
279
351
  # Enables support for --work-dir=.
280
- util.info(f"Not removing existing work-dir: '{self.args['work-dir']}' is within {os.getcwd()=}")
352
+ util.info(f"Not removing existing work-dir: '{self.args['work-dir']}' is within",
353
+ f"{os.getcwd()=}")
281
354
  elif str(Path(self.args['work-dir'])).startswith(str(Path('/'))):
282
355
  # Do not allow other absolute path work dirs if it already exists.
283
356
  # This prevents you from --work-dir=~ and eda wipes out your home dir.
284
- self.error(f'Cannot use work-dir={self.args["work-dir"]} starting with absolute path "/"')
357
+ self.error(f'Cannot use work-dir={self.args["work-dir"]} starting with',
358
+ 'fabsolute path "/"')
285
359
  elif str(Path('..')) in str(Path(self.args['work-dir'])):
286
360
  # Do not allow other ../ work dirs if it already exists.
287
- self.error(f'Cannot use work-dir={self.args["work-dir"]} with up-hierarchy ../ paths')
361
+ self.error(f'Cannot use work-dir={self.args["work-dir"]} with up-hierarchy'
362
+ '../ paths')
288
363
  else:
289
364
  # If we made it this far, on a directory that exists, that appears safe
290
365
  # to delete and re-create:
@@ -295,22 +370,32 @@ class Command:
295
370
  else:
296
371
  util.safe_mkdir(self.args['work-dir'])
297
372
  util.debug(f'create_work_dir: created {self.args["work-dir"]}')
298
- if (self.args['keep']):
299
- open(keep_file, 'w').close()
373
+
374
+ if self.args['keep']:
375
+ open(keep_file, 'w', encoding='utf-8').close() # pylint: disable=consider-using-with
300
376
  util.debug(f'create_work_dir: created {keep_file=}')
301
377
  return self.args['work-dir']
302
378
 
303
- def exec(self, work_dir, command_list, background=False, stop_on_error=True,
304
- quiet=False, tee_fpath=None, shell=False):
379
+
380
+ def exec(self, work_dir: str, command_list: list, background: bool = False,
381
+ stop_on_error: bool = True, quiet: bool = False, tee_fpath: str = '',
382
+ shell: bool = False) -> (str, str, int):
383
+ '''Runs a command via subprocess and returns a tuple of (stderr, stdout, rc)
384
+
385
+ - work_dir: is passed to the subprocess call, so os.chdir(..) is not invoked.
386
+ - command_list: should be a list of individual strings w/out spaces.
387
+ - background: if False, does not print to stdout (but will print to any
388
+ tee_fpath or the global log
389
+ - shell: arg passed to subprocess, defaults to False.
390
+ '''
391
+
305
392
  if not tee_fpath and getattr(command_list, 'tee_fpath', None):
306
393
  tee_fpath = getattr(command_list, 'tee_fpath', '')
307
394
  if not quiet:
308
395
  util.info(f"exec: {' '.join(command_list)} (in {work_dir}, {tee_fpath=})")
309
- original_cwd = util.getcwd()
310
- os.chdir(work_dir)
311
396
 
312
- stdout, stderr, return_code = util.subprocess_run_background(
313
- work_dir=None, # we've already called os.chdir(work_dir).
397
+ stdout, stderr, return_code = subprocess_run_background(
398
+ work_dir=work_dir,
314
399
  command_list=command_list,
315
400
  background=background,
316
401
  fake=self.args.get('fake', False),
@@ -318,16 +403,23 @@ class Command:
318
403
  shell=shell
319
404
  )
320
405
 
321
- os.chdir(original_cwd)
322
406
  if return_code:
323
407
  self.status += return_code
324
- if stop_on_error: self.error(f"exec: returned with error (return code: {return_code})")
325
- else : util.debug(f"exec: returned with error (return code: {return_code})")
408
+ if stop_on_error:
409
+ self.error(f"exec: returned with error (return code: {return_code})")
410
+ else:
411
+ util.debug(f"exec: returned with error (return code: {return_code})")
326
412
  else:
327
413
  util.debug(f"exec: returned without error (return code: {return_code})")
328
414
  return stderr, stdout, return_code
329
415
 
330
- def set_arg(self, key, value):
416
+ def set_arg( # pylint: disable=too-many-branches
417
+ self, key, value
418
+ ) -> None:
419
+ '''Sets self.args[key] with value, and self.modified_args[key]=True
420
+
421
+ Does type checking, handles list type append
422
+ '''
331
423
 
332
424
  # Do some minimal type handling, preserving the type(self.args[key])
333
425
  if key not in self.args:
@@ -360,14 +452,14 @@ class Command:
360
452
  else:
361
453
  self.args[key] = True
362
454
  else:
363
- raise Exception(f'set_arg({key=}, {value=}) bool, {type(cur_value)=} {type(value)=}')
455
+ assert False, f'set_arg({key=}, {value=}) bool {type(cur_value)=} {type(value)=}'
364
456
 
365
457
  elif isinstance(cur_value, int):
366
458
  # if int, attempt to convert string or bool --> int
367
459
  if isinstance(value, (bool, int, str)):
368
460
  self.args[key] = int(value)
369
461
  else:
370
- raise Exception(f'set_arg({key=}, {value=}) int, {type(cur_value)=} {type(value)=}')
462
+ assert False, f'set_arg({key=}, {value=}) int {type(cur_value)=} {type(value)=}'
371
463
 
372
464
 
373
465
  else:
@@ -378,7 +470,9 @@ class Command:
378
470
  util.debug(f'Set arg["{key}"]="{self.args[key]}"')
379
471
 
380
472
 
381
- def get_argparser(self, parser_arg_list=None) -> argparse.ArgumentParser:
473
+ def get_argparser( # pylint: disable=too-many-branches
474
+ self, parser_arg_list=None
475
+ ) -> argparse.ArgumentParser:
382
476
  ''' Returns an argparse.ArgumentParser() based on self.args (dict)
383
477
 
384
478
  If parser_arg_list is not None, the ArgumentParser() is created using only the keys in
@@ -386,8 +480,8 @@ class Command:
386
480
  '''
387
481
 
388
482
  # Preference is --args-with-dashes, which then become parsed.args_with_dashes, b/c
389
- # parsed.args-with-dashes is not legal python. Some of self.args.keys() still have - or _, so
390
- # this will handle both.
483
+ # parsed.args-with-dashes is not legal python. Some of self.args.keys() still have - or _,
484
+ # so this will handle both.
391
485
  # Also, preference is for self.args.keys(), to be str with - dashes
392
486
  parser = argparse.ArgumentParser(prog='eda', add_help=False, allow_abbrev=False)
393
487
  bool_action_kwargs = util.get_argparse_bool_action_kwargs()
@@ -421,19 +515,19 @@ class Command:
421
515
  help_kwargs = {'help': f'{type(value).__name__} default={value}'}
422
516
 
423
517
 
424
- # It's important to set the default=None on these, except for list types where default is []
425
- # If the parsed Namespace has values set to None or [], we do not update. This means that as deps
426
- # are processed that have args set, they cannot override the top level args that were already set.
427
- # nor be overriden by defaults.
428
- if type(value) is bool:
429
- # For bool, support --key and --no-key with this action=argparse.BooleanOptionalAction.
430
- # Note, this means you cannot use --some-bool=True, or --some-bool=False, has to be --some-bool
431
- # or --no-some-bool.
518
+ # It's important to set the default=None on these, except for list types where default
519
+ # is []. If the parsed Namespace has values set to None or [], we do not update. This
520
+ # means that as deps are processed that have args set, they cannot override the top
521
+ # level args that were already set, nor be overriden by defaults.
522
+ if isinstance(value, bool):
523
+ # For bool, support --key and --no-key with action=argparse.BooleanOptionalAction.
524
+ # Note, this means you cannot use --some-bool=True, or --some-bool=False, has to
525
+ # be --some-bool or --no-some-bool.
432
526
  parser.add_argument(
433
527
  *arguments, default=None, **bool_action_kwargs, **help_kwargs)
434
- elif type(value) is list:
528
+ elif isinstance(value, list):
435
529
  parser.add_argument(*arguments, default=value, action='append', **help_kwargs)
436
- elif type(value) in [int, str]:
530
+ elif isinstance(value, (int, str)):
437
531
  parser.add_argument(*arguments, default=value, type=type(value), **help_kwargs)
438
532
  elif value is None:
439
533
  parser.add_argument(*arguments, default=None, **help_kwargs)
@@ -444,10 +538,10 @@ class Command:
444
538
 
445
539
 
446
540
  def run_argparser_on_list(
447
- self, tokens: list, parser_arg_list=None, apply_parsed_args:bool=True
541
+ self, tokens: list, parser_arg_list=None, apply_parsed_args: bool = True
448
542
  ) -> (dict, list):
449
- ''' Creates an argparse.ArgumentParser() for all the keys in self.args, and attempts to parse
450
- from the provided list. Parsed args are applied to self.args.
543
+ ''' Creates an argparse.ArgumentParser() for all the keys in self.args, and attempts to
544
+ parse from the provided list. Parsed args are applied to self.args.
451
545
 
452
546
  Returns a tuple of (parsed argparse.Namespace obj, list of unparsed args)
453
547
 
@@ -461,17 +555,16 @@ class Command:
461
555
  parsed, unparsed = parser.parse_known_args(tokens + [''])
462
556
  unparsed = list(filter(None, unparsed))
463
557
  except argparse.ArgumentError:
464
- self.error(f'problem {command_name=} attempting to parse_known_args for {tokens=}')
558
+ self.error(f'problem {self.command_name=} attempting to parse_known_args for {tokens=}')
465
559
 
466
560
  parsed_as_dict = vars(parsed)
467
561
 
468
562
  args_to_be_applied = {}
469
563
 
470
- for key,value in parsed_as_dict.items():
471
- # key should have _ instead of POSIX dashes, but we still support dashes like self.args['build-file'],
472
- # etc.
564
+ for key, value in parsed_as_dict.items():
565
+ # key currently has _ instead of POSIX dashes, so we convert to dashes, for
566
+ # most self.args, like self.args['build-file'], etc.
473
567
  if key not in self.args and '_' in key:
474
- # try with dashes instead of _
475
568
  key = key.replace('_', '-')
476
569
  assert key in self.args, f'{key=} not in {self.args=}'
477
570
 
@@ -483,7 +576,8 @@ class Command:
483
576
  return parsed, unparsed
484
577
 
485
578
 
486
- def apply_args_from_dict(self, args_to_be_applied:dict) -> list:
579
+ def apply_args_from_dict(self, args_to_be_applied: dict) -> list:
580
+ '''Given a dict of key/value for self.args, apply them to self.args'''
487
581
  util.debug('apply_args_from_dict() -- called by:',
488
582
  f'{self.command_name=}, {self.__class__.__name__=},',
489
583
  f'{args_to_be_applied=}')
@@ -505,7 +599,7 @@ class Command:
505
599
  f"with different cur value (cur value={self.args.get(key, None)})")
506
600
  continue
507
601
  if self.args[key] != value:
508
- util.debug(f"Command.run_argparser_on_list - setting set_arg b/c",
602
+ util.debug("Command.run_argparser_on_list - setting set_arg b/c",
509
603
  f" argparse -- {key=} {value=} (cur value={self.args[key]})")
510
604
  self.set_arg(key, value) # Note this has special handling for lists already.
511
605
  self.modified_args[key] = True
@@ -523,7 +617,7 @@ class Command:
523
617
  _, unparsed = self.run_argparser_on_list(tokens)
524
618
  if process_all and len(unparsed) > 0:
525
619
  self.error(f"Didn't understand argument: '{unparsed=}' in",
526
- f" {self.command_name=} context")
620
+ f" {self.command_name=} context, {pwd=}")
527
621
 
528
622
  return unparsed
529
623
 
@@ -556,27 +650,33 @@ class Command:
556
650
 
557
651
  Allows command handlers to make thier own customizations from --config-yaml=YAML.
558
652
  '''
559
- pass
653
+ return
560
654
 
561
655
  def update_tool_config(self) -> None:
562
656
  '''Returns None. Hook for classes like CommandSim to make tool specific overrides.'''
563
- pass
657
+ return
564
658
 
565
659
  def write_eda_config_and_args(self):
660
+ '''Attempts to write eda_output_config.yml to our work-dir'''
566
661
  if not self.args.get('work-dir', None):
567
662
  util.warning(f'Ouput work-dir not set, saving ouput eda_config to {os.getcwd()}')
568
- util.write_eda_config_and_args(dirpath=self.args.get('work-dir', os.getcwd()),
569
- command_obj_ref=self)
663
+ eda_config.write_eda_config_and_args(
664
+ dirpath=self.args.get('work-dir', os.getcwd()), command_obj_ref=self
665
+ )
570
666
 
571
667
  def is_export_enabled(self) -> bool:
572
- # check if any self.args['export'] is set in any way (but not set to False
573
- # or empty list)
574
- return any([arg.startswith('export') and v for arg,v in self.args.items()])
668
+ '''Returns True if any self.args['export'] is set in any way (but not set to False
669
+ or empty list)'''
670
+ return any(arg.startswith('export') and v for arg,v in self.args.items())
575
671
 
576
672
  def run(self) -> None:
673
+ '''Alias for do_it(self)'''
577
674
  self.do_it()
578
675
 
579
676
  def do_it(self) -> None:
677
+ '''Main entrypoint of running a command, usually called by process_tokens.
678
+
679
+ process_tokens(..) is the starting entrypoint from eda.py'''
580
680
  self.write_eda_config_and_args()
581
681
  self.error(f"No tool bound to command '{self.command_name}', you",
582
682
  " probably need to setup tool, or use '--tool <name>'")
@@ -592,7 +692,9 @@ class Command:
592
692
  self.set_tool_defines()
593
693
 
594
694
 
595
- def help(self, tokens: list = []) -> None:
695
+ def help( # pylint: disable=dangerous-default-value,too-many-branches
696
+ self, tokens: list = []
697
+ ) -> None:
596
698
  '''Since we don't quite follow standard argparger help()/usage(), we'll format our own
597
699
 
598
700
  if self.args_help has additional help information.
@@ -600,7 +702,7 @@ class Command:
600
702
 
601
703
  # Indent long lines (>100) to indent=56 (this is where we leave off w/ {vstr:12} below.
602
704
  def indent_me(text:str):
603
- return util.indent_wrap_long_text(text, width=100, indent=56)
705
+ return indent_wrap_long_text(text, width=100, indent=56)
604
706
 
605
707
  util.info('Help:')
606
708
  # using bare 'print' here, since help was requested, avoids --color and --quiet
@@ -619,14 +721,14 @@ class Command:
619
721
  lines.append(f"Generic help for command='{self.command_name}'"
620
722
  f" (using '{self.__class__.__name__}')")
621
723
  else:
622
- lines.append(f"Generic help (from class Command):")
724
+ lines.append("Generic help (from class Command):")
623
725
 
624
726
  # Attempt to run argparser on args, but don't error if it fails.
625
727
  unparsed = []
626
728
  if tokens:
627
729
  try:
628
730
  _, unparsed = self.run_argparser_on_list(tokens=tokens)
629
- except:
731
+ except Exception:
630
732
  pass
631
733
 
632
734
  for k in sorted(self.args.keys()):
@@ -635,13 +737,13 @@ class Command:
635
737
  khelp = self.args_help.get(k, '')
636
738
  if khelp:
637
739
  khelp = f' - {khelp}'
638
- if type(v) == bool :
740
+ if isinstance(v, bool):
639
741
  lines.append(indent_me(f" --{k:20} : boolean : {vstr:12}{khelp}"))
640
- elif type(v) == int:
742
+ elif isinstance(v, int):
641
743
  lines.append(indent_me(f" --{k:20} : integer : {vstr:12}{khelp}"))
642
- elif type(v) == list:
744
+ elif isinstance(v, list):
643
745
  lines.append(indent_me(f" --{k:20} : list : {vstr:12}{khelp}"))
644
- elif type(v) == str:
746
+ elif isinstance(v, str):
645
747
  vstr = "'" + v + "'"
646
748
  lines.append(indent_me(f" --{k:20} : string : {vstr:12}{khelp}"))
647
749
  else:
@@ -654,7 +756,10 @@ class Command:
654
756
  print(f'Unparsed args: {unparsed}')
655
757
 
656
758
 
657
- class CommandDesign(Command):
759
+ class CommandDesign(Command): # pylint: disable=too-many-instance-attributes
760
+ '''CommandDesign is the eda base class for command handlers that need to track files.
761
+
762
+ This is the base class for CommandSim, CommandSynth, and others.'''
658
763
 
659
764
  # Used by for DEPS work_dir_add_srcs@ commands, by class methods:
660
765
  # update_file_lists_for_work_dir(..), and resolve_target(..)
@@ -675,11 +780,16 @@ class CommandDesign(Command):
675
780
  self.args_help.update({
676
781
  'seed': 'design seed, default is 31-bit non-zero urandom',
677
782
  'top': 'TOP level verilog/SV module or VHDL entity for this target',
678
- 'all-sv': ('Maintain .sv and .v in single file list.'
679
- ' False: .sv flist separate from .v flist and separate compile(s)'
680
- ' True: .sv and .v files compiled together if possible'),
681
- 'unprocessed-plusargs': 'Args that began with +, but were not +define+ or +incdir+, +<name>, ' \
682
- + ' or +<name>=<value>. These become tool dependent, for example "sim" commands will treat as sim-plusargs',
783
+ 'all-sv': (
784
+ 'Maintain .sv and .v in single file list.'
785
+ ' False: .sv flist separate from .v flist and separate compile(s)'
786
+ ' True: .sv and .v files compiled together if possible'
787
+ ),
788
+ 'unprocessed-plusargs': (
789
+ 'Args that began with +, but were not +define+ or +incdir+, +<name>, '
790
+ ' or +<name>=<value>. These become tool dependent, for example "sim" commands will'
791
+ ' treat as sim-plusargs'
792
+ ),
683
793
  })
684
794
  self.defines = {}
685
795
  self.incdirs = []
@@ -697,12 +807,18 @@ class CommandDesign(Command):
697
807
  for (d,v) in self.config.get('defines', {}).items():
698
808
  self.defines[d] = v
699
809
 
700
- self.cached_deps = {} # key = abspath of DEPS markup file, has sub-dicts for 'data' and 'line_numbers'.
810
+ # cached_deps: key = abspath of DEPS markup file, value is a dict with
811
+ # keys 'data' and 'line_numbers'
812
+ self.cached_deps = {}
701
813
  self.targets_dict = {} # key = targets that we've already processed in DEPS files
702
814
  self.last_added_source_file_inferred_top = ''
703
815
 
704
- def run_dep_commands(self):
705
- # Run any shell@ commands from DEPS files
816
+
817
+ def run_dep_commands(self) -> None:
818
+ '''Run shell/peakrdl style commands from DEPS files
819
+
820
+ These are deferred to maintain the deps ordering, and run in that order.
821
+ '''
706
822
  self.run_dep_shell_commands()
707
823
  # Update any work_dir_add_srcs@ in our self.files, self.files_v, etc, b/c
708
824
  # self.args['work-dir'] now exists.
@@ -710,7 +826,10 @@ class CommandDesign(Command):
710
826
  # Link any non-sources to our work-dir:
711
827
  self.update_non_source_files_in_work_dir()
712
828
 
713
- def run_dep_shell_commands(self):
829
+
830
+ def run_dep_shell_commands(self) -> None:
831
+ '''Specifically runs shell command from DEPS files'''
832
+
714
833
  # Runs from self.args['work-dir']
715
834
  all_cmds_lists = []
716
835
 
@@ -729,7 +848,8 @@ class CommandDesign(Command):
729
848
  log_fnames_count.update({target_node: lognum + 1})
730
849
  all_cmds_lists += [
731
850
  [], # blank line
732
- # comment, where it came from, log to {node}__shell_{lognum}.log (or tee name from DEPS.yml)
851
+ # comment, where it came from, log to {node}__shell_{lognum}.log
852
+ # (or tee name from DEPS.yml)
733
853
  [f'# command {i}: target: {d["target_path"]} : {target_node} --> {log}'],
734
854
  ]
735
855
  if not run_from_work_dir:
@@ -743,10 +863,12 @@ class CommandDesign(Command):
743
863
 
744
864
  d['exec_list'] = clist # update to tee_fpath is set.
745
865
 
746
- util.write_shell_command_file(dirpath=self.args['work-dir'], filename='pre_compile_dep_shell_commands.sh',
747
- command_lists=all_cmds_lists)
866
+ util.write_shell_command_file(
867
+ dirpath=self.args['work-dir'], filename='pre_compile_dep_shell_commands.sh',
868
+ command_lists=all_cmds_lists
869
+ )
748
870
 
749
- for i,d in enumerate(self.dep_shell_commands):
871
+ for i, d in enumerate(self.dep_shell_commands):
750
872
  util.info(f'run_dep_shell_commands {i=}: {d=}')
751
873
  clist = util.ShellCommandList(d['exec_list'])
752
874
  tee_fpath=clist.tee_fpath
@@ -764,16 +886,23 @@ class CommandDesign(Command):
764
886
  self.exec(run_from_dir, clist, tee_fpath=tee_fpath,
765
887
  shell=self.config.get('deps_subprocess_shell', False))
766
888
 
767
- def update_file_lists_for_work_dir(self):
768
- if len(self.dep_work_dir_add_srcs) == 0:
889
+
890
+ def update_file_lists_for_work_dir(self) -> None:
891
+ '''Handles any source files that were creating by "shell" style commands in the
892
+
893
+ work-dir, these need to be added via self.add_file(..) in the correct order
894
+ of the dependencies. Basically, any files that were left with @EDA-WORK-DIR@ prefix
895
+ need to be patched and added.
896
+ '''
897
+ if not self.dep_work_dir_add_srcs:
769
898
  return
770
899
 
771
- # If we encounter any @EDA-WORK_DIR@some_file.v in self.files, self.files_v, etc, then replace it with:
772
- # self.args['work-dir'] / some_file.v:
900
+ # If we encounter any @EDA-WORK_DIR@some_file.v in self.files, self.files_v, etc,
901
+ # then replace it with: self.args['work-dir'] / some_file.v:
773
902
  _work_dir_add_srcs_path_string_len = len(self._work_dir_add_srcs_path_string)
774
903
  work_dir_abspath = os.path.abspath(self.args['work-dir'])
775
- for key in list(self.files.keys()): # list so it's not an iterator, we're updating self.files.
776
- if type(key) is str and key.startswith(self._work_dir_add_srcs_path_string):
904
+ for key in list(self.files.keys()): # list so it's not an iterator, updates self.files.
905
+ if isinstance(key, str) and key.startswith(self._work_dir_add_srcs_path_string):
777
906
  new_key = os.path.join(work_dir_abspath, key[_work_dir_add_srcs_path_string_len :])
778
907
  self.files.pop(key)
779
908
  self.files[new_key] = True
@@ -782,12 +911,23 @@ class CommandDesign(Command):
782
911
  self.files_sdc]
783
912
  for my_file_list in my_file_lists_list:
784
913
  for i,value in enumerate(my_file_list):
785
- if value and type(value) is str and value.startswith(self._work_dir_add_srcs_path_string):
786
- new_value = os.path.join(work_dir_abspath, value[_work_dir_add_srcs_path_string_len :])
914
+ if value and isinstance(value, str) and \
915
+ value.startswith(self._work_dir_add_srcs_path_string):
916
+ new_value = os.path.join(
917
+ work_dir_abspath, value[_work_dir_add_srcs_path_string_len :]
918
+ )
787
919
  my_file_list[i] = new_value
788
920
  util.debug(f"file lists: replaced {value} with {new_value}")
789
921
 
790
- def update_non_source_files_in_work_dir(self):
922
+
923
+ def update_non_source_files_in_work_dir(self) -> None:
924
+ '''For non-source files (that are tracked as 'reqs' in DEPS markup files) these
925
+
926
+ need to be copied or linked to the work-dir. For example, if some SV assumes it
927
+ can $readmemh('file_that_is_here.txt') but we're running out of work-dir. Linking
928
+ is the easy work-around vs trying to run-in-place of all SV files.
929
+ '''
930
+
791
931
  for fname in self.files_non_source:
792
932
  _, leaf_fname = os.path.split(fname)
793
933
  destfile = os.path.join(self.args['work-dir'], leaf_fname)
@@ -797,16 +937,25 @@ class CommandDesign(Command):
797
937
  util.info(f'{fname=} {self.files_caller_info=}')
798
938
  self.error(f'Non-source file (reqs?) {relfname=} does not exist from {caller_info}')
799
939
  elif not os.path.exists(destfile):
800
- util.debug(f'updating non-source file to work-dir: Linked {fname=} to {destfile=}, from {caller_info}')
940
+ util.debug(f'updating non-source file to work-dir: Linked {fname=} to {destfile=},',
941
+ f'from {caller_info}')
801
942
  if sys.platform == "win32":
802
943
  shutil.copyfile(fname, destfile) # On Windows, fall back to copying
803
944
  else:
804
945
  os.symlink(src=fname, dst=destfile)
805
946
 
806
- def get_top_name(self, name):
947
+ @staticmethod
948
+ def get_top_name(name: str) -> str:
949
+ '''Attempt to get the 'top' module name from a file, such as path/to/mine.sv will
950
+
951
+ return "mine"'''
952
+ # TODO(drew): Use the helper method in util for this instead to peek in file contents?
807
953
  return os.path.splitext(os.path.basename(name))[0]
808
954
 
809
- def process_plusarg(self, plusarg: str, pwd: str = os.getcwd()):
955
+
956
+ def process_plusarg( # pylint: disable=too-many-branches
957
+ self, plusarg: str, pwd: str = os.getcwd()
958
+ ) -> None:
810
959
  '''Retuns str, parses a +define+, +incdir+, +key=value str; adds to internal.
811
960
 
812
961
  Adds to self.defines, self.incdirs,
@@ -818,12 +967,12 @@ class CommandDesign(Command):
818
967
  # args that come from shlex.quote(token), such as:
819
968
  # token = '\'+define+OC_ROOT="/foo/bar/opencos"\''
820
969
  # So we strip all outer ' or " on the plusarg:
821
- plusarg = util.strip_outer_quotes(plusarg)
970
+ plusarg = strip_outer_quotes(plusarg)
822
971
  if not pwd:
823
972
  pwd = ''
824
973
 
825
974
  if plusarg.startswith('+define+'):
826
- plusarg = plusarg.lstrip('+define+')
975
+ plusarg = plusarg[len('+define+'):]
827
976
  m = re.match(r'^(\w+)$', plusarg)
828
977
  if m:
829
978
  k = m.group(1)
@@ -831,11 +980,12 @@ class CommandDesign(Command):
831
980
  util.debug(f"Defined {k}")
832
981
  return None
833
982
  m = re.match(r'^(\w+)\=(\S+)$', plusarg)
834
- if not m: m = re.match(r'^(\w+)\=(\"[^\"]*\")$', plusarg)
983
+ if not m:
984
+ m = re.match(r'^(\w+)\=(\"[^\"]*\")$', plusarg)
835
985
  if m:
836
986
  k = m.group(1)
837
987
  v = m.group(2)
838
- if v and type(v) is str:
988
+ if v and isinstance(v, str):
839
989
  if v.startswith('%PWD%/'):
840
990
  v = v.replace('%PWD%', os.path.abspath(pwd))
841
991
  if v.startswith('%SEED%'):
@@ -847,7 +997,7 @@ class CommandDesign(Command):
847
997
  return None
848
998
 
849
999
  if plusarg.startswith('+incdir+'):
850
- plusarg = plusarg.lstrip('+incdir+')
1000
+ plusarg = plusarg[len('+incdir+'):]
851
1001
  m = re.match(r'^(\S+)$', plusarg)
852
1002
  if m:
853
1003
  incdir = m.group(1)
@@ -863,40 +1013,59 @@ class CommandDesign(Command):
863
1013
  if not self.config.get('bare_plusarg_supported', False):
864
1014
  self.error(f"bare plusarg(s) are not supported: {plusarg}'")
865
1015
  return None
866
- elif plusarg not in self.args['unprocessed-plusargs']:
1016
+ if plusarg not in self.args['unprocessed-plusargs']:
867
1017
  self.args['unprocessed-plusargs'].append(plusarg)
868
1018
  # For anything added to unprocessed-plusarg, we have to return it, to let
869
1019
  # derived classes have the option to handle it
870
1020
  return plusarg
871
- else:
872
- self.error(f"Didn't understand +plusarg: '{plusarg}'")
1021
+
1022
+ self.error(f"Didn't understand +plusarg: '{plusarg}'")
873
1023
  return None
874
1024
 
875
1025
 
876
- def append_shell_commands(self, cmds : list):
877
- # Each entry in cmds (list) should be a dict with keys ['target_node', 'target_path', 'exec_list']
1026
+ def append_shell_commands(self, cmds : list) -> None:
1027
+ ''' Given a cmds (list), where each item is a dict with:
1028
+
1029
+ { 'target_node': str,
1030
+ 'target_path': str,
1031
+ 'exec_list': list
1032
+ }
1033
+ add to this class's dep_shell_commands for deferred processing
1034
+ '''
1035
+
878
1036
  for entry in cmds:
879
- if entry is None or type(entry) is not dict:
1037
+ if entry is None or not isinstance(entry, dict):
880
1038
  continue
881
1039
  if entry in self.dep_shell_commands:
882
- # we've already run this exact command (target node, target path, exec list), don't run it
883
- # again
1040
+ # we've already run this exact command (target node, target path, exec list),
1041
+ # don't run it again
884
1042
  continue
885
1043
 
886
1044
  assert 'exec_list' in entry, f'{entry=}'
887
1045
  util.debug(f'adding - dep_shell_command: {entry=}')
888
1046
  self.dep_shell_commands.append(entry)
889
1047
 
890
- def append_work_dir_add_srcs(self, add_srcs: list, caller_info: str):
891
- # Each entry in add_srcs (list) should be a dict with keys ['target_node', 'target_path', 'file_list']
1048
+
1049
+ def append_work_dir_add_srcs(self, add_srcs: list, caller_info: str) -> None:
1050
+ '''Given add_srcs (list), where each item is a dict with:
1051
+
1052
+ { 'target_node': str,
1053
+ 'target_path': str,
1054
+ 'file_list': list
1055
+ }
1056
+
1057
+ adds files to set self.dep_work_dir_add_src, and call add_file on it, basically
1058
+ resolving adding source files that were generated and residing in the work-dir.
1059
+ '''
892
1060
  for entry in add_srcs:
893
- if entry is None or type(entry) is not dict:
1061
+ if entry is None or not isinstance(entry, dict):
894
1062
  continue
895
1063
 
896
1064
  work_dir_files = entry['file_list']
897
1065
  for filename in work_dir_files:
898
- # Unfortunately, self.args['work-dir'] doesn't exist yet and hasn't been set, so we'll add these
899
- # files as '@EDA-WORK_DIR@' + filename, and have to replace the EDA-WORK_DIR@ string later in our flow.
1066
+ # Unfortunately, self.args['work-dir'] doesn't exist yet and hasn't been set,
1067
+ # so we'll add these files as '@EDA-WORK_DIR@' + filename, and have to replace
1068
+ # the EDA-WORK_DIR@ string later in our flow.
900
1069
  filename_use = self._work_dir_add_srcs_path_string + filename
901
1070
  dep_key_tuple = (
902
1071
  entry['target_path'],
@@ -911,14 +1080,25 @@ class CommandDesign(Command):
911
1080
  self.dep_work_dir_add_srcs.add(dep_key_tuple) # add to set()
912
1081
  elif dep_key_tuple not in self.dep_work_dir_add_srcs:
913
1082
  # we've already added the file so this dep was skipped for this one file.
914
- util.warning(f'work_dir_add_srcs@ {dep_key_tuple=} but {filename_use=}' \
915
- + 'is already in self.files (duplicate dependency on generated file?)')
1083
+ util.warning(f'work_dir_add_srcs@ {dep_key_tuple=} but {filename_use=}',
1084
+ 'is already in self.files (duplicate dependency on generated',
1085
+ 'file?)')
1086
+
916
1087
 
1088
+ def resolve_target(
1089
+ self, target: str, no_recursion: bool = False, caller_info: str = ''
1090
+ ) -> bool:
1091
+ '''Returns True if target is found. Entry point for resolving a CLI path based target name.
917
1092
 
918
- def resolve_target(self, target, no_recursion=False, caller_info=''):
919
- util.debug("Entered resolve_target(%s)" % (target))
920
- # self.target is a name we grab for the job (i.e. for naming work dir etc). we don't want the path prefix.
921
- # TODO: too messy -- there's also a self.target, args['job-name'], args['work-dir'], args['design'], args['top'], args['sub-work-dir'] ...
1093
+ Will recursively call resolve_target_core, looking for a filename existing
1094
+ or DEPS markup file in that path and a target (key) in the markup table.
1095
+
1096
+ This will populate all source file dependencies, commands, and other DEPS markup
1097
+ features as the target is resolved.
1098
+ '''
1099
+ util.debug(f"Entered resolve_target({target})")
1100
+ # self.target is a name we grab for the job (i.e. for naming work dir etc). we don't want
1101
+ # the path prefix.
922
1102
 
923
1103
  self.target_path, self.target = os.path.split(target)
924
1104
 
@@ -932,7 +1112,7 @@ class CommandDesign(Command):
932
1112
  # If the target is a file (we're at the root here processing CLI arg tokens)
933
1113
  # and that file exists and has an extension, then there's no reason to go looking
934
1114
  # in DEPS files, add the file and return True.
935
- file_base, file_ext = os.path.splitext(fpath)
1115
+ _, file_ext = os.path.splitext(fpath)
936
1116
  if forced_extension or file_ext:
937
1117
  self.add_file(fpath, caller_info=caller_info,
938
1118
  forced_extension=forced_extension)
@@ -940,23 +1120,25 @@ class CommandDesign(Command):
940
1120
 
941
1121
  return self.resolve_target_core(target, no_recursion, caller_info)
942
1122
 
943
- def resolve_target_core(self, target, no_recursion, caller_info=''):
944
- util.debug("Entered resolve_target_core(%s)" % (target))
1123
+ def resolve_target_core( # pylint: disable=too-many-locals,too-many-branches
1124
+ self, target: str, no_recursion: bool, caller_info: str = ''
1125
+ ) -> bool:
1126
+ '''Returns True if target is found. recursive point for resolving path or DEPS markup
1127
+ target names.'''
1128
+
1129
+ util.debug(f"Entered resolve_target_core({target=})")
945
1130
  found_target = False
946
- util.debug("Starting to resolve target '%s'" % (target))
947
1131
  target_path, target_node = os.path.split(target)
948
1132
 
949
- deps = None
950
- data = None
1133
+ deps, data, deps_file = None, None, None
951
1134
  found_deps_file = False
952
1135
 
953
1136
  if self.config['deps_markup_supported']:
954
- deps = deps_helpers.DepsFile(
1137
+ deps = DepsFile(
955
1138
  command_design_ref=self, target_path=target_path, cache=self.cached_deps
956
1139
  )
957
1140
  deps_file = deps.deps_file
958
1141
  data = deps.data
959
- data_only_lines = deps.line_numbers
960
1142
 
961
1143
  # Continue if we have data, otherwise look for files other than DEPS.<yml|yaml>
962
1144
  if data is not None:
@@ -969,7 +1151,7 @@ class CommandDesign(Command):
969
1151
 
970
1152
  # For convenience, use an external class for this DEPS.yml table/dict
971
1153
  # This could be re-used for any markup DEPS.json, DEPS.toml, DEPS.py, etc.
972
- deps_processor = deps_helpers.DepsProcessor(
1154
+ deps_processor = DepsProcessor(
973
1155
  command_design_ref = self,
974
1156
  deps_entry = entry,
975
1157
  target = target,
@@ -990,10 +1172,11 @@ class CommandDesign(Command):
990
1172
  # Recurse on the returned deps (ordered list), if they haven't already been traversed.
991
1173
  for x in deps_targets_to_resolve:
992
1174
  caller_info = deps.gen_caller_info(target_node)
993
- if x and type(x) is tuple:
1175
+ if x and isinstance(x, tuple):
994
1176
  # if deps_processor.process_deps_entry() gave us a tuple, it's an
995
1177
  # unprocessed 'command' that we kept in order until now. Append it.
996
- assert len(x) == 2, f'command tuple {x=} must be len 2, {target_node=} {deps_file=}'
1178
+ assert len(x) == 2, \
1179
+ f'command tuple {x=} must be len 2, {target_node=} {deps_file=}'
997
1180
  shell_commands_list, work_dir_add_srcs_list = x
998
1181
  self.append_shell_commands( cmds=shell_commands_list )
999
1182
  self.append_work_dir_add_srcs( add_srcs=work_dir_add_srcs_list,
@@ -1007,32 +1190,40 @@ class CommandDesign(Command):
1007
1190
  forced_extension=forced_extension)
1008
1191
  else:
1009
1192
  util.debug(f' ... Calling resolve_target_core({x=})')
1010
- found_target |= self.resolve_target_core( x, no_recursion, caller_info=caller_info)
1193
+ found_target |= self.resolve_target_core(
1194
+ x, no_recursion, caller_info=caller_info
1195
+ )
1011
1196
 
1012
1197
 
1013
1198
  # Done with DEPS.yml if it existed.
1014
1199
 
1015
1200
  if not found_target:
1016
- util.debug("Haven't been able to resolve %s via DEPS" % (target))
1201
+ util.debug(f"Haven't been able to resolve {target=} via DEPS")
1017
1202
  known_file_extensions_for_source = []
1018
- for x in ['verilog', 'systemverilog', 'vhdl', 'cpp']:
1019
- known_file_extensions_for_source += self.config.get('file_extensions', {}).get(x, [])
1203
+ for x in ('verilog', 'systemverilog', 'vhdl', 'cpp'):
1204
+ known_file_extensions_for_source += self.config.get(
1205
+ 'file_extensions', {}).get(x, [])
1020
1206
  for e in known_file_extensions_for_source:
1021
1207
  try_file = target + e
1022
- util.debug("Looking for %s" % (try_file))
1208
+ util.debug(f"Looking for file {try_file}")
1023
1209
  if os.path.exists(try_file):
1024
- self.add_file(try_file, caller_info=str('n/a::' + target + '::n/a'))
1210
+ self.add_file(try_file, caller_info=f'n/a::{target}::n/a')
1025
1211
  found_target = True
1026
1212
  break # move on to the next target
1027
1213
  if not found_target: # if STILL not found_this_target...
1028
- self.error("Unable to resolve target '%s'" % (target))
1214
+ self.error(f"Unable to resolve {target=}")
1029
1215
 
1030
1216
  # if we've found any target since being called, it means we found the one we were called for
1031
1217
  return found_target
1032
1218
 
1033
- def add_file(self, filename, use_abspath=True, add_to_non_sources=False,
1034
- caller_info:str='', forced_extension:str=''):
1035
- file_base, file_ext = os.path.splitext(filename)
1219
+ def add_file(
1220
+ self, filename: str, use_abspath: bool = True, add_to_non_sources: bool = False,
1221
+ caller_info: str = '', forced_extension: str = ''
1222
+ ) -> str:
1223
+ '''Given a filename, add it to one of self.files_sv or similar lists
1224
+
1225
+ based on file extension or prefix directive.'''
1226
+ _, file_ext = os.path.splitext(filename)
1036
1227
  if use_abspath:
1037
1228
  file_abspath = os.path.abspath(filename)
1038
1229
  else:
@@ -1040,8 +1231,8 @@ class CommandDesign(Command):
1040
1231
 
1041
1232
 
1042
1233
  if file_abspath in self.files:
1043
- util.debug("Not adding file %s, already have it" % (file_abspath))
1044
- return
1234
+ util.debug(f"Not adding file {file_abspath}, already have it")
1235
+ return ''
1045
1236
 
1046
1237
  known_file_ext_dict = self.config.get('file_extensions', {})
1047
1238
  v_file_ext_list = known_file_ext_dict.get('verilog', [])
@@ -1064,34 +1255,43 @@ class CommandDesign(Command):
1064
1255
 
1065
1256
  if add_to_non_sources:
1066
1257
  self.files_non_source.append(file_abspath)
1067
- util.debug("Added non-source file file %s as %s" % (filename, file_abspath))
1258
+ util.debug(f"Added non-source file file {filename} as {file_abspath}")
1068
1259
  elif file_ext in v_file_ext_list and not self.args['all-sv']:
1069
1260
  self.files_v.append(file_abspath)
1070
- util.debug("Added Verilog file %s as %s" % (filename, file_abspath))
1071
- elif file_ext in sv_file_ext_list or ((file_ext in v_file_ext_list) and self.args['all-sv']):
1261
+ util.debug(f"Added Verilog file {filename} as {file_abspath}")
1262
+ elif file_ext in sv_file_ext_list or \
1263
+ ((file_ext in v_file_ext_list) and self.args['all-sv']):
1072
1264
  self.files_sv.append(file_abspath)
1073
- util.debug("Added SystemVerilog file %s as %s" % (filename, file_abspath))
1265
+ util.debug(f"Added SystemVerilog file {filename} as {file_abspath}")
1074
1266
  elif file_ext in vhdl_file_ext_list:
1075
1267
  self.files_vhd.append(file_abspath)
1076
- util.debug("Added VHDL file %s as %s" % (filename, file_abspath))
1268
+ util.debug(f"Added VHDL file {filename} as {file_abspath}")
1077
1269
  elif file_ext in cpp_file_ext_list:
1078
1270
  self.files_cpp.append(file_abspath)
1079
- util.debug("Added C++ file %s as %s" % (filename, file_abspath))
1271
+ util.debug(f"Added C++ file {filename} as {file_abspath}")
1080
1272
  elif file_ext in sdc_file_ext_list:
1081
1273
  self.files_sdc.append(file_abspath)
1082
- util.debug("Added SDC file %s as %s" % (filename, file_abspath))
1274
+ util.debug(f"Added SDC file {filename} as {file_abspath}")
1083
1275
  else:
1084
1276
  # unknown file extension. In these cases we link the file to the working directory
1085
- # so it is available (for example, a .mem file that is expected to exist with relative path)
1277
+ # so it is available (for example, a .mem file that is expected to exist with relative
1278
+ # path)
1086
1279
  self.files_non_source.append(file_abspath)
1087
- util.debug("Added non-source file %s as %s" % (filename, file_abspath))
1280
+ util.debug(f"Added non-source file {filename} as {file_abspath}")
1088
1281
 
1089
1282
  self.files[file_abspath] = True
1090
1283
  self.files_caller_info[file_abspath] = caller_info
1091
1284
  return file_abspath
1092
1285
 
1093
- def process_tokens(self, tokens: list, process_all: bool = True,
1094
- pwd: str = os.getcwd()) -> list:
1286
+ def process_tokens( # pylint: disable=too-many-locals,too-many-branches,too-many-statements
1287
+ self, tokens: list, process_all: bool = True,
1288
+ pwd: str = os.getcwd()
1289
+ ) -> list:
1290
+ '''Returns a list of unparsed args (self.args did not have these keys present)
1291
+
1292
+ Entrypoint from eda.py calling a command handler obj.process_tokens(args).
1293
+ Dervied classes are expected to call this parent process_tokens to parse and handle
1294
+ their self.args dict'''
1095
1295
 
1096
1296
  util.debug(f'CommandDesign - process_tokens start - {tokens=}')
1097
1297
 
@@ -1105,19 +1305,21 @@ class CommandDesign(Command):
1105
1305
  # walk the list, remove all items after we're done.
1106
1306
  remove_list = []
1107
1307
  for token in unparsed:
1108
- # Since this is a raw argparser, we may have args that come from shlex.quote(token), such as:
1109
- # token = '\'+define+OC_ROOT="/foo/bar/opencos"\''
1110
- # So we have to check for strings that have been escaped for shell with extra single quotes.
1308
+ # Since this is a raw argparser, we may have args that come from shlex.quote(token),
1309
+ # such as:
1310
+ # token = '\'+define+OC_ROOT="/foo/bar/opencos"\''
1311
+ # So we have to check for strings that have been escaped for shell with extra single
1312
+ # quotes.
1111
1313
  m = re.match(r"^\'?\+\w+", token)
1112
1314
  if m:
1113
1315
  # Copy and strip all outer ' or " on the plusarg:
1114
- plusarg = util.strip_outer_quotes(token)
1316
+ plusarg = strip_outer_quotes(token)
1115
1317
  self.process_plusarg(plusarg, pwd=pwd)
1116
1318
  remove_list.append(token)
1117
1319
  for x in remove_list:
1118
1320
  unparsed.remove(x)
1119
1321
 
1120
- if len(unparsed) == 0 and self.error_on_no_files_or_targets:
1322
+ if not unparsed and self.error_on_no_files_or_targets:
1121
1323
  # derived classes can set error_on_no_files_or_targets=True
1122
1324
  # For example: CommandSim will error (requires files/targets),
1123
1325
  # CommandWaves does not (files/targets not required)
@@ -1125,11 +1327,11 @@ class CommandDesign(Command):
1125
1327
  # check the DEPS markup file for a single target, and if so run it
1126
1328
  # on the one and only target.
1127
1329
  if self.config['deps_markup_supported']:
1128
- deps = deps_helpers.DepsFile(
1330
+ deps = DepsFile(
1129
1331
  command_design_ref=self, target_path=os.getcwd(), cache=self.cached_deps
1130
1332
  )
1131
1333
  if deps.deps_file and deps.data:
1132
- all_targets = deps_helpers.deps_data_get_all_targets(deps.data)
1334
+ all_targets = deps_data_get_all_targets(deps.data)
1133
1335
  if all_targets:
1134
1336
  target = all_targets[-1]
1135
1337
  unparsed.append(target)
@@ -1151,7 +1353,7 @@ class CommandDesign(Command):
1151
1353
  file_exists, fpath, forced_extension = files.get_source_file(token)
1152
1354
  if file_exists:
1153
1355
  file_abspath = os.path.abspath(fpath)
1154
- file_base, file_ext = os.path.splitext(file_abspath)
1356
+ _, file_ext = os.path.splitext(file_abspath)
1155
1357
  if not forced_extension and not file_ext:
1156
1358
  # This probably isn't a file we want to use
1157
1359
  util.warning(f'looking for deps {token=}, found {file_abspath=}' \
@@ -1171,11 +1373,12 @@ class CommandDesign(Command):
1171
1373
  remove_list.append(token)
1172
1374
  continue # done with token, consume it, we added the file.
1173
1375
 
1174
- # we appear to be dealing with a target name which needs to be resolved (usually recursively)
1376
+ # we appear to be dealing with a target name which needs to be resolved (usually
1377
+ # recursively)
1175
1378
  if token.startswith(os.sep):
1176
1379
  target_name = token # if it's absolute path, don't prepend anything
1177
1380
  else:
1178
- target_name = os.path.join(".", token) # prepend ./so that we always have a <path>/<file>
1381
+ target_name = os.path.join(".", token) # prepend ./ to make it like: <path>/<file>
1179
1382
 
1180
1383
  util.debug(f'Calling self.resolve_target on {target_name=} ({token=})')
1181
1384
  if self.resolve_target(target_name, caller_info=caller_info):
@@ -1224,49 +1427,81 @@ class CommandDesign(Command):
1224
1427
  self.target = self.args['top']
1225
1428
 
1226
1429
  if self.error_on_missing_top and not self.args.get('top', ''):
1227
- self.error(f"Did not get a --top or DEPS top, required to run command",
1430
+ self.error("Did not get a --top or DEPS top, required to run command",
1228
1431
  f"'{self.command_name}' for tool={self.args.get('tool', None)}")
1229
1432
 
1230
1433
  return unparsed
1231
1434
 
1232
1435
 
1233
- def get_command_line_args(self, remove_args:list=[], remove_args_startswith:list=[]) -> list:
1436
+ def get_command_line_args( # pylint: disable=dangerous-default-value
1437
+ self, remove_args: list = [], remove_args_startswith: list = []
1438
+ ) -> list:
1234
1439
  '''Returns a list of all the args if you wanted to re-run this command
1235
1440
  (excludes eda, command, target).'''
1236
1441
 
1237
1442
  # This will not set bool's that are False, does not add --no-<somearg>
1238
- # nor --<somearg>=False
1239
- # This will not set str's that are empty.
1240
- # this will not set ints that are 0
1443
+ # nor --<somearg>=False (it's unclear which are 'store_true' v/ bool-action-kwargs)
1444
+ # This will not set str's that are empty
1445
+ # this will not set ints that are 0 (unless modified)
1446
+ # TODO(drew): we may want to revisit this if we tracked default arg values,
1447
+ # or perhaps if they have been "modified".
1241
1448
  ret = []
1242
1449
  for k,v in self.args.items():
1243
1450
 
1244
1451
  # Some args cannot be extracted and work, so omit these:
1245
1452
  if k in ['top-path'] + remove_args:
1246
1453
  continue
1247
- if any([k.startswith(x) for x in remove_args_startswith]):
1454
+ if any(k.startswith(x) for x in remove_args_startswith):
1248
1455
  continue
1249
1456
 
1250
- if type(v) is bool and v:
1457
+ is_modified = self.modified_args.get(k, False)
1458
+
1459
+ if isinstance(v, bool) and v:
1251
1460
  ret.append(f'--{k}')
1252
- elif type(v) is int and bool(v):
1461
+ elif isinstance(v, int) and (bool(v) or is_modified):
1253
1462
  ret.append(f'--{k}={v}')
1254
- elif type(v) is str and v:
1463
+ elif isinstance(v, str) and v:
1255
1464
  ret.append(f'--{k}={v}')
1256
- elif type(v) is list:
1465
+ elif isinstance(v, list):
1257
1466
  for item in v:
1258
- if item or type(item) not in [bool, str]:
1467
+ if item or isinstance(item, (bool, str)):
1259
1468
  # don't print bool/str that are blank.
1260
1469
  ret.append(f'--{k}={item}') # lists append
1261
1470
 
1262
1471
  return ret
1263
1472
 
1264
1473
 
1265
- _threads_start = 0
1266
- _threads_done = 0
1474
+ #_THREADS_START = 0
1475
+ #_THREADS_DONE = 0
1476
+
1477
+ class ThreadStats:
1478
+ '''To avoid globals for two ints, keep a class holder so CommandParallel and
1479
+ CommandParallelWorker can share values'''
1480
+
1481
+ done = 0
1482
+ started = 0
1483
+
1484
+ def all_done(self) -> bool:
1485
+ '''Returns True if all started jobs are done'''
1486
+ return self.started == self.done
1487
+
1488
+ def get_remaining(self) -> int:
1489
+ '''Returns remaining number of jobs'''
1490
+ return max(self.started - self.done, 0)
1491
+
1492
+ def __repr__(self) -> str:
1493
+ '''Pretty print DONE/STARTED'''
1494
+ return f'{self.done}/{self.started}'
1495
+
1267
1496
 
1268
1497
  class CommandParallelWorker(threading.Thread):
1269
- def __init__(self, n, work_queue, done_queue):
1498
+ '''Helper class for a single threaded job, often run by a CommandParallel command
1499
+ handler class'''
1500
+
1501
+ def __init__(
1502
+ self, n, work_queue: queue.Queue, done_queue: queue.Queue,
1503
+ threads_stats: ThreadStats
1504
+ ):
1270
1505
  threading.Thread.__init__(self)
1271
1506
  self.n = n
1272
1507
  self.work_queue = work_queue
@@ -1276,54 +1511,46 @@ class CommandParallelWorker(threading.Thread):
1276
1511
  self.proc = None
1277
1512
  self.pid = None
1278
1513
  self.last_timer_debug = 0
1514
+ self.threads_stats = threads_stats # ref to shared object
1279
1515
  util.debug(f"WORKER_{n}: START")
1280
1516
 
1281
1517
  def run(self):
1282
- global _threads_start
1283
- global _threads_done
1518
+ '''Runs this single job via threading. This is typically created and called by
1519
+
1520
+ a CommandParallel class handler
1521
+ '''
1522
+
1284
1523
  while True:
1285
1524
  # Get the work from the queue and expand the tuple
1286
- i, command_list, job_name, work_dir = self.work_queue.get()
1525
+ i, command_list, job_name, _ = self.work_queue.get()
1287
1526
  self.job_name = job_name
1288
1527
  try:
1289
1528
  util.debug(f"WORKER_{self.n}: Running job {i}: {job_name}")
1290
- PIPE=subprocess.PIPE
1291
- STDOUT=subprocess.STDOUT
1292
1529
  util.debug(f"WORKER_{self.n}: Calling Popen")
1293
- proc = subprocess.Popen(command_list, stdout=PIPE, stderr=STDOUT)
1530
+ proc = subprocess.Popen( # pylint: disable=consider-using-with
1531
+ command_list, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
1532
+ )
1294
1533
  self.proc = proc
1295
1534
  util.debug(f"WORKER_{self.n}: Opened process, PID={proc.pid}")
1296
1535
  self.pid = proc.pid
1297
- _threads_start += 1
1298
- while proc.returncode == None:
1536
+ self.threads_stats.started += 1
1537
+ while proc.returncode is None:
1299
1538
  try:
1300
1539
  if (time.time() - self.last_timer_debug) > 10:
1301
1540
  util.debug(f"WORKER_{self.n}: Calling proc.communicate")
1302
1541
  stdout, stderr = proc.communicate(timeout=0.5)
1303
- util.debug(f"WORKER_{self.n}: got: \n*** stdout:\n{stdout}\n*** stderr:{stderr}")
1542
+ util.debug(f"WORKER_{self.n}: got: \n*** stdout:\n{stdout}\n***",
1543
+ f"stderr:{stderr}")
1304
1544
  except subprocess.TimeoutExpired:
1305
1545
  if (time.time() - self.last_timer_debug) > 10:
1306
- util.debug(f"WORKER_{self.n}: Timer expired, stop_request={self.stop_request}")
1546
+ util.debug(f"WORKER_{self.n}: Timer expired, stop_request=",
1547
+ f"{self.stop_request}")
1307
1548
  self.last_timer_debug = time.time()
1308
- pass
1309
1549
  if self.stop_request:
1310
1550
  util.debug(f"WORKER_{self.n}: got stop request, issuing SIGINT")
1311
1551
  proc.send_signal(signal.SIGINT)
1312
1552
  util.debug(f"WORKER_{self.n}: got stop request, calling proc.wait")
1313
1553
  proc.wait()
1314
- if False and self.stop_request:
1315
- util.debug(f"WORKER_{self.n}: got stop request, issuing proc.terminate")
1316
- proc.terminate()
1317
- util.debug(f"WORKER_{self.n}: proc poll returns is now {proc.poll()}")
1318
- try:
1319
- util.debug(f"WORKER_{self.n}: Calling proc.communicate")
1320
- stdout, stderr = proc.communicate(timeout=0.2) # for completeness, in case we ever pipe/search stdout/stderr
1321
- util.debug(f"WORKER_{self.n}: got: \n*** stdout:\n{stdout}\n*** stderr:{stderr}")
1322
- except subprocess.TimeoutExpired:
1323
- util.debug(f"WORKER_{self.n}: timeout waiting for comminicate after terminate")
1324
- except:
1325
- pass
1326
- util.debug(f"WORKER_{self.n}: proc poll returns is now {proc.poll()}")
1327
1554
 
1328
1555
  util.debug(f"WORKER_{self.n}: -- out of while loop")
1329
1556
  self.pid = None
@@ -1332,99 +1559,113 @@ class CommandParallelWorker(threading.Thread):
1332
1559
  util.debug(f"WORKER_{self.n}: proc poll returns is now {proc.poll()}")
1333
1560
  try:
1334
1561
  util.debug(f"WORKER_{self.n}: Calling proc.communicate one last time")
1335
- stdout, stderr = proc.communicate(timeout=0.1) # for completeness, in case we ever pipe/search stdout/stderr
1336
- util.debug(f"WORKER_{self.n}: got: \n*** stdout:\n{stdout}\n*** stderr:{stderr}")
1562
+ stdout, stderr = proc.communicate(timeout=0.1)
1563
+ util.debug(f"WORKER_{self.n}: got: \n*** stdout:\n{stdout}\n***",
1564
+ f"stderr:{stderr}")
1337
1565
  except subprocess.TimeoutExpired:
1338
1566
  util.debug(f"WORKER_{self.n}: timeout waiting for communicate after loop?")
1339
- except:
1340
- pass
1567
+ except Exception as e:
1568
+ util.debug(f"WORKER_{self.n}: timeout with exception {e=}")
1569
+
1341
1570
  return_code = proc.poll()
1342
- util.debug(f"WORKER_{self.n}: Finished job {i}: {job_name} with return code {return_code}")
1571
+ util.debug(f"WORKER_{self.n}: Finished job {i}: {job_name} with return code",
1572
+ f"{return_code}")
1343
1573
  self.done_queue.put((i, job_name, return_code))
1344
1574
  finally:
1345
1575
  util.debug(f"WORKER_{self.n}: -- in finally block")
1346
1576
  self.work_queue.task_done()
1347
- _threads_done += 1
1577
+ self.threads_stats.done += 1
1348
1578
 
1349
1579
 
1350
1580
  class CommandParallel(Command):
1581
+ '''Base class command handler for running multiple eda jobs, such as for child classes
1582
+
1583
+ in commands.multi or commands.sweep
1584
+ '''
1351
1585
  def __init__(self, config: dict):
1352
1586
  Command.__init__(self, config=config)
1353
1587
  self.jobs = []
1354
1588
  self.jobs_status = []
1355
1589
  self.args['parallel'] = 1
1356
1590
  self.worker_threads = []
1591
+ self.threads_stats = ThreadStats()
1357
1592
 
1358
- def __del__(self):
1359
- util.debug(f"In Command.__del__, threads done/started: {_threads_done}/{_threads_start}")
1360
- if _threads_start == _threads_done:
1593
+ def __del__(self): # pylint: disable=too-many-branches
1594
+ util.debug(f"In Command.__del__, threads done/started: {self.threads_stats}")
1595
+ if self.threads_stats.all_done():
1361
1596
  return
1362
- util.warning(f"Need to shut down {_threads_start-_threads_done} worker threads...")
1597
+ util.warning(f"Need to shut down {self.threads_stats.get_remaining()} worker threads...")
1363
1598
  for w in self.worker_threads:
1364
1599
  if w.proc:
1365
1600
  util.warning(f"Requesting stop of PID {w.pid}: {w.job_name}")
1366
1601
  w.stop_request = True
1367
- for i in range(10):
1368
- util.debug(f"Threads done/started: {_threads_done}/{_threads_start}")
1369
- if _threads_start == _threads_done:
1370
- util.info(f"All threads done")
1602
+ for _ in range(10):
1603
+ util.debug(f"Threads done/started: {self.threads_stats}")
1604
+ if self.threads_stats.started == self.threads_stats.done:
1605
+ util.info("All threads done")
1371
1606
  return
1372
1607
  time.sleep(1)
1373
- subprocess.Popen(['stty', 'sane']).wait()
1374
- util.debug(f"Scanning workers again")
1608
+ subprocess.Popen(['stty', 'sane']).wait() # pylint: disable=consider-using-with
1609
+ util.debug("Scanning workers again")
1375
1610
  for w in self.worker_threads:
1376
1611
  if w.proc:
1377
1612
  util.info(f"need to SIGINT WORKER_{w.n}, may need manual cleanup, check 'ps'")
1378
1613
  if w.pid:
1379
1614
  os.kill(w.pid, signal.SIGINT)
1380
- for i in range(5):
1381
- util.debug(f"Threads done/started: {_threads_done}/{_threads_start}")
1382
- if _threads_start == _threads_done:
1383
- util.info(f"All threads done")
1615
+ for _ in range(5):
1616
+ util.debug(f"Threads done/started: {self.threads_stats}")
1617
+ if self.threads_stats.all_done():
1618
+ util.info("All threads done")
1384
1619
  return
1385
1620
  time.sleep(1)
1386
- subprocess.Popen(['stty', 'sane']).wait()
1387
- util.debug(f"Scanning workers again")
1621
+ subprocess.Popen(['stty', 'sane']).wait() # pylint: disable=consider-using-with
1622
+ util.debug("Scanning workers again")
1388
1623
  for w in self.worker_threads:
1389
1624
  if w.proc:
1390
1625
  util.info(f"need to TERM WORKER_{w.n}, probably needs manual cleanup, check 'ps'")
1391
1626
  if w.pid:
1392
1627
  os.kill(w.pid, signal.SIGTERM)
1393
- for i in range(5):
1394
- util.debug(f"Threads done/started: {_threads_done}/{_threads_start}")
1395
- if _threads_start == _threads_done:
1396
- util.info(f"All threads done")
1628
+ for _ in range(5):
1629
+ util.debug(f"Threads done/started: {self.threads_stats}")
1630
+ if self.threads_stats.all_done():
1631
+ util.info("All threads done")
1397
1632
  return
1398
1633
  time.sleep(1)
1399
- subprocess.Popen(['stty', 'sane']).wait()
1400
- util.debug(f"Scanning workers again")
1634
+ subprocess.Popen(['stty', 'sane']).wait() # pylint: disable=consider-using-with
1635
+ util.debug("Scanning workers again")
1401
1636
  for w in self.worker_threads:
1402
1637
  if w.proc:
1403
1638
  util.info(f"need to KILL WORKER_{w.n}, probably needs manual cleanup, check 'ps'")
1404
1639
  if w.pid:
1405
1640
  os.kill(w.pid, signal.SIGKILL)
1406
1641
  util.stop_log()
1407
- subprocess.Popen(['stty', 'sane']).wait()
1642
+ subprocess.Popen(['stty', 'sane']).wait() # pylint: disable=consider-using-with
1408
1643
 
1409
- def run_jobs(self, command):
1410
- # this is where we actually run the jobs. it's a messy piece of code and prob could use refactoring
1411
- # but the goal was to share as much as possible (job start, end, pass/fail judgement, etc) while
1412
- # supporting various mode combinations (parallel mode, verbose mode, fancy mode, etc) and keeping the
1413
- # UI output functional and awesome sauce
1644
+ def run_jobs( # pylint: disable=too-many-locals,too-many-branches,too-many-statements
1645
+ self, command: str
1646
+ ) -> None:
1647
+ '''Runs all "jobs" in self.jobs list, either serially or in parallel'''
1648
+
1649
+ # this is where we actually run the jobs. it's a messy piece of code and prob could use
1650
+ # refactoring but the goal was to share as much as possible (job start, end, pass/fail
1651
+ # judgement, etc) while supporting various mode combinations (parallel mode, verbose mode,
1652
+ # fancy mode, etc) and keeping the UI output functional and awesome sauce
1414
1653
 
1415
1654
  # walk targets to find the longest name, for display reasons
1416
1655
  longest_job_name = 0
1417
1656
  total_jobs = len(self.jobs)
1418
1657
  self.jobs_status = [None] * total_jobs
1419
1658
  for i in range(total_jobs):
1420
- l = len(self.jobs[i]['name'])
1421
- if l>longest_job_name: longest_job_name = l
1659
+ longest_job_name = max(longest_job_name, len(self.jobs[i]['name']))
1422
1660
 
1423
1661
  run_parallel = self.args['parallel'] > 1
1424
1662
 
1425
1663
  # figure out the width to print various numbers
1426
1664
  jobs_digits = len(f"{total_jobs}")
1427
- jobs_fmt = "%%%dd" % jobs_digits # ugh, for printing out a number with N digits
1665
+
1666
+ job_done_return_code = None
1667
+ job_done_run_time = 0
1668
+ job_done_name = ''
1428
1669
 
1429
1670
  # run the jobs!
1430
1671
  running_jobs = {}
@@ -1435,9 +1676,10 @@ class CommandParallel(Command):
1435
1676
  jobs_launched = 0
1436
1677
  num_parallel = min(len(self.jobs), self.args['parallel'])
1437
1678
  # 16 should really be the size of window or ?
1438
- (columns,lines) = shutil.get_terminal_size()
1439
- # we will enter fancy mode if we are parallel and we can leave 6 lines of regular scrolling output
1440
- fancy_mode = util.args['fancy'] and (num_parallel > 1) and (num_parallel <= (lines-6))
1679
+ _, lines = shutil.get_terminal_size()
1680
+ # we will enter fancy mode if we are parallel and we can leave 6 lines of regular scrolling
1681
+ # output
1682
+ fancy_mode = all([util.args['fancy'], num_parallel > 1, num_parallel <= (lines-6)])
1441
1683
  multi_cwd = util.getcwd() + os.sep
1442
1684
 
1443
1685
  self.patch_jobs_for_duplicate_target_names()
@@ -1448,8 +1690,12 @@ class CommandParallel(Command):
1448
1690
  work_queue = queue.Queue()
1449
1691
  done_queue = queue.Queue()
1450
1692
  for x in range(num_parallel):
1451
- worker = CommandParallelWorker(x, work_queue, done_queue)
1452
- # Setting daemon to True will let the main thread exit even though the workers are blocking
1693
+ worker = CommandParallelWorker(
1694
+ n=x, work_queue=work_queue, done_queue=done_queue,
1695
+ threads_stats=self.threads_stats
1696
+ )
1697
+ # Setting daemon to True will let the main thread exit even though the workers are
1698
+ # blocking
1453
1699
  worker.daemon = True
1454
1700
  worker.start()
1455
1701
  self.worker_threads.append(worker)
@@ -1460,35 +1706,48 @@ class CommandParallel(Command):
1460
1706
  for x in range(num_parallel):
1461
1707
  util.fancy_print(f"Starting worker {x}", x)
1462
1708
 
1463
- while len(self.jobs) or len(running_jobs.items()):
1709
+ while self.jobs or running_jobs:
1464
1710
  job_done = False
1465
1711
  job_done_quiet = False
1466
1712
  anything_done = False
1467
1713
 
1468
1714
  def sprint_job_line(job_number=0, job_name="", final=False, hide_stats=False):
1469
- return (f"INFO: [EDA] " +
1470
- util.string_or_space(f"[job {jobs_fmt%job_number}/{jobs_fmt%total_jobs} ", final) +
1471
- util.string_or_space(f"| pass ", hide_stats or final) +
1472
- util.string_or_space(f"{jobs_fmt%len(passed_jobs)}/{jobs_fmt%jobs_complete} ", hide_stats) +
1473
- util.string_or_space(f"@ {(100*(jobs_complete))/total_jobs:5.1f}%", hide_stats or final) +
1474
- util.string_or_space(f"] ", final) +
1475
- f"{command} {(job_name+' ').ljust(longest_job_name+3,'.')}")
1476
-
1477
- # for any kind of run (parallel or not, fancy or not, verbose or not) ... can we launch a job?
1478
- if len(self.jobs) and (len(running_jobs.items()) < num_parallel):
1715
+ return (
1716
+ "INFO: [EDA] " +
1717
+ string_or_space(
1718
+ f"[job {job_number:0{jobs_digits}d}/{total_jobs:0{jobs_digits}d} ",
1719
+ final) +
1720
+ string_or_space("| pass ", hide_stats or final) +
1721
+ string_or_space(
1722
+ f"{len(passed_jobs):0{jobs_digits}d}/{jobs_complete:0{jobs_digits}d} ",
1723
+ hide_stats) +
1724
+ string_or_space(f"@ {(100 * (jobs_complete)) / total_jobs : 5.1f}%",
1725
+ hide_stats or final) +
1726
+ string_or_space("] ", final) +
1727
+ f"{command} {(job_name+' ').ljust(longest_job_name+3,'.')}"
1728
+ )
1729
+
1730
+ # for any kind of run (parallel or not, fancy or not, verbose or not) ...
1731
+ # can we launch a job?
1732
+ if self.jobs and (len(running_jobs) < num_parallel):
1479
1733
  # we are launching a job
1480
1734
  jobs_launched += 1
1481
1735
  anything_done = True
1482
1736
  job = self.jobs.pop(0)
1483
- if job['name'].startswith(multi_cwd): job['name'] = job['name'][len(multi_cwd):]
1484
- # in all but fancy mode, we will print this text at the launch of a job. It may get a newline below
1737
+ # TODO(drew): it might be nice to pass more items on 'job' dict, like
1738
+ # logfile or job-name, so CommandSweep or CommandMulti don't have to set
1739
+ # via args. on their command_list.
1740
+ if job['name'].startswith(multi_cwd):
1741
+ job['name'] = job['name'][len(multi_cwd):]
1742
+ # in all but fancy mode, we will print this text at the launch of a job. It may
1743
+ # get a newline below
1485
1744
  job_text = sprint_job_line(jobs_launched, job['name'], hide_stats=run_parallel)
1486
1745
  command_list = job['command_list']
1487
1746
  cwd = util.getcwd()
1488
1747
 
1489
1748
  if run_parallel:
1490
1749
  # multithreaded job launch: add to queue
1491
- worker = workers.pop(0) # we don't actually know which thread will pick up, but GUI will be consistent
1750
+ worker = workers.pop(0)
1492
1751
  running_jobs[str(jobs_launched)] = { 'name' : job['name'],
1493
1752
  'number' : jobs_launched,
1494
1753
  'worker' : worker,
@@ -1498,15 +1757,20 @@ class CommandParallel(Command):
1498
1757
  suffix = "<START>"
1499
1758
  if fancy_mode:
1500
1759
  util.fancy_print(job_text+suffix, worker)
1760
+ elif failed_jobs:
1761
+ # if we aren't in fancy mode, we will print a START line, periodic RUNNING
1762
+ # lines, and PASS/FAIL line per-job
1763
+ util.print_orange(job_text + Colors.yellow + suffix)
1501
1764
  else:
1502
- # if we aren't in fancy mode, we will print a START line, periodic RUNNING lines, and PASS/FAIL line per-job
1503
- if len(failed_jobs): util.print_orange(job_text + util.string_yellow + suffix)
1504
- else: util.print_yellow(job_text + util.string_yellow + suffix)
1765
+ util.print_yellow(job_text + Colors.yellow + suffix)
1505
1766
  else:
1506
- # single-threaded job launch, we are going to print out job info as we start each job... no newline
1507
- # since non-verbose silences the job and prints only <PASS>/<FAIL> after the trailing "..." we leave here
1508
- if len(failed_jobs): util.print_orange(job_text, end="")
1509
- else: util.print_yellow(job_text, end="")
1767
+ # single-threaded job launch, we are going to print out job info as we start
1768
+ # each job... no newline. since non-verbose silences the job and prints only
1769
+ # <PASS>/<FAIL> after the trailing "..." we leave here
1770
+ if failed_jobs:
1771
+ util.print_orange(job_text, end="")
1772
+ else:
1773
+ util.print_yellow(job_text, end="")
1510
1774
  job_done_number = jobs_launched
1511
1775
  job_done_name = job['name']
1512
1776
  job_start_time = time.time()
@@ -1518,13 +1782,16 @@ class CommandParallel(Command):
1518
1782
  _, _, job_done_return_code = self.exec(
1519
1783
  cwd, command_list, background=False, stop_on_error=False, quiet=False
1520
1784
  )
1521
- # reprint the job text previously printed before running job(and given "\n" after the trailing "...")
1785
+ # reprint the job text previously printed before running job (and given
1786
+ # "\n" after the trailing "...")
1522
1787
  else:
1523
1788
  # run job, swallowing output (hope you have a logfile)
1524
1789
  _, _, job_done_return_code = self.exec(
1525
1790
  cwd, command_list, background=True, stop_on_error=False, quiet=True
1526
1791
  )
1527
- job_done_quiet = True # in this case, we have the job start text (trailing "...", no newline) printed
1792
+ # in this case, we have the job start text (trailing "...", no newline)
1793
+ # printed
1794
+ job_done_quiet = True
1528
1795
  job_done = True
1529
1796
  job_done_run_time = time.time() - job_start_time
1530
1797
  # Since we consumed the job, use the job['index'] to track the per-job status:
@@ -1532,15 +1799,16 @@ class CommandParallel(Command):
1532
1799
  if run_parallel:
1533
1800
  # parallel run, check for completed job
1534
1801
  if done_queue.qsize():
1535
- # we're collecting a finished job from a worker thread. note we will only reap one job per iter of the big
1536
- # loop, so as to share job completion code at the bottom
1802
+ # we're collecting a finished job from a worker thread. note we will only
1803
+ # reap one job per iter of the big loop, so as to share job completion code
1804
+ # at the bottom
1537
1805
  anything_done = True
1538
1806
  job_done = True
1539
1807
  job_done_number, job_done_name, job_done_return_code = done_queue.get()
1540
1808
  t = running_jobs[str(job_done_number)]
1541
1809
  # in fancy mode, we need to clear the worker line related to this job.
1542
1810
  if fancy_mode:
1543
- util.fancy_print(f"INFO: [EDA] Parallel: Worker Idle ...", t['worker'])
1811
+ util.fancy_print("INFO: [EDA] Parallel: Worker Idle ...", t['worker'])
1544
1812
  job_done_run_time = time.time() - t['start_time']
1545
1813
  util.debug(f"removing job #{job_done_number} from running jobs")
1546
1814
  del running_jobs[str(job_done_number)]
@@ -1552,33 +1820,40 @@ class CommandParallel(Command):
1552
1820
  if (fancy_mode or (time.time() - t['update_time']) > 30):
1553
1821
  t['update_time'] = time.time()
1554
1822
  job_text = sprint_job_line(t['number'], t['name'], hide_stats=True)
1555
- suffix = f"<RUNNING: {util.sprint_time(time.time() - t['start_time'])}>"
1823
+ suffix = f"<RUNNING: {sprint_time(time.time() - t['start_time'])}>"
1556
1824
  if fancy_mode:
1557
1825
  util.fancy_print(f"{job_text}{suffix}", t['worker'])
1826
+ elif failed_jobs:
1827
+ util.print_orange(job_text + Colors.yellow + suffix)
1558
1828
  else:
1559
- if len(failed_jobs): util.print_orange(job_text+util.string_yellow+suffix)
1560
- else: util.print_yellow(job_text+util.string_yellow+suffix)
1829
+ util.print_yellow(job_text + Colors.yellow + suffix)
1561
1830
 
1562
1831
  # shared job completion code
1563
- # single or multi-threaded, we can arrive here to harvest <= 1 jobs, and need {job, return_code} valid, and
1564
- # we expect the start of a status line to have been printed, ready for pass/fail
1832
+ # single or multi-threaded, we can arrive here to harvest <= 1 jobs, and need
1833
+ # {job, return_code} valid, and we expect the start of a status line to have been
1834
+ # printed, ready for pass/fail
1565
1835
  if job_done:
1566
1836
  jobs_complete += 1
1567
1837
  if job_done_return_code is None or job_done_return_code:
1568
- # embed the color code, to change color of pass/fail during the util.print_orange/yellow below
1838
+ # embed the color code, to change color of pass/fail during the
1839
+ # util.print_orange/yellow below
1569
1840
  if job_done_return_code == 124:
1570
- # bash uses 124 for bash timeout errors, if that was preprended to the command list.
1571
- suffix = f"{util.string_red}<TOUT: {util.sprint_time(job_done_run_time)}>"
1841
+ # bash uses 124 for bash timeout errors, if that was preprended to the
1842
+ # command list.
1843
+ suffix = f"{Colors.red}<TOUT: {sprint_time(job_done_run_time)}>"
1572
1844
  else:
1573
- suffix = f"{util.string_red}<FAIL: {util.sprint_time(job_done_run_time)}>"
1845
+ suffix = f"{Colors.red}<FAIL: {sprint_time(job_done_run_time)}>"
1574
1846
  failed_jobs.append(job_done_name)
1575
1847
  else:
1576
- suffix = f"{util.string_green}<PASS: {util.sprint_time(job_done_run_time)}>"
1848
+ suffix = f"{Colors.green}<PASS: {sprint_time(job_done_run_time)}>"
1577
1849
  passed_jobs.append(job_done_name)
1578
1850
  # we want to print in one shot, because in fancy modes that's all that we're allowed
1579
- job_done_text = "" if job_done_quiet else sprint_job_line(job_done_number, job_done_name)
1580
- if len(failed_jobs): util.print_orange(f"{job_done_text}{suffix}")
1581
- else: util.print_yellow(f"{job_done_text}{suffix}")
1851
+ job_done_text = "" if job_done_quiet else sprint_job_line(job_done_number,
1852
+ job_done_name)
1853
+ if failed_jobs:
1854
+ util.print_orange(f"{job_done_text}{suffix}")
1855
+ else:
1856
+ util.print_yellow(f"{job_done_text}{suffix}")
1582
1857
  self.jobs_status[job_done_number-1] = job_done_return_code
1583
1858
 
1584
1859
  if not anything_done:
@@ -1586,23 +1861,26 @@ class CommandParallel(Command):
1586
1861
 
1587
1862
  if total_jobs:
1588
1863
  emoji = "< :) >" if (len(passed_jobs) == total_jobs) else "< :( >"
1589
- util.info(sprint_job_line(final=True,job_name="jobs passed")+emoji, start="")
1864
+ util.info(sprint_job_line(final=True, job_name="jobs passed") + emoji, start="")
1590
1865
  else:
1591
- util.info(f"Parallel: <No jobs found>")
1866
+ util.info("Parallel: <No jobs found>")
1592
1867
  # Make sure all jobs have a set status:
1593
- for i,rc in enumerate(self.jobs_status):
1594
- if rc is None or type(rc) != int:
1868
+ for i, rc in enumerate(self.jobs_status):
1869
+ if rc is None or not isinstance(rc, int):
1595
1870
  self.error(f'job {i=} {rc=} did not return a proper return code')
1596
- jobs_status[i] = 1
1871
+ self.jobs_status[i] = 2
1597
1872
 
1598
1873
  # if self.status > 0, then keep it non-zero, else set it if we still have running jobs.
1599
1874
  if self.status == 0:
1600
- self.status = 0 if len(self.jobs_status) == 0 else max(self.jobs_status)
1875
+ if self.jobs_status:
1876
+ self.status = max(self.jobs_status)
1877
+ # else keep at 0, empty list.
1601
1878
  util.fancy_stop()
1602
1879
 
1603
1880
  @staticmethod
1604
1881
  def get_name_from_target(target: str) -> str:
1605
- return target.replace('../', '').lstrip('./')
1882
+ '''Given a target path, strip leftmost path separators to get a shorter string name'''
1883
+ return target.replace('../', '').lstrip('./').lstrip(os.path.sep)
1606
1884
 
1607
1885
 
1608
1886
  def update_args_list(self, args: list, tool: str) -> None:
@@ -1646,7 +1924,7 @@ class CommandParallel(Command):
1646
1924
  tokens=tokens.copy(),
1647
1925
  apply_parsed_args=False,
1648
1926
  )
1649
- util.debug(f'{self.command_name}: {single_cmd_unparsed=}')
1927
+ util.debug(f'{self.command_name}: {single_cmd_parsed=}, {single_cmd_unparsed=}')
1650
1928
 
1651
1929
  # There should not be any single_cmd_unparsed args starting with '-'
1652
1930
  bad_remaining_args = [x for x in single_cmd_unparsed if x.startswith('-')]
@@ -1663,34 +1941,36 @@ class CommandParallel(Command):
1663
1941
  '''Examines list self.jobs, and if leaf target names are duplicate will
1664
1942
  patch each command's job-name to:
1665
1943
  --job-name=path.leaf.command[.tool]
1944
+
1945
+ Also do for --force-logfile
1666
1946
  '''
1667
1947
 
1668
- def get_job_name(job_dict: dict) -> str:
1669
- '''Fishes the job-name out of an entry in self.jobs'''
1948
+ def get_job_arg(job_dict: dict, arg_name: str) -> str:
1949
+ '''Fishes the arg_name out of an entry in self.jobs'''
1670
1950
  for i, item in enumerate(job_dict['command_list']):
1671
- if item.startswith('--job-name='):
1672
- _, name = item.split('--job-name=')
1951
+ if item.startswith(f'--{arg_name}='):
1952
+ _, name = item.split(f'--{arg_name}=')
1673
1953
  return name
1674
- elif item == '--job-name':
1954
+ if item == f'--{arg_name}':
1675
1955
  return job_dict['command_list'][i + 1]
1676
1956
  return ''
1677
1957
 
1678
- def replace_job_name(job_dict: dict, new_job_name: str) -> dict:
1679
- '''Replaces the job-name in an entry in self.jobs'''
1958
+ def replace_job_arg(job_dict: dict, arg_name: str, new_value: str) -> bool:
1959
+ '''Replaces the arg_name's value in an entry in self.jobs'''
1680
1960
  for i, item in enumerate(job_dict['command_list']):
1681
- if item.startswith('--job-name='):
1682
- job_dict['command_list'][i] = '--job-name=' + new_job_name
1683
- return job_dict
1684
- elif item == '--job-name':
1685
- job_dict['command_list'][i + 1] = new_job_name
1686
- return job_dict
1687
- return job_dict
1961
+ if item.startswith(f'--{arg_name}='):
1962
+ job_dict['command_list'][i] = f'--{arg_name}=' + new_value
1963
+ return True
1964
+ if item == f'--{arg_name}':
1965
+ job_dict['command_list'][i + 1] = new_value
1966
+ return True
1967
+ return False
1688
1968
 
1689
1969
 
1690
1970
  job_names_count_dict = {}
1691
1971
  for job_dict in self.jobs:
1692
1972
 
1693
- key = get_job_name(job_dict)
1973
+ key = get_job_arg(job_dict, arg_name='job-name')
1694
1974
  if not key:
1695
1975
  self.error(f'{job_dict=} needs to have a --job-name= arg attached')
1696
1976
  if key not in job_names_count_dict:
@@ -1699,16 +1979,29 @@ class CommandParallel(Command):
1699
1979
  job_names_count_dict[key] += 1
1700
1980
 
1701
1981
  for i, job_dict in enumerate(self.jobs):
1702
- key = get_job_name(job_dict)
1982
+ key = get_job_arg(job_dict, 'job-name')
1703
1983
  if job_names_count_dict[key] < 2:
1704
1984
  continue
1705
1985
 
1706
1986
  tpath, _ = os.path.split(job_dict['target'])
1707
1987
 
1708
1988
  # prepend path information to job-name:
1709
- patched_sub_work_dir = False
1710
1989
  patched_target_path = os.path.relpath(tpath).replace(os.sep, '_')
1711
1990
  new_job_name = f'{patched_target_path}.{key}'
1712
- new_job_dict = replace_job_name(job_dict, new_job_name)
1713
- self.jobs[i] = new_job_dict
1991
+ replace_job_arg(job_dict, arg_name='job-name', new_value=new_job_name)
1992
+
1993
+ # prepend path information to force-logfile (if present):
1994
+ force_logfile = get_job_arg(job_dict, arg_name='force-logfile')
1995
+ if force_logfile:
1996
+ left, right = os.path.split(force_logfile)
1997
+ new_force_logfile = os.path.join(left, f'{patched_target_path}.{right}')
1998
+ replace_job_arg(
1999
+ job_dict, arg_name='force-logfile', new_value=new_force_logfile
2000
+ )
2001
+ util.debug(
2002
+ f'Patched job {job_dict["name"]}: --force-logfile={new_force_logfile}'
2003
+ )
2004
+
2005
+
2006
+ self.jobs[i] = job_dict
1714
2007
  util.debug(f'Patched job {job_dict["name"]}: --job-name={new_job_name}')