opencos-eda 0.2.48__py3-none-any.whl → 0.2.49__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- opencos/__init__.py +4 -2
- opencos/_version.py +10 -7
- opencos/commands/flist.py +8 -7
- opencos/commands/multi.py +13 -14
- opencos/commands/sweep.py +3 -2
- opencos/deps/__init__.py +0 -0
- opencos/deps/defaults.py +69 -0
- opencos/deps/deps_commands.py +419 -0
- opencos/deps/deps_file.py +326 -0
- opencos/deps/deps_processor.py +670 -0
- opencos/deps_schema.py +7 -8
- opencos/eda.py +84 -64
- opencos/eda_base.py +572 -316
- opencos/eda_config.py +80 -14
- opencos/eda_extract_targets.py +22 -14
- opencos/eda_tool_helper.py +33 -7
- opencos/export_helper.py +166 -86
- opencos/export_json_convert.py +31 -23
- opencos/files.py +2 -1
- opencos/hw/__init__.py +0 -0
- opencos/{oc_cli.py → hw/oc_cli.py} +9 -4
- opencos/names.py +0 -4
- opencos/peakrdl_cleanup.py +13 -7
- opencos/seed.py +19 -11
- opencos/tests/helpers.py +3 -2
- opencos/tests/test_deps_helpers.py +35 -32
- opencos/tests/test_eda.py +36 -29
- opencos/tests/test_eda_elab.py +5 -3
- opencos/tests/test_eda_synth.py +1 -1
- opencos/tests/test_oc_cli.py +1 -1
- opencos/tests/test_tools.py +3 -2
- opencos/tools/iverilog.py +2 -2
- opencos/tools/modelsim_ase.py +2 -2
- opencos/tools/riviera.py +1 -1
- opencos/tools/slang.py +1 -1
- opencos/tools/surelog.py +1 -1
- opencos/tools/verilator.py +1 -1
- opencos/tools/vivado.py +1 -1
- opencos/tools/yosys.py +4 -3
- opencos/util.py +374 -468
- opencos/utils/__init__.py +0 -0
- opencos/utils/markup_helpers.py +98 -0
- opencos/utils/str_helpers.py +111 -0
- opencos/utils/subprocess_helpers.py +108 -0
- {opencos_eda-0.2.48.dist-info → opencos_eda-0.2.49.dist-info}/METADATA +1 -1
- opencos_eda-0.2.49.dist-info/RECORD +88 -0
- {opencos_eda-0.2.48.dist-info → opencos_eda-0.2.49.dist-info}/entry_points.txt +1 -1
- opencos/deps_helpers.py +0 -1346
- opencos_eda-0.2.48.dist-info/RECORD +0 -79
- /opencos/{pcie.py → hw/pcie.py} +0 -0
- {opencos_eda-0.2.48.dist-info → opencos_eda-0.2.49.dist-info}/WHEEL +0 -0
- {opencos_eda-0.2.48.dist-info → opencos_eda-0.2.49.dist-info}/licenses/LICENSE +0 -0
- {opencos_eda-0.2.48.dist-info → opencos_eda-0.2.49.dist-info}/licenses/LICENSE.spdx +0 -0
- {opencos_eda-0.2.48.dist-info → opencos_eda-0.2.49.dist-info}/top_level.txt +0 -0
opencos/eda_base.py
CHANGED
|
@@ -1,22 +1,36 @@
|
|
|
1
|
+
'''opencos.eda_base holds base classes for opencos.tools and opencos.commands
|
|
2
|
+
|
|
3
|
+
Note that opencos.Command.process_tokens (on a derived class) is the main entrypoint
|
|
4
|
+
from eda.py, it will find a command handler class for a command + tool, and call
|
|
5
|
+
its obj.process_tokens(args)
|
|
6
|
+
'''
|
|
7
|
+
|
|
8
|
+
# TODO(drew): Clean up the following pylint
|
|
9
|
+
# pylint: disable=too-many-lines
|
|
10
|
+
|
|
1
11
|
import argparse
|
|
2
12
|
import copy
|
|
3
|
-
import glob
|
|
4
13
|
import subprocess
|
|
5
14
|
import os
|
|
6
15
|
import queue
|
|
7
16
|
import re
|
|
8
17
|
import shutil
|
|
9
|
-
import shlex
|
|
10
18
|
import signal
|
|
11
19
|
import sys
|
|
12
20
|
import threading
|
|
13
21
|
import time
|
|
14
22
|
from pathlib import Path
|
|
15
23
|
|
|
16
|
-
import
|
|
17
|
-
from opencos import seed, deps_helpers, util, files
|
|
24
|
+
from opencos import seed, util, files
|
|
18
25
|
from opencos import eda_config
|
|
19
|
-
|
|
26
|
+
|
|
27
|
+
from opencos.util import Colors
|
|
28
|
+
from opencos.utils.str_helpers import sprint_time, strip_outer_quotes, string_or_space, \
|
|
29
|
+
indent_wrap_long_text
|
|
30
|
+
from opencos.utils.subprocess_helpers import subprocess_run_background
|
|
31
|
+
|
|
32
|
+
from opencos.deps.deps_file import DepsFile, deps_data_get_all_targets
|
|
33
|
+
from opencos.deps.deps_processor import DepsProcessor
|
|
20
34
|
|
|
21
35
|
|
|
22
36
|
def print_base_help() -> None:
|
|
@@ -28,9 +42,12 @@ def print_base_help() -> None:
|
|
|
28
42
|
|
|
29
43
|
|
|
30
44
|
def get_argparser() -> argparse.ArgumentParser:
|
|
45
|
+
'''Returns the ArgumentParser for general eda CLI'''
|
|
31
46
|
parser = argparse.ArgumentParser(prog='eda options', add_help=False, allow_abbrev=False)
|
|
32
47
|
parser.add_argument('-q', '--quit', action='store_true',
|
|
33
|
-
help=
|
|
48
|
+
help=(
|
|
49
|
+
'For interactive mode (eda called with no options, command, or'
|
|
50
|
+
' targets)'))
|
|
34
51
|
parser.add_argument('--exit', action='store_true', help='same as --quit')
|
|
35
52
|
parser.add_argument('-h', '--help', action='store_true')
|
|
36
53
|
|
|
@@ -40,22 +57,25 @@ def get_argparser() -> argparse.ArgumentParser:
|
|
|
40
57
|
help='Tool to use for this command, such as: modelsim_ase, verilator,' \
|
|
41
58
|
+ ' modelsim_ase=/path/to/bin/vsim, verilator=/path/to/bin/verilator')
|
|
42
59
|
parser.add_argument('--eda-safe', action='store_true',
|
|
43
|
-
help='disable all DEPS file deps shell commands, overrides values from
|
|
60
|
+
help=('disable all DEPS file deps shell commands, overrides values from'
|
|
61
|
+
' --config-yml'))
|
|
44
62
|
return parser
|
|
45
63
|
|
|
46
64
|
|
|
47
65
|
def get_argparser_short_help() -> str:
|
|
66
|
+
'''Returns str for the eda_base argparser (--help, --tool, etc)'''
|
|
48
67
|
return util.get_argparser_short_help(parser=get_argparser())
|
|
49
68
|
|
|
50
69
|
|
|
51
|
-
def get_eda_exec(command:str=''):
|
|
70
|
+
def get_eda_exec(command: str = '') -> str:
|
|
71
|
+
'''Returns the full path of `eda` executable to be used for a given eda <command>'''
|
|
52
72
|
# NOTE(drew): This is kind of flaky. 'eda multi' reinvokes 'eda'. But the executable for 'eda'
|
|
53
73
|
# is one of:
|
|
54
74
|
# 1. pip3 install opencos-eda
|
|
55
75
|
# -- script 'eda', installed from PyPi
|
|
56
76
|
# 2. pip3 uninstall .; python3 -m build; pip3 install
|
|
57
77
|
# -- script 'eda' but installed from local.
|
|
58
|
-
# 2. (opencos repo)/bin/eda - a python wrapper to link to (opencos repo)/opencos/eda.py
|
|
78
|
+
# 2. (opencos repo)/bin/eda - a python wrapper to link to (opencos repo)/opencos/eda.py, but
|
|
59
79
|
# packages cannot be run standalone, they need to be called as: python3 -m opencos.eda,
|
|
60
80
|
# and do not work with relative paths. This only works if env OC_ROOT is set or can be found.
|
|
61
81
|
# 3. If you ran 'source bin/addpath' then you are always using the local (opencos repo)/bin/eda
|
|
@@ -64,11 +84,13 @@ def get_eda_exec(command:str=''):
|
|
|
64
84
|
# Can we run from OC_ROOT/bin/eda?
|
|
65
85
|
oc_root = util.get_oc_root()
|
|
66
86
|
if not oc_root:
|
|
67
|
-
util.error(f"Need 'eda' in our path to run 'eda {command}', could not find env
|
|
87
|
+
util.error(f"Need 'eda' in our path to run 'eda {command}', could not find env",
|
|
88
|
+
f"OC_ROOT, {eda_path=}, {oc_root=}")
|
|
68
89
|
else:
|
|
69
90
|
bin_eda = os.path.join(oc_root, 'bin', 'eda')
|
|
70
91
|
if not os.path.exists(bin_eda):
|
|
71
|
-
util.error(f"Need 'eda' in our path to run 'eda {command}', cound not find
|
|
92
|
+
util.error(f"Need 'eda' in our path to run 'eda {command}', cound not find",
|
|
93
|
+
f"bin/, {eda_path=}, {oc_root=}, {bin_eda=}")
|
|
72
94
|
else:
|
|
73
95
|
util.info(f"'eda' not in path, using {bin_eda=} for 'eda' {command} executable")
|
|
74
96
|
eda_path = os.path.abspath(bin_eda)
|
|
@@ -76,9 +98,10 @@ def get_eda_exec(command:str=''):
|
|
|
76
98
|
return eda_path
|
|
77
99
|
|
|
78
100
|
|
|
79
|
-
def which_tool(command, config):
|
|
80
|
-
'''Returns which tool will be used for a command, given the command_handlers in
|
|
81
|
-
|
|
101
|
+
def which_tool(command: str, config: dict) -> str:
|
|
102
|
+
'''Returns which tool (str) will be used for a command, given the command_handlers in
|
|
103
|
+
|
|
104
|
+
config dict.'''
|
|
82
105
|
if config is None:
|
|
83
106
|
util.error(f'which_tool({command=}) called w/out config')
|
|
84
107
|
if not command in config.get('command_handler', {}):
|
|
@@ -136,9 +159,10 @@ class Tool:
|
|
|
136
159
|
self.set_tool_config_from_config()
|
|
137
160
|
|
|
138
161
|
def set_exe(self, config: dict) -> None:
|
|
162
|
+
'''Sets self._EXE based on config'''
|
|
139
163
|
if self._TOOL and self._TOOL in config.get('auto_tools_order', [{}])[0]:
|
|
140
164
|
exe = config.get('auto_tools_order', [{}])[0][self._TOOL].get('exe', '')
|
|
141
|
-
if exe and
|
|
165
|
+
if exe and isinstance(exe, list):
|
|
142
166
|
exe = exe[0] # pick first
|
|
143
167
|
if exe and exe != self._EXE:
|
|
144
168
|
util.info(f'Override for {self._TOOL} using exe {exe}')
|
|
@@ -155,9 +179,16 @@ class Tool:
|
|
|
155
179
|
return self._VERSION
|
|
156
180
|
|
|
157
181
|
def set_tool_defines(self) -> None:
|
|
158
|
-
|
|
182
|
+
'''Derived classes may override, sets any additional defines based on tool.'''
|
|
183
|
+
return
|
|
184
|
+
|
|
159
185
|
|
|
160
186
|
class Command:
|
|
187
|
+
'''Base class for all: eda <command>
|
|
188
|
+
|
|
189
|
+
The Command class should be used when you don't require files, otherwise consider
|
|
190
|
+
CommandDesign.
|
|
191
|
+
'''
|
|
161
192
|
|
|
162
193
|
command_name: str = ''
|
|
163
194
|
|
|
@@ -170,20 +201,20 @@ class Command:
|
|
|
170
201
|
"keep" : False,
|
|
171
202
|
"force" : False,
|
|
172
203
|
"fake" : False,
|
|
173
|
-
"stop-before-compile": False,
|
|
204
|
+
"stop-before-compile": False,
|
|
174
205
|
"stop-after-compile": False,
|
|
175
|
-
"stop-after-elaborate": False,
|
|
206
|
+
"stop-after-elaborate": False,
|
|
176
207
|
"lint": False, # Same as stop-after-elaborate
|
|
177
208
|
"eda-dir" : "eda.work", # all eda jobs go in here
|
|
178
209
|
"job-name" : "", # this is used to create a certain dir under "eda_dir"
|
|
179
|
-
"work-dir" : "", #
|
|
180
|
-
"sub-work-dir" : "", # this can be used to name the dir built under <eda-dir
|
|
210
|
+
"work-dir" : "", # default is usually <eda-dir>/<job-name> or <eda-dir>/<target>.<cmd>
|
|
211
|
+
"sub-work-dir" : "", # this can be used to name the dir built under <eda-dir>
|
|
181
212
|
"work-dir-use-target-dir": False,
|
|
182
213
|
"suffix" : "",
|
|
183
214
|
"design" : "", # not sure how this relates to top
|
|
184
215
|
'export': False,
|
|
185
|
-
'export-run': False,
|
|
186
|
-
'export-json': False,
|
|
216
|
+
'export-run': False, # run from the exported location if possible
|
|
217
|
+
'export-json': False, # generates an export.json suitable for a testrunner
|
|
187
218
|
'enable-tags': [],
|
|
188
219
|
'disable-tags': [],
|
|
189
220
|
'test-mode': False,
|
|
@@ -214,6 +245,7 @@ class Command:
|
|
|
214
245
|
self.status = 0
|
|
215
246
|
self.errors_log_f = None
|
|
216
247
|
|
|
248
|
+
|
|
217
249
|
def error(self, *args, **kwargs) -> None:
|
|
218
250
|
'''Returns None, child classes can call self.error(..) instead of util.error,
|
|
219
251
|
|
|
@@ -225,10 +257,11 @@ class Command:
|
|
|
225
257
|
if self.args['work-dir']:
|
|
226
258
|
if not self.errors_log_f:
|
|
227
259
|
try:
|
|
228
|
-
self.errors_log_f = open(
|
|
229
|
-
os.path.join(self.args['work-dir'], 'eda.errors.log'), 'w'
|
|
260
|
+
self.errors_log_f = open( # pylint: disable=consider-using-with
|
|
261
|
+
os.path.join(self.args['work-dir'], 'eda.errors.log'), 'w',
|
|
262
|
+
encoding='utf-8'
|
|
230
263
|
)
|
|
231
|
-
except:
|
|
264
|
+
except FileNotFoundError:
|
|
232
265
|
pass
|
|
233
266
|
if self.errors_log_f:
|
|
234
267
|
print(
|
|
@@ -245,17 +278,29 @@ class Command:
|
|
|
245
278
|
util.error(f"command '{self.command_name}' has previous errors")
|
|
246
279
|
return self.status > 0
|
|
247
280
|
|
|
248
|
-
def which_tool(self, command:str):
|
|
281
|
+
def which_tool(self, command:str) -> str:
|
|
282
|
+
'''Returns a str for the tool name used for the requested command'''
|
|
249
283
|
return which_tool(command, config=self.config)
|
|
250
284
|
|
|
251
|
-
def create_work_dir(
|
|
285
|
+
def create_work_dir( # pylint: disable=too-many-branches,too-many-statements
|
|
286
|
+
self
|
|
287
|
+
) -> str:
|
|
288
|
+
'''Creates the working directory and populates self.args['work-dir']
|
|
289
|
+
|
|
290
|
+
Generally uses ./ self.args['eda-dir'] / <target-name>.<command> /
|
|
291
|
+
however, self.args['job-name'] or ['sub-work-dir'] can override that.
|
|
292
|
+
|
|
293
|
+
Additionally, the work-dir is attempted to be deleted if it already exists
|
|
294
|
+
AND it is beneath the caller's original working directory. We also will
|
|
295
|
+
not delete 'work-dir' that is ./ or $PWD.
|
|
296
|
+
'''
|
|
252
297
|
util.debug(f"create_work_dir: {self.args['eda-dir']=} {self.args['work-dir']=}")
|
|
253
298
|
if self.args['work-dir-use-target-dir']:
|
|
254
299
|
if not self.target_path:
|
|
255
300
|
self.target_path = '.'
|
|
256
|
-
util.info(
|
|
257
|
-
|
|
258
|
-
|
|
301
|
+
util.info("create_work_dir: --work-dir-use-target-dir: using:",
|
|
302
|
+
f"{os.path.abspath(self.target_path)}",
|
|
303
|
+
f'target={self.target}')
|
|
259
304
|
self.args['work-dir'] = self.target_path
|
|
260
305
|
self.args['sub-work-dir'] = ''
|
|
261
306
|
return self.args['work-dir']
|
|
@@ -263,28 +308,37 @@ class Command:
|
|
|
263
308
|
if not os.path.exists(self.args['eda-dir']):
|
|
264
309
|
util.safe_mkdir(self.args['eda-dir'])
|
|
265
310
|
util.info(f"create_work_dir: created {self.args['eda-dir']}")
|
|
311
|
+
|
|
266
312
|
if self.args['design'] == "":
|
|
267
313
|
if ('top' in self.args) and (self.args['top'] != ""):
|
|
268
314
|
self.args['design'] = self.args['top']
|
|
269
|
-
util.debug(f"create_work_dir: set {self.args['design']=} from
|
|
315
|
+
util.debug(f"create_work_dir: set {self.args['design']=} from",
|
|
316
|
+
f"{self.args['top']=}, since it was empty")
|
|
270
317
|
else:
|
|
271
318
|
self.args['design'] = "design" # generic, i.e. to create work dir "design_upload"
|
|
272
|
-
util.debug(f"create_work_dir: set {self.args['design']=} to 'design', since it
|
|
319
|
+
util.debug(f"create_work_dir: set {self.args['design']=} to 'design', since it",
|
|
320
|
+
"was empty and we have no top")
|
|
321
|
+
|
|
273
322
|
if self.target == "":
|
|
274
323
|
self.target = self.args['design']
|
|
275
324
|
util.debug(f"create_work_dir: set {self.target=} from design name, since it was empty")
|
|
325
|
+
|
|
276
326
|
if self.args['work-dir'] == '':
|
|
277
327
|
if self.args['sub-work-dir'] == '':
|
|
278
328
|
if self.args['job-name'] != '':
|
|
279
329
|
self.args['sub-work-dir'] = self.args['job-name']
|
|
280
|
-
util.debug(f"create_work_dir: set {self.args['sub-work-dir']=} from
|
|
330
|
+
util.debug(f"create_work_dir: set {self.args['sub-work-dir']=} from",
|
|
331
|
+
f"{self.args['job-name']=}, since it was empty")
|
|
281
332
|
else:
|
|
282
333
|
self.args['sub-work-dir'] = f'{self.target}.{self.command_name}'
|
|
283
|
-
util.debug(f"create_work_dir: set {self.args['sub-work-dir']=} from
|
|
334
|
+
util.debug(f"create_work_dir: set {self.args['sub-work-dir']=} from",
|
|
335
|
+
f"{self.target=} and {self.command_name=}, since it was empty and",
|
|
336
|
+
"we have no job-name")
|
|
284
337
|
self.args['work-dir'] = os.path.join(self.args['eda-dir'], self.args['sub-work-dir'])
|
|
285
338
|
util.debug(f"create_work_dir: set {self.args['work-dir']=}")
|
|
339
|
+
|
|
286
340
|
keep_file = os.path.join(self.args['work-dir'], "eda.keep")
|
|
287
|
-
if
|
|
341
|
+
if os.path.exists(self.args['work-dir']):
|
|
288
342
|
if os.path.exists(keep_file) and not self.args['force']:
|
|
289
343
|
self.error(f"Cannot remove old work dir due to '{keep_file}'")
|
|
290
344
|
elif os.path.abspath(self.args['work-dir']) in os.getcwd():
|
|
@@ -293,16 +347,19 @@ class Command:
|
|
|
293
347
|
# --work-dir=$PWD
|
|
294
348
|
# --work-dir=/some/path/almost/here
|
|
295
349
|
# Allow it, but preserve the existing directory, we don't want to blow away
|
|
296
|
-
# files up-
|
|
350
|
+
# files up-hier from us.
|
|
297
351
|
# Enables support for --work-dir=.
|
|
298
|
-
util.info(f"Not removing existing work-dir: '{self.args['work-dir']}' is within
|
|
352
|
+
util.info(f"Not removing existing work-dir: '{self.args['work-dir']}' is within",
|
|
353
|
+
f"{os.getcwd()=}")
|
|
299
354
|
elif str(Path(self.args['work-dir'])).startswith(str(Path('/'))):
|
|
300
355
|
# Do not allow other absolute path work dirs if it already exists.
|
|
301
356
|
# This prevents you from --work-dir=~ and eda wipes out your home dir.
|
|
302
|
-
self.error(f'Cannot use work-dir={self.args["work-dir"]} starting with
|
|
357
|
+
self.error(f'Cannot use work-dir={self.args["work-dir"]} starting with',
|
|
358
|
+
'fabsolute path "/"')
|
|
303
359
|
elif str(Path('..')) in str(Path(self.args['work-dir'])):
|
|
304
360
|
# Do not allow other ../ work dirs if it already exists.
|
|
305
|
-
self.error(f'Cannot use work-dir={self.args["work-dir"]} with up-hierarchy
|
|
361
|
+
self.error(f'Cannot use work-dir={self.args["work-dir"]} with up-hierarchy'
|
|
362
|
+
'../ paths')
|
|
306
363
|
else:
|
|
307
364
|
# If we made it this far, on a directory that exists, that appears safe
|
|
308
365
|
# to delete and re-create:
|
|
@@ -313,22 +370,32 @@ class Command:
|
|
|
313
370
|
else:
|
|
314
371
|
util.safe_mkdir(self.args['work-dir'])
|
|
315
372
|
util.debug(f'create_work_dir: created {self.args["work-dir"]}')
|
|
316
|
-
|
|
317
|
-
|
|
373
|
+
|
|
374
|
+
if self.args['keep']:
|
|
375
|
+
open(keep_file, 'w', encoding='utf-8').close() # pylint: disable=consider-using-with
|
|
318
376
|
util.debug(f'create_work_dir: created {keep_file=}')
|
|
319
377
|
return self.args['work-dir']
|
|
320
378
|
|
|
321
|
-
|
|
322
|
-
|
|
379
|
+
|
|
380
|
+
def exec(self, work_dir: str, command_list: list, background: bool = False,
|
|
381
|
+
stop_on_error: bool = True, quiet: bool = False, tee_fpath: str = '',
|
|
382
|
+
shell: bool = False) -> (str, str, int):
|
|
383
|
+
'''Runs a command via subprocess and returns a tuple of (stderr, stdout, rc)
|
|
384
|
+
|
|
385
|
+
- work_dir: is passed to the subprocess call, so os.chdir(..) is not invoked.
|
|
386
|
+
- command_list: should be a list of individual strings w/out spaces.
|
|
387
|
+
- background: if False, does not print to stdout (but will print to any
|
|
388
|
+
tee_fpath or the global log
|
|
389
|
+
- shell: arg passed to subprocess, defaults to False.
|
|
390
|
+
'''
|
|
391
|
+
|
|
323
392
|
if not tee_fpath and getattr(command_list, 'tee_fpath', None):
|
|
324
393
|
tee_fpath = getattr(command_list, 'tee_fpath', '')
|
|
325
394
|
if not quiet:
|
|
326
395
|
util.info(f"exec: {' '.join(command_list)} (in {work_dir}, {tee_fpath=})")
|
|
327
|
-
original_cwd = util.getcwd()
|
|
328
|
-
os.chdir(work_dir)
|
|
329
396
|
|
|
330
|
-
stdout, stderr, return_code =
|
|
331
|
-
work_dir=
|
|
397
|
+
stdout, stderr, return_code = subprocess_run_background(
|
|
398
|
+
work_dir=work_dir,
|
|
332
399
|
command_list=command_list,
|
|
333
400
|
background=background,
|
|
334
401
|
fake=self.args.get('fake', False),
|
|
@@ -336,16 +403,23 @@ class Command:
|
|
|
336
403
|
shell=shell
|
|
337
404
|
)
|
|
338
405
|
|
|
339
|
-
os.chdir(original_cwd)
|
|
340
406
|
if return_code:
|
|
341
407
|
self.status += return_code
|
|
342
|
-
if stop_on_error:
|
|
343
|
-
|
|
408
|
+
if stop_on_error:
|
|
409
|
+
self.error(f"exec: returned with error (return code: {return_code})")
|
|
410
|
+
else:
|
|
411
|
+
util.debug(f"exec: returned with error (return code: {return_code})")
|
|
344
412
|
else:
|
|
345
413
|
util.debug(f"exec: returned without error (return code: {return_code})")
|
|
346
414
|
return stderr, stdout, return_code
|
|
347
415
|
|
|
348
|
-
def set_arg(
|
|
416
|
+
def set_arg( # pylint: disable=too-many-branches
|
|
417
|
+
self, key, value
|
|
418
|
+
) -> None:
|
|
419
|
+
'''Sets self.args[key] with value, and self.modified_args[key]=True
|
|
420
|
+
|
|
421
|
+
Does type checking, handles list type append
|
|
422
|
+
'''
|
|
349
423
|
|
|
350
424
|
# Do some minimal type handling, preserving the type(self.args[key])
|
|
351
425
|
if key not in self.args:
|
|
@@ -378,14 +452,14 @@ class Command:
|
|
|
378
452
|
else:
|
|
379
453
|
self.args[key] = True
|
|
380
454
|
else:
|
|
381
|
-
|
|
455
|
+
assert False, f'set_arg({key=}, {value=}) bool {type(cur_value)=} {type(value)=}'
|
|
382
456
|
|
|
383
457
|
elif isinstance(cur_value, int):
|
|
384
458
|
# if int, attempt to convert string or bool --> int
|
|
385
459
|
if isinstance(value, (bool, int, str)):
|
|
386
460
|
self.args[key] = int(value)
|
|
387
461
|
else:
|
|
388
|
-
|
|
462
|
+
assert False, f'set_arg({key=}, {value=}) int {type(cur_value)=} {type(value)=}'
|
|
389
463
|
|
|
390
464
|
|
|
391
465
|
else:
|
|
@@ -396,7 +470,9 @@ class Command:
|
|
|
396
470
|
util.debug(f'Set arg["{key}"]="{self.args[key]}"')
|
|
397
471
|
|
|
398
472
|
|
|
399
|
-
def get_argparser(
|
|
473
|
+
def get_argparser( # pylint: disable=too-many-branches
|
|
474
|
+
self, parser_arg_list=None
|
|
475
|
+
) -> argparse.ArgumentParser:
|
|
400
476
|
''' Returns an argparse.ArgumentParser() based on self.args (dict)
|
|
401
477
|
|
|
402
478
|
If parser_arg_list is not None, the ArgumentParser() is created using only the keys in
|
|
@@ -404,8 +480,8 @@ class Command:
|
|
|
404
480
|
'''
|
|
405
481
|
|
|
406
482
|
# Preference is --args-with-dashes, which then become parsed.args_with_dashes, b/c
|
|
407
|
-
# parsed.args-with-dashes is not legal python. Some of self.args.keys() still have - or _,
|
|
408
|
-
# this will handle both.
|
|
483
|
+
# parsed.args-with-dashes is not legal python. Some of self.args.keys() still have - or _,
|
|
484
|
+
# so this will handle both.
|
|
409
485
|
# Also, preference is for self.args.keys(), to be str with - dashes
|
|
410
486
|
parser = argparse.ArgumentParser(prog='eda', add_help=False, allow_abbrev=False)
|
|
411
487
|
bool_action_kwargs = util.get_argparse_bool_action_kwargs()
|
|
@@ -439,19 +515,19 @@ class Command:
|
|
|
439
515
|
help_kwargs = {'help': f'{type(value).__name__} default={value}'}
|
|
440
516
|
|
|
441
517
|
|
|
442
|
-
# It's important to set the default=None on these, except for list types where default
|
|
443
|
-
# If the parsed Namespace has values set to None or [], we do not update. This
|
|
444
|
-
# are processed that have args set, they cannot override the top
|
|
445
|
-
# nor be overriden by defaults.
|
|
446
|
-
if
|
|
447
|
-
# For bool, support --key and --no-key with
|
|
448
|
-
# Note, this means you cannot use --some-bool=True, or --some-bool=False, has to
|
|
449
|
-
# or --no-some-bool.
|
|
518
|
+
# It's important to set the default=None on these, except for list types where default
|
|
519
|
+
# is []. If the parsed Namespace has values set to None or [], we do not update. This
|
|
520
|
+
# means that as deps are processed that have args set, they cannot override the top
|
|
521
|
+
# level args that were already set, nor be overriden by defaults.
|
|
522
|
+
if isinstance(value, bool):
|
|
523
|
+
# For bool, support --key and --no-key with action=argparse.BooleanOptionalAction.
|
|
524
|
+
# Note, this means you cannot use --some-bool=True, or --some-bool=False, has to
|
|
525
|
+
# be --some-bool or --no-some-bool.
|
|
450
526
|
parser.add_argument(
|
|
451
527
|
*arguments, default=None, **bool_action_kwargs, **help_kwargs)
|
|
452
|
-
elif
|
|
528
|
+
elif isinstance(value, list):
|
|
453
529
|
parser.add_argument(*arguments, default=value, action='append', **help_kwargs)
|
|
454
|
-
elif
|
|
530
|
+
elif isinstance(value, (int, str)):
|
|
455
531
|
parser.add_argument(*arguments, default=value, type=type(value), **help_kwargs)
|
|
456
532
|
elif value is None:
|
|
457
533
|
parser.add_argument(*arguments, default=None, **help_kwargs)
|
|
@@ -462,10 +538,10 @@ class Command:
|
|
|
462
538
|
|
|
463
539
|
|
|
464
540
|
def run_argparser_on_list(
|
|
465
|
-
self, tokens: list, parser_arg_list=None, apply_parsed_args:bool=True
|
|
541
|
+
self, tokens: list, parser_arg_list=None, apply_parsed_args: bool = True
|
|
466
542
|
) -> (dict, list):
|
|
467
|
-
''' Creates an argparse.ArgumentParser() for all the keys in self.args, and attempts to
|
|
468
|
-
from the provided list. Parsed args are applied to self.args.
|
|
543
|
+
''' Creates an argparse.ArgumentParser() for all the keys in self.args, and attempts to
|
|
544
|
+
parse from the provided list. Parsed args are applied to self.args.
|
|
469
545
|
|
|
470
546
|
Returns a tuple of (parsed argparse.Namespace obj, list of unparsed args)
|
|
471
547
|
|
|
@@ -479,17 +555,16 @@ class Command:
|
|
|
479
555
|
parsed, unparsed = parser.parse_known_args(tokens + [''])
|
|
480
556
|
unparsed = list(filter(None, unparsed))
|
|
481
557
|
except argparse.ArgumentError:
|
|
482
|
-
self.error(f'problem {command_name=} attempting to parse_known_args for {tokens=}')
|
|
558
|
+
self.error(f'problem {self.command_name=} attempting to parse_known_args for {tokens=}')
|
|
483
559
|
|
|
484
560
|
parsed_as_dict = vars(parsed)
|
|
485
561
|
|
|
486
562
|
args_to_be_applied = {}
|
|
487
563
|
|
|
488
|
-
for key,value in parsed_as_dict.items():
|
|
489
|
-
# key
|
|
490
|
-
# etc.
|
|
564
|
+
for key, value in parsed_as_dict.items():
|
|
565
|
+
# key currently has _ instead of POSIX dashes, so we convert to dashes, for
|
|
566
|
+
# most self.args, like self.args['build-file'], etc.
|
|
491
567
|
if key not in self.args and '_' in key:
|
|
492
|
-
# try with dashes instead of _
|
|
493
568
|
key = key.replace('_', '-')
|
|
494
569
|
assert key in self.args, f'{key=} not in {self.args=}'
|
|
495
570
|
|
|
@@ -501,7 +576,8 @@ class Command:
|
|
|
501
576
|
return parsed, unparsed
|
|
502
577
|
|
|
503
578
|
|
|
504
|
-
def apply_args_from_dict(self, args_to_be_applied:dict) -> list:
|
|
579
|
+
def apply_args_from_dict(self, args_to_be_applied: dict) -> list:
|
|
580
|
+
'''Given a dict of key/value for self.args, apply them to self.args'''
|
|
505
581
|
util.debug('apply_args_from_dict() -- called by:',
|
|
506
582
|
f'{self.command_name=}, {self.__class__.__name__=},',
|
|
507
583
|
f'{args_to_be_applied=}')
|
|
@@ -523,7 +599,7 @@ class Command:
|
|
|
523
599
|
f"with different cur value (cur value={self.args.get(key, None)})")
|
|
524
600
|
continue
|
|
525
601
|
if self.args[key] != value:
|
|
526
|
-
util.debug(
|
|
602
|
+
util.debug("Command.run_argparser_on_list - setting set_arg b/c",
|
|
527
603
|
f" argparse -- {key=} {value=} (cur value={self.args[key]})")
|
|
528
604
|
self.set_arg(key, value) # Note this has special handling for lists already.
|
|
529
605
|
self.modified_args[key] = True
|
|
@@ -541,7 +617,7 @@ class Command:
|
|
|
541
617
|
_, unparsed = self.run_argparser_on_list(tokens)
|
|
542
618
|
if process_all and len(unparsed) > 0:
|
|
543
619
|
self.error(f"Didn't understand argument: '{unparsed=}' in",
|
|
544
|
-
f" {self.command_name=} context")
|
|
620
|
+
f" {self.command_name=} context, {pwd=}")
|
|
545
621
|
|
|
546
622
|
return unparsed
|
|
547
623
|
|
|
@@ -574,27 +650,33 @@ class Command:
|
|
|
574
650
|
|
|
575
651
|
Allows command handlers to make thier own customizations from --config-yaml=YAML.
|
|
576
652
|
'''
|
|
577
|
-
|
|
653
|
+
return
|
|
578
654
|
|
|
579
655
|
def update_tool_config(self) -> None:
|
|
580
656
|
'''Returns None. Hook for classes like CommandSim to make tool specific overrides.'''
|
|
581
|
-
|
|
657
|
+
return
|
|
582
658
|
|
|
583
659
|
def write_eda_config_and_args(self):
|
|
660
|
+
'''Attempts to write eda_output_config.yml to our work-dir'''
|
|
584
661
|
if not self.args.get('work-dir', None):
|
|
585
662
|
util.warning(f'Ouput work-dir not set, saving ouput eda_config to {os.getcwd()}')
|
|
586
|
-
|
|
587
|
-
|
|
663
|
+
eda_config.write_eda_config_and_args(
|
|
664
|
+
dirpath=self.args.get('work-dir', os.getcwd()), command_obj_ref=self
|
|
665
|
+
)
|
|
588
666
|
|
|
589
667
|
def is_export_enabled(self) -> bool:
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
return any(
|
|
668
|
+
'''Returns True if any self.args['export'] is set in any way (but not set to False
|
|
669
|
+
or empty list)'''
|
|
670
|
+
return any(arg.startswith('export') and v for arg,v in self.args.items())
|
|
593
671
|
|
|
594
672
|
def run(self) -> None:
|
|
673
|
+
'''Alias for do_it(self)'''
|
|
595
674
|
self.do_it()
|
|
596
675
|
|
|
597
676
|
def do_it(self) -> None:
|
|
677
|
+
'''Main entrypoint of running a command, usually called by process_tokens.
|
|
678
|
+
|
|
679
|
+
process_tokens(..) is the starting entrypoint from eda.py'''
|
|
598
680
|
self.write_eda_config_and_args()
|
|
599
681
|
self.error(f"No tool bound to command '{self.command_name}', you",
|
|
600
682
|
" probably need to setup tool, or use '--tool <name>'")
|
|
@@ -610,7 +692,9 @@ class Command:
|
|
|
610
692
|
self.set_tool_defines()
|
|
611
693
|
|
|
612
694
|
|
|
613
|
-
def help(
|
|
695
|
+
def help( # pylint: disable=dangerous-default-value,too-many-branches
|
|
696
|
+
self, tokens: list = []
|
|
697
|
+
) -> None:
|
|
614
698
|
'''Since we don't quite follow standard argparger help()/usage(), we'll format our own
|
|
615
699
|
|
|
616
700
|
if self.args_help has additional help information.
|
|
@@ -618,7 +702,7 @@ class Command:
|
|
|
618
702
|
|
|
619
703
|
# Indent long lines (>100) to indent=56 (this is where we leave off w/ {vstr:12} below.
|
|
620
704
|
def indent_me(text:str):
|
|
621
|
-
return
|
|
705
|
+
return indent_wrap_long_text(text, width=100, indent=56)
|
|
622
706
|
|
|
623
707
|
util.info('Help:')
|
|
624
708
|
# using bare 'print' here, since help was requested, avoids --color and --quiet
|
|
@@ -637,14 +721,14 @@ class Command:
|
|
|
637
721
|
lines.append(f"Generic help for command='{self.command_name}'"
|
|
638
722
|
f" (using '{self.__class__.__name__}')")
|
|
639
723
|
else:
|
|
640
|
-
lines.append(
|
|
724
|
+
lines.append("Generic help (from class Command):")
|
|
641
725
|
|
|
642
726
|
# Attempt to run argparser on args, but don't error if it fails.
|
|
643
727
|
unparsed = []
|
|
644
728
|
if tokens:
|
|
645
729
|
try:
|
|
646
730
|
_, unparsed = self.run_argparser_on_list(tokens=tokens)
|
|
647
|
-
except:
|
|
731
|
+
except Exception:
|
|
648
732
|
pass
|
|
649
733
|
|
|
650
734
|
for k in sorted(self.args.keys()):
|
|
@@ -653,13 +737,13 @@ class Command:
|
|
|
653
737
|
khelp = self.args_help.get(k, '')
|
|
654
738
|
if khelp:
|
|
655
739
|
khelp = f' - {khelp}'
|
|
656
|
-
if
|
|
740
|
+
if isinstance(v, bool):
|
|
657
741
|
lines.append(indent_me(f" --{k:20} : boolean : {vstr:12}{khelp}"))
|
|
658
|
-
elif
|
|
742
|
+
elif isinstance(v, int):
|
|
659
743
|
lines.append(indent_me(f" --{k:20} : integer : {vstr:12}{khelp}"))
|
|
660
|
-
elif
|
|
744
|
+
elif isinstance(v, list):
|
|
661
745
|
lines.append(indent_me(f" --{k:20} : list : {vstr:12}{khelp}"))
|
|
662
|
-
elif
|
|
746
|
+
elif isinstance(v, str):
|
|
663
747
|
vstr = "'" + v + "'"
|
|
664
748
|
lines.append(indent_me(f" --{k:20} : string : {vstr:12}{khelp}"))
|
|
665
749
|
else:
|
|
@@ -672,7 +756,10 @@ class Command:
|
|
|
672
756
|
print(f'Unparsed args: {unparsed}')
|
|
673
757
|
|
|
674
758
|
|
|
675
|
-
class CommandDesign(Command):
|
|
759
|
+
class CommandDesign(Command): # pylint: disable=too-many-instance-attributes
|
|
760
|
+
'''CommandDesign is the eda base class for command handlers that need to track files.
|
|
761
|
+
|
|
762
|
+
This is the base class for CommandSim, CommandSynth, and others.'''
|
|
676
763
|
|
|
677
764
|
# Used by for DEPS work_dir_add_srcs@ commands, by class methods:
|
|
678
765
|
# update_file_lists_for_work_dir(..), and resolve_target(..)
|
|
@@ -693,11 +780,16 @@ class CommandDesign(Command):
|
|
|
693
780
|
self.args_help.update({
|
|
694
781
|
'seed': 'design seed, default is 31-bit non-zero urandom',
|
|
695
782
|
'top': 'TOP level verilog/SV module or VHDL entity for this target',
|
|
696
|
-
'all-sv': (
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
783
|
+
'all-sv': (
|
|
784
|
+
'Maintain .sv and .v in single file list.'
|
|
785
|
+
' False: .sv flist separate from .v flist and separate compile(s)'
|
|
786
|
+
' True: .sv and .v files compiled together if possible'
|
|
787
|
+
),
|
|
788
|
+
'unprocessed-plusargs': (
|
|
789
|
+
'Args that began with +, but were not +define+ or +incdir+, +<name>, '
|
|
790
|
+
' or +<name>=<value>. These become tool dependent, for example "sim" commands will'
|
|
791
|
+
' treat as sim-plusargs'
|
|
792
|
+
),
|
|
701
793
|
})
|
|
702
794
|
self.defines = {}
|
|
703
795
|
self.incdirs = []
|
|
@@ -715,12 +807,18 @@ class CommandDesign(Command):
|
|
|
715
807
|
for (d,v) in self.config.get('defines', {}).items():
|
|
716
808
|
self.defines[d] = v
|
|
717
809
|
|
|
718
|
-
|
|
810
|
+
# cached_deps: key = abspath of DEPS markup file, value is a dict with
|
|
811
|
+
# keys 'data' and 'line_numbers'
|
|
812
|
+
self.cached_deps = {}
|
|
719
813
|
self.targets_dict = {} # key = targets that we've already processed in DEPS files
|
|
720
814
|
self.last_added_source_file_inferred_top = ''
|
|
721
815
|
|
|
722
|
-
|
|
723
|
-
|
|
816
|
+
|
|
817
|
+
def run_dep_commands(self) -> None:
|
|
818
|
+
'''Run shell/peakrdl style commands from DEPS files
|
|
819
|
+
|
|
820
|
+
These are deferred to maintain the deps ordering, and run in that order.
|
|
821
|
+
'''
|
|
724
822
|
self.run_dep_shell_commands()
|
|
725
823
|
# Update any work_dir_add_srcs@ in our self.files, self.files_v, etc, b/c
|
|
726
824
|
# self.args['work-dir'] now exists.
|
|
@@ -728,7 +826,10 @@ class CommandDesign(Command):
|
|
|
728
826
|
# Link any non-sources to our work-dir:
|
|
729
827
|
self.update_non_source_files_in_work_dir()
|
|
730
828
|
|
|
731
|
-
|
|
829
|
+
|
|
830
|
+
def run_dep_shell_commands(self) -> None:
|
|
831
|
+
'''Specifically runs shell command from DEPS files'''
|
|
832
|
+
|
|
732
833
|
# Runs from self.args['work-dir']
|
|
733
834
|
all_cmds_lists = []
|
|
734
835
|
|
|
@@ -747,7 +848,8 @@ class CommandDesign(Command):
|
|
|
747
848
|
log_fnames_count.update({target_node: lognum + 1})
|
|
748
849
|
all_cmds_lists += [
|
|
749
850
|
[], # blank line
|
|
750
|
-
# comment, where it came from, log to {node}__shell_{lognum}.log
|
|
851
|
+
# comment, where it came from, log to {node}__shell_{lognum}.log
|
|
852
|
+
# (or tee name from DEPS.yml)
|
|
751
853
|
[f'# command {i}: target: {d["target_path"]} : {target_node} --> {log}'],
|
|
752
854
|
]
|
|
753
855
|
if not run_from_work_dir:
|
|
@@ -761,10 +863,12 @@ class CommandDesign(Command):
|
|
|
761
863
|
|
|
762
864
|
d['exec_list'] = clist # update to tee_fpath is set.
|
|
763
865
|
|
|
764
|
-
util.write_shell_command_file(
|
|
765
|
-
|
|
866
|
+
util.write_shell_command_file(
|
|
867
|
+
dirpath=self.args['work-dir'], filename='pre_compile_dep_shell_commands.sh',
|
|
868
|
+
command_lists=all_cmds_lists
|
|
869
|
+
)
|
|
766
870
|
|
|
767
|
-
for i,d in enumerate(self.dep_shell_commands):
|
|
871
|
+
for i, d in enumerate(self.dep_shell_commands):
|
|
768
872
|
util.info(f'run_dep_shell_commands {i=}: {d=}')
|
|
769
873
|
clist = util.ShellCommandList(d['exec_list'])
|
|
770
874
|
tee_fpath=clist.tee_fpath
|
|
@@ -782,16 +886,23 @@ class CommandDesign(Command):
|
|
|
782
886
|
self.exec(run_from_dir, clist, tee_fpath=tee_fpath,
|
|
783
887
|
shell=self.config.get('deps_subprocess_shell', False))
|
|
784
888
|
|
|
785
|
-
|
|
786
|
-
|
|
889
|
+
|
|
890
|
+
def update_file_lists_for_work_dir(self) -> None:
|
|
891
|
+
'''Handles any source files that were creating by "shell" style commands in the
|
|
892
|
+
|
|
893
|
+
work-dir, these need to be added via self.add_file(..) in the correct order
|
|
894
|
+
of the dependencies. Basically, any files that were left with @EDA-WORK-DIR@ prefix
|
|
895
|
+
need to be patched and added.
|
|
896
|
+
'''
|
|
897
|
+
if not self.dep_work_dir_add_srcs:
|
|
787
898
|
return
|
|
788
899
|
|
|
789
|
-
# If we encounter any @EDA-WORK_DIR@some_file.v in self.files, self.files_v, etc,
|
|
790
|
-
# self.args['work-dir'] / some_file.v:
|
|
900
|
+
# If we encounter any @EDA-WORK_DIR@some_file.v in self.files, self.files_v, etc,
|
|
901
|
+
# then replace it with: self.args['work-dir'] / some_file.v:
|
|
791
902
|
_work_dir_add_srcs_path_string_len = len(self._work_dir_add_srcs_path_string)
|
|
792
903
|
work_dir_abspath = os.path.abspath(self.args['work-dir'])
|
|
793
|
-
for key in list(self.files.keys()): # list so it's not an iterator,
|
|
794
|
-
if
|
|
904
|
+
for key in list(self.files.keys()): # list so it's not an iterator, updates self.files.
|
|
905
|
+
if isinstance(key, str) and key.startswith(self._work_dir_add_srcs_path_string):
|
|
795
906
|
new_key = os.path.join(work_dir_abspath, key[_work_dir_add_srcs_path_string_len :])
|
|
796
907
|
self.files.pop(key)
|
|
797
908
|
self.files[new_key] = True
|
|
@@ -800,12 +911,23 @@ class CommandDesign(Command):
|
|
|
800
911
|
self.files_sdc]
|
|
801
912
|
for my_file_list in my_file_lists_list:
|
|
802
913
|
for i,value in enumerate(my_file_list):
|
|
803
|
-
if value and
|
|
804
|
-
|
|
914
|
+
if value and isinstance(value, str) and \
|
|
915
|
+
value.startswith(self._work_dir_add_srcs_path_string):
|
|
916
|
+
new_value = os.path.join(
|
|
917
|
+
work_dir_abspath, value[_work_dir_add_srcs_path_string_len :]
|
|
918
|
+
)
|
|
805
919
|
my_file_list[i] = new_value
|
|
806
920
|
util.debug(f"file lists: replaced {value} with {new_value}")
|
|
807
921
|
|
|
808
|
-
|
|
922
|
+
|
|
923
|
+
def update_non_source_files_in_work_dir(self) -> None:
|
|
924
|
+
'''For non-source files (that are tracked as 'reqs' in DEPS markup files) these
|
|
925
|
+
|
|
926
|
+
need to be copied or linked to the work-dir. For example, if some SV assumes it
|
|
927
|
+
can $readmemh('file_that_is_here.txt') but we're running out of work-dir. Linking
|
|
928
|
+
is the easy work-around vs trying to run-in-place of all SV files.
|
|
929
|
+
'''
|
|
930
|
+
|
|
809
931
|
for fname in self.files_non_source:
|
|
810
932
|
_, leaf_fname = os.path.split(fname)
|
|
811
933
|
destfile = os.path.join(self.args['work-dir'], leaf_fname)
|
|
@@ -815,16 +937,25 @@ class CommandDesign(Command):
|
|
|
815
937
|
util.info(f'{fname=} {self.files_caller_info=}')
|
|
816
938
|
self.error(f'Non-source file (reqs?) {relfname=} does not exist from {caller_info}')
|
|
817
939
|
elif not os.path.exists(destfile):
|
|
818
|
-
util.debug(f'updating non-source file to work-dir: Linked {fname=} to {destfile=},
|
|
940
|
+
util.debug(f'updating non-source file to work-dir: Linked {fname=} to {destfile=},',
|
|
941
|
+
f'from {caller_info}')
|
|
819
942
|
if sys.platform == "win32":
|
|
820
943
|
shutil.copyfile(fname, destfile) # On Windows, fall back to copying
|
|
821
944
|
else:
|
|
822
945
|
os.symlink(src=fname, dst=destfile)
|
|
823
946
|
|
|
824
|
-
|
|
947
|
+
@staticmethod
|
|
948
|
+
def get_top_name(name: str) -> str:
|
|
949
|
+
'''Attempt to get the 'top' module name from a file, such as path/to/mine.sv will
|
|
950
|
+
|
|
951
|
+
return "mine"'''
|
|
952
|
+
# TODO(drew): Use the helper method in util for this instead to peek in file contents?
|
|
825
953
|
return os.path.splitext(os.path.basename(name))[0]
|
|
826
954
|
|
|
827
|
-
|
|
955
|
+
|
|
956
|
+
def process_plusarg( # pylint: disable=too-many-branches
|
|
957
|
+
self, plusarg: str, pwd: str = os.getcwd()
|
|
958
|
+
) -> None:
|
|
828
959
|
'''Retuns str, parses a +define+, +incdir+, +key=value str; adds to internal.
|
|
829
960
|
|
|
830
961
|
Adds to self.defines, self.incdirs,
|
|
@@ -836,12 +967,12 @@ class CommandDesign(Command):
|
|
|
836
967
|
# args that come from shlex.quote(token), such as:
|
|
837
968
|
# token = '\'+define+OC_ROOT="/foo/bar/opencos"\''
|
|
838
969
|
# So we strip all outer ' or " on the plusarg:
|
|
839
|
-
plusarg =
|
|
970
|
+
plusarg = strip_outer_quotes(plusarg)
|
|
840
971
|
if not pwd:
|
|
841
972
|
pwd = ''
|
|
842
973
|
|
|
843
974
|
if plusarg.startswith('+define+'):
|
|
844
|
-
plusarg = plusarg
|
|
975
|
+
plusarg = plusarg[len('+define+'):]
|
|
845
976
|
m = re.match(r'^(\w+)$', plusarg)
|
|
846
977
|
if m:
|
|
847
978
|
k = m.group(1)
|
|
@@ -849,11 +980,12 @@ class CommandDesign(Command):
|
|
|
849
980
|
util.debug(f"Defined {k}")
|
|
850
981
|
return None
|
|
851
982
|
m = re.match(r'^(\w+)\=(\S+)$', plusarg)
|
|
852
|
-
if not m:
|
|
983
|
+
if not m:
|
|
984
|
+
m = re.match(r'^(\w+)\=(\"[^\"]*\")$', plusarg)
|
|
853
985
|
if m:
|
|
854
986
|
k = m.group(1)
|
|
855
987
|
v = m.group(2)
|
|
856
|
-
if v and
|
|
988
|
+
if v and isinstance(v, str):
|
|
857
989
|
if v.startswith('%PWD%/'):
|
|
858
990
|
v = v.replace('%PWD%', os.path.abspath(pwd))
|
|
859
991
|
if v.startswith('%SEED%'):
|
|
@@ -865,7 +997,7 @@ class CommandDesign(Command):
|
|
|
865
997
|
return None
|
|
866
998
|
|
|
867
999
|
if plusarg.startswith('+incdir+'):
|
|
868
|
-
plusarg = plusarg
|
|
1000
|
+
plusarg = plusarg[len('+incdir+'):]
|
|
869
1001
|
m = re.match(r'^(\S+)$', plusarg)
|
|
870
1002
|
if m:
|
|
871
1003
|
incdir = m.group(1)
|
|
@@ -881,40 +1013,59 @@ class CommandDesign(Command):
|
|
|
881
1013
|
if not self.config.get('bare_plusarg_supported', False):
|
|
882
1014
|
self.error(f"bare plusarg(s) are not supported: {plusarg}'")
|
|
883
1015
|
return None
|
|
884
|
-
|
|
1016
|
+
if plusarg not in self.args['unprocessed-plusargs']:
|
|
885
1017
|
self.args['unprocessed-plusargs'].append(plusarg)
|
|
886
1018
|
# For anything added to unprocessed-plusarg, we have to return it, to let
|
|
887
1019
|
# derived classes have the option to handle it
|
|
888
1020
|
return plusarg
|
|
889
|
-
|
|
890
|
-
|
|
1021
|
+
|
|
1022
|
+
self.error(f"Didn't understand +plusarg: '{plusarg}'")
|
|
891
1023
|
return None
|
|
892
1024
|
|
|
893
1025
|
|
|
894
|
-
def append_shell_commands(self, cmds : list):
|
|
895
|
-
|
|
1026
|
+
def append_shell_commands(self, cmds : list) -> None:
|
|
1027
|
+
''' Given a cmds (list), where each item is a dict with:
|
|
1028
|
+
|
|
1029
|
+
{ 'target_node': str,
|
|
1030
|
+
'target_path': str,
|
|
1031
|
+
'exec_list': list
|
|
1032
|
+
}
|
|
1033
|
+
add to this class's dep_shell_commands for deferred processing
|
|
1034
|
+
'''
|
|
1035
|
+
|
|
896
1036
|
for entry in cmds:
|
|
897
|
-
if entry is None or
|
|
1037
|
+
if entry is None or not isinstance(entry, dict):
|
|
898
1038
|
continue
|
|
899
1039
|
if entry in self.dep_shell_commands:
|
|
900
|
-
# we've already run this exact command (target node, target path, exec list),
|
|
901
|
-
# again
|
|
1040
|
+
# we've already run this exact command (target node, target path, exec list),
|
|
1041
|
+
# don't run it again
|
|
902
1042
|
continue
|
|
903
1043
|
|
|
904
1044
|
assert 'exec_list' in entry, f'{entry=}'
|
|
905
1045
|
util.debug(f'adding - dep_shell_command: {entry=}')
|
|
906
1046
|
self.dep_shell_commands.append(entry)
|
|
907
1047
|
|
|
908
|
-
|
|
909
|
-
|
|
1048
|
+
|
|
1049
|
+
def append_work_dir_add_srcs(self, add_srcs: list, caller_info: str) -> None:
|
|
1050
|
+
'''Given add_srcs (list), where each item is a dict with:
|
|
1051
|
+
|
|
1052
|
+
{ 'target_node': str,
|
|
1053
|
+
'target_path': str,
|
|
1054
|
+
'file_list': list
|
|
1055
|
+
}
|
|
1056
|
+
|
|
1057
|
+
adds files to set self.dep_work_dir_add_src, and call add_file on it, basically
|
|
1058
|
+
resolving adding source files that were generated and residing in the work-dir.
|
|
1059
|
+
'''
|
|
910
1060
|
for entry in add_srcs:
|
|
911
|
-
if entry is None or
|
|
1061
|
+
if entry is None or not isinstance(entry, dict):
|
|
912
1062
|
continue
|
|
913
1063
|
|
|
914
1064
|
work_dir_files = entry['file_list']
|
|
915
1065
|
for filename in work_dir_files:
|
|
916
|
-
# Unfortunately, self.args['work-dir'] doesn't exist yet and hasn't been set,
|
|
917
|
-
# files as '@EDA-WORK_DIR@' + filename, and have to replace
|
|
1066
|
+
# Unfortunately, self.args['work-dir'] doesn't exist yet and hasn't been set,
|
|
1067
|
+
# so we'll add these files as '@EDA-WORK_DIR@' + filename, and have to replace
|
|
1068
|
+
# the EDA-WORK_DIR@ string later in our flow.
|
|
918
1069
|
filename_use = self._work_dir_add_srcs_path_string + filename
|
|
919
1070
|
dep_key_tuple = (
|
|
920
1071
|
entry['target_path'],
|
|
@@ -929,14 +1080,25 @@ class CommandDesign(Command):
|
|
|
929
1080
|
self.dep_work_dir_add_srcs.add(dep_key_tuple) # add to set()
|
|
930
1081
|
elif dep_key_tuple not in self.dep_work_dir_add_srcs:
|
|
931
1082
|
# we've already added the file so this dep was skipped for this one file.
|
|
932
|
-
util.warning(f'work_dir_add_srcs@ {dep_key_tuple=} but {filename_use=}'
|
|
933
|
-
|
|
1083
|
+
util.warning(f'work_dir_add_srcs@ {dep_key_tuple=} but {filename_use=}',
|
|
1084
|
+
'is already in self.files (duplicate dependency on generated',
|
|
1085
|
+
'file?)')
|
|
1086
|
+
|
|
934
1087
|
|
|
1088
|
+
def resolve_target(
|
|
1089
|
+
self, target: str, no_recursion: bool = False, caller_info: str = ''
|
|
1090
|
+
) -> bool:
|
|
1091
|
+
'''Returns True if target is found. Entry point for resolving a CLI path based target name.
|
|
935
1092
|
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
1093
|
+
Will recursively call resolve_target_core, looking for a filename existing
|
|
1094
|
+
or DEPS markup file in that path and a target (key) in the markup table.
|
|
1095
|
+
|
|
1096
|
+
This will populate all source file dependencies, commands, and other DEPS markup
|
|
1097
|
+
features as the target is resolved.
|
|
1098
|
+
'''
|
|
1099
|
+
util.debug(f"Entered resolve_target({target})")
|
|
1100
|
+
# self.target is a name we grab for the job (i.e. for naming work dir etc). we don't want
|
|
1101
|
+
# the path prefix.
|
|
940
1102
|
|
|
941
1103
|
self.target_path, self.target = os.path.split(target)
|
|
942
1104
|
|
|
@@ -950,7 +1112,7 @@ class CommandDesign(Command):
|
|
|
950
1112
|
# If the target is a file (we're at the root here processing CLI arg tokens)
|
|
951
1113
|
# and that file exists and has an extension, then there's no reason to go looking
|
|
952
1114
|
# in DEPS files, add the file and return True.
|
|
953
|
-
|
|
1115
|
+
_, file_ext = os.path.splitext(fpath)
|
|
954
1116
|
if forced_extension or file_ext:
|
|
955
1117
|
self.add_file(fpath, caller_info=caller_info,
|
|
956
1118
|
forced_extension=forced_extension)
|
|
@@ -958,23 +1120,25 @@ class CommandDesign(Command):
|
|
|
958
1120
|
|
|
959
1121
|
return self.resolve_target_core(target, no_recursion, caller_info)
|
|
960
1122
|
|
|
961
|
-
def resolve_target_core(
|
|
962
|
-
|
|
1123
|
+
def resolve_target_core( # pylint: disable=too-many-locals,too-many-branches
|
|
1124
|
+
self, target: str, no_recursion: bool, caller_info: str = ''
|
|
1125
|
+
) -> bool:
|
|
1126
|
+
'''Returns True if target is found. recursive point for resolving path or DEPS markup
|
|
1127
|
+
target names.'''
|
|
1128
|
+
|
|
1129
|
+
util.debug(f"Entered resolve_target_core({target=})")
|
|
963
1130
|
found_target = False
|
|
964
|
-
util.debug("Starting to resolve target '%s'" % (target))
|
|
965
1131
|
target_path, target_node = os.path.split(target)
|
|
966
1132
|
|
|
967
|
-
deps = None
|
|
968
|
-
data = None
|
|
1133
|
+
deps, data, deps_file = None, None, None
|
|
969
1134
|
found_deps_file = False
|
|
970
1135
|
|
|
971
1136
|
if self.config['deps_markup_supported']:
|
|
972
|
-
deps =
|
|
1137
|
+
deps = DepsFile(
|
|
973
1138
|
command_design_ref=self, target_path=target_path, cache=self.cached_deps
|
|
974
1139
|
)
|
|
975
1140
|
deps_file = deps.deps_file
|
|
976
1141
|
data = deps.data
|
|
977
|
-
data_only_lines = deps.line_numbers
|
|
978
1142
|
|
|
979
1143
|
# Continue if we have data, otherwise look for files other than DEPS.<yml|yaml>
|
|
980
1144
|
if data is not None:
|
|
@@ -987,7 +1151,7 @@ class CommandDesign(Command):
|
|
|
987
1151
|
|
|
988
1152
|
# For convenience, use an external class for this DEPS.yml table/dict
|
|
989
1153
|
# This could be re-used for any markup DEPS.json, DEPS.toml, DEPS.py, etc.
|
|
990
|
-
deps_processor =
|
|
1154
|
+
deps_processor = DepsProcessor(
|
|
991
1155
|
command_design_ref = self,
|
|
992
1156
|
deps_entry = entry,
|
|
993
1157
|
target = target,
|
|
@@ -1008,10 +1172,11 @@ class CommandDesign(Command):
|
|
|
1008
1172
|
# Recurse on the returned deps (ordered list), if they haven't already been traversed.
|
|
1009
1173
|
for x in deps_targets_to_resolve:
|
|
1010
1174
|
caller_info = deps.gen_caller_info(target_node)
|
|
1011
|
-
if x and
|
|
1175
|
+
if x and isinstance(x, tuple):
|
|
1012
1176
|
# if deps_processor.process_deps_entry() gave us a tuple, it's an
|
|
1013
1177
|
# unprocessed 'command' that we kept in order until now. Append it.
|
|
1014
|
-
assert len(x) == 2,
|
|
1178
|
+
assert len(x) == 2, \
|
|
1179
|
+
f'command tuple {x=} must be len 2, {target_node=} {deps_file=}'
|
|
1015
1180
|
shell_commands_list, work_dir_add_srcs_list = x
|
|
1016
1181
|
self.append_shell_commands( cmds=shell_commands_list )
|
|
1017
1182
|
self.append_work_dir_add_srcs( add_srcs=work_dir_add_srcs_list,
|
|
@@ -1025,32 +1190,40 @@ class CommandDesign(Command):
|
|
|
1025
1190
|
forced_extension=forced_extension)
|
|
1026
1191
|
else:
|
|
1027
1192
|
util.debug(f' ... Calling resolve_target_core({x=})')
|
|
1028
|
-
found_target |= self.resolve_target_core(
|
|
1193
|
+
found_target |= self.resolve_target_core(
|
|
1194
|
+
x, no_recursion, caller_info=caller_info
|
|
1195
|
+
)
|
|
1029
1196
|
|
|
1030
1197
|
|
|
1031
1198
|
# Done with DEPS.yml if it existed.
|
|
1032
1199
|
|
|
1033
1200
|
if not found_target:
|
|
1034
|
-
util.debug("Haven't been able to resolve
|
|
1201
|
+
util.debug(f"Haven't been able to resolve {target=} via DEPS")
|
|
1035
1202
|
known_file_extensions_for_source = []
|
|
1036
|
-
for x in
|
|
1037
|
-
known_file_extensions_for_source += self.config.get(
|
|
1203
|
+
for x in ('verilog', 'systemverilog', 'vhdl', 'cpp'):
|
|
1204
|
+
known_file_extensions_for_source += self.config.get(
|
|
1205
|
+
'file_extensions', {}).get(x, [])
|
|
1038
1206
|
for e in known_file_extensions_for_source:
|
|
1039
1207
|
try_file = target + e
|
|
1040
|
-
util.debug("Looking for
|
|
1208
|
+
util.debug(f"Looking for file {try_file}")
|
|
1041
1209
|
if os.path.exists(try_file):
|
|
1042
|
-
self.add_file(try_file, caller_info=
|
|
1210
|
+
self.add_file(try_file, caller_info=f'n/a::{target}::n/a')
|
|
1043
1211
|
found_target = True
|
|
1044
1212
|
break # move on to the next target
|
|
1045
1213
|
if not found_target: # if STILL not found_this_target...
|
|
1046
|
-
self.error("Unable to resolve target
|
|
1214
|
+
self.error(f"Unable to resolve {target=}")
|
|
1047
1215
|
|
|
1048
1216
|
# if we've found any target since being called, it means we found the one we were called for
|
|
1049
1217
|
return found_target
|
|
1050
1218
|
|
|
1051
|
-
def add_file(
|
|
1052
|
-
|
|
1053
|
-
|
|
1219
|
+
def add_file(
|
|
1220
|
+
self, filename: str, use_abspath: bool = True, add_to_non_sources: bool = False,
|
|
1221
|
+
caller_info: str = '', forced_extension: str = ''
|
|
1222
|
+
) -> str:
|
|
1223
|
+
'''Given a filename, add it to one of self.files_sv or similar lists
|
|
1224
|
+
|
|
1225
|
+
based on file extension or prefix directive.'''
|
|
1226
|
+
_, file_ext = os.path.splitext(filename)
|
|
1054
1227
|
if use_abspath:
|
|
1055
1228
|
file_abspath = os.path.abspath(filename)
|
|
1056
1229
|
else:
|
|
@@ -1058,8 +1231,8 @@ class CommandDesign(Command):
|
|
|
1058
1231
|
|
|
1059
1232
|
|
|
1060
1233
|
if file_abspath in self.files:
|
|
1061
|
-
util.debug("Not adding file
|
|
1062
|
-
return
|
|
1234
|
+
util.debug(f"Not adding file {file_abspath}, already have it")
|
|
1235
|
+
return ''
|
|
1063
1236
|
|
|
1064
1237
|
known_file_ext_dict = self.config.get('file_extensions', {})
|
|
1065
1238
|
v_file_ext_list = known_file_ext_dict.get('verilog', [])
|
|
@@ -1082,34 +1255,43 @@ class CommandDesign(Command):
|
|
|
1082
1255
|
|
|
1083
1256
|
if add_to_non_sources:
|
|
1084
1257
|
self.files_non_source.append(file_abspath)
|
|
1085
|
-
util.debug("Added non-source file file
|
|
1258
|
+
util.debug(f"Added non-source file file {filename} as {file_abspath}")
|
|
1086
1259
|
elif file_ext in v_file_ext_list and not self.args['all-sv']:
|
|
1087
1260
|
self.files_v.append(file_abspath)
|
|
1088
|
-
util.debug("Added Verilog file
|
|
1089
|
-
elif file_ext in sv_file_ext_list or
|
|
1261
|
+
util.debug(f"Added Verilog file {filename} as {file_abspath}")
|
|
1262
|
+
elif file_ext in sv_file_ext_list or \
|
|
1263
|
+
((file_ext in v_file_ext_list) and self.args['all-sv']):
|
|
1090
1264
|
self.files_sv.append(file_abspath)
|
|
1091
|
-
util.debug("Added SystemVerilog file
|
|
1265
|
+
util.debug(f"Added SystemVerilog file {filename} as {file_abspath}")
|
|
1092
1266
|
elif file_ext in vhdl_file_ext_list:
|
|
1093
1267
|
self.files_vhd.append(file_abspath)
|
|
1094
|
-
util.debug("Added VHDL file
|
|
1268
|
+
util.debug(f"Added VHDL file {filename} as {file_abspath}")
|
|
1095
1269
|
elif file_ext in cpp_file_ext_list:
|
|
1096
1270
|
self.files_cpp.append(file_abspath)
|
|
1097
|
-
util.debug("Added C++ file
|
|
1271
|
+
util.debug(f"Added C++ file {filename} as {file_abspath}")
|
|
1098
1272
|
elif file_ext in sdc_file_ext_list:
|
|
1099
1273
|
self.files_sdc.append(file_abspath)
|
|
1100
|
-
util.debug("Added SDC file
|
|
1274
|
+
util.debug(f"Added SDC file {filename} as {file_abspath}")
|
|
1101
1275
|
else:
|
|
1102
1276
|
# unknown file extension. In these cases we link the file to the working directory
|
|
1103
|
-
# so it is available (for example, a .mem file that is expected to exist with relative
|
|
1277
|
+
# so it is available (for example, a .mem file that is expected to exist with relative
|
|
1278
|
+
# path)
|
|
1104
1279
|
self.files_non_source.append(file_abspath)
|
|
1105
|
-
util.debug("Added non-source file
|
|
1280
|
+
util.debug(f"Added non-source file {filename} as {file_abspath}")
|
|
1106
1281
|
|
|
1107
1282
|
self.files[file_abspath] = True
|
|
1108
1283
|
self.files_caller_info[file_abspath] = caller_info
|
|
1109
1284
|
return file_abspath
|
|
1110
1285
|
|
|
1111
|
-
def process_tokens(
|
|
1112
|
-
|
|
1286
|
+
def process_tokens( # pylint: disable=too-many-locals,too-many-branches,too-many-statements
|
|
1287
|
+
self, tokens: list, process_all: bool = True,
|
|
1288
|
+
pwd: str = os.getcwd()
|
|
1289
|
+
) -> list:
|
|
1290
|
+
'''Returns a list of unparsed args (self.args did not have these keys present)
|
|
1291
|
+
|
|
1292
|
+
Entrypoint from eda.py calling a command handler obj.process_tokens(args).
|
|
1293
|
+
Dervied classes are expected to call this parent process_tokens to parse and handle
|
|
1294
|
+
their self.args dict'''
|
|
1113
1295
|
|
|
1114
1296
|
util.debug(f'CommandDesign - process_tokens start - {tokens=}')
|
|
1115
1297
|
|
|
@@ -1123,19 +1305,21 @@ class CommandDesign(Command):
|
|
|
1123
1305
|
# walk the list, remove all items after we're done.
|
|
1124
1306
|
remove_list = []
|
|
1125
1307
|
for token in unparsed:
|
|
1126
|
-
# Since this is a raw argparser, we may have args that come from shlex.quote(token),
|
|
1127
|
-
#
|
|
1128
|
-
#
|
|
1308
|
+
# Since this is a raw argparser, we may have args that come from shlex.quote(token),
|
|
1309
|
+
# such as:
|
|
1310
|
+
# token = '\'+define+OC_ROOT="/foo/bar/opencos"\''
|
|
1311
|
+
# So we have to check for strings that have been escaped for shell with extra single
|
|
1312
|
+
# quotes.
|
|
1129
1313
|
m = re.match(r"^\'?\+\w+", token)
|
|
1130
1314
|
if m:
|
|
1131
1315
|
# Copy and strip all outer ' or " on the plusarg:
|
|
1132
|
-
plusarg =
|
|
1316
|
+
plusarg = strip_outer_quotes(token)
|
|
1133
1317
|
self.process_plusarg(plusarg, pwd=pwd)
|
|
1134
1318
|
remove_list.append(token)
|
|
1135
1319
|
for x in remove_list:
|
|
1136
1320
|
unparsed.remove(x)
|
|
1137
1321
|
|
|
1138
|
-
if
|
|
1322
|
+
if not unparsed and self.error_on_no_files_or_targets:
|
|
1139
1323
|
# derived classes can set error_on_no_files_or_targets=True
|
|
1140
1324
|
# For example: CommandSim will error (requires files/targets),
|
|
1141
1325
|
# CommandWaves does not (files/targets not required)
|
|
@@ -1143,11 +1327,11 @@ class CommandDesign(Command):
|
|
|
1143
1327
|
# check the DEPS markup file for a single target, and if so run it
|
|
1144
1328
|
# on the one and only target.
|
|
1145
1329
|
if self.config['deps_markup_supported']:
|
|
1146
|
-
deps =
|
|
1330
|
+
deps = DepsFile(
|
|
1147
1331
|
command_design_ref=self, target_path=os.getcwd(), cache=self.cached_deps
|
|
1148
1332
|
)
|
|
1149
1333
|
if deps.deps_file and deps.data:
|
|
1150
|
-
all_targets =
|
|
1334
|
+
all_targets = deps_data_get_all_targets(deps.data)
|
|
1151
1335
|
if all_targets:
|
|
1152
1336
|
target = all_targets[-1]
|
|
1153
1337
|
unparsed.append(target)
|
|
@@ -1169,7 +1353,7 @@ class CommandDesign(Command):
|
|
|
1169
1353
|
file_exists, fpath, forced_extension = files.get_source_file(token)
|
|
1170
1354
|
if file_exists:
|
|
1171
1355
|
file_abspath = os.path.abspath(fpath)
|
|
1172
|
-
|
|
1356
|
+
_, file_ext = os.path.splitext(file_abspath)
|
|
1173
1357
|
if not forced_extension and not file_ext:
|
|
1174
1358
|
# This probably isn't a file we want to use
|
|
1175
1359
|
util.warning(f'looking for deps {token=}, found {file_abspath=}' \
|
|
@@ -1189,11 +1373,12 @@ class CommandDesign(Command):
|
|
|
1189
1373
|
remove_list.append(token)
|
|
1190
1374
|
continue # done with token, consume it, we added the file.
|
|
1191
1375
|
|
|
1192
|
-
# we appear to be dealing with a target name which needs to be resolved (usually
|
|
1376
|
+
# we appear to be dealing with a target name which needs to be resolved (usually
|
|
1377
|
+
# recursively)
|
|
1193
1378
|
if token.startswith(os.sep):
|
|
1194
1379
|
target_name = token # if it's absolute path, don't prepend anything
|
|
1195
1380
|
else:
|
|
1196
|
-
target_name = os.path.join(".", token) # prepend ./
|
|
1381
|
+
target_name = os.path.join(".", token) # prepend ./ to make it like: <path>/<file>
|
|
1197
1382
|
|
|
1198
1383
|
util.debug(f'Calling self.resolve_target on {target_name=} ({token=})')
|
|
1199
1384
|
if self.resolve_target(target_name, caller_info=caller_info):
|
|
@@ -1242,49 +1427,81 @@ class CommandDesign(Command):
|
|
|
1242
1427
|
self.target = self.args['top']
|
|
1243
1428
|
|
|
1244
1429
|
if self.error_on_missing_top and not self.args.get('top', ''):
|
|
1245
|
-
self.error(
|
|
1430
|
+
self.error("Did not get a --top or DEPS top, required to run command",
|
|
1246
1431
|
f"'{self.command_name}' for tool={self.args.get('tool', None)}")
|
|
1247
1432
|
|
|
1248
1433
|
return unparsed
|
|
1249
1434
|
|
|
1250
1435
|
|
|
1251
|
-
def get_command_line_args(
|
|
1436
|
+
def get_command_line_args( # pylint: disable=dangerous-default-value
|
|
1437
|
+
self, remove_args: list = [], remove_args_startswith: list = []
|
|
1438
|
+
) -> list:
|
|
1252
1439
|
'''Returns a list of all the args if you wanted to re-run this command
|
|
1253
1440
|
(excludes eda, command, target).'''
|
|
1254
1441
|
|
|
1255
1442
|
# This will not set bool's that are False, does not add --no-<somearg>
|
|
1256
|
-
# nor --<somearg>=False
|
|
1257
|
-
# This will not set str's that are empty
|
|
1258
|
-
# this will not set ints that are 0
|
|
1443
|
+
# nor --<somearg>=False (it's unclear which are 'store_true' v/ bool-action-kwargs)
|
|
1444
|
+
# This will not set str's that are empty
|
|
1445
|
+
# this will not set ints that are 0 (unless modified)
|
|
1446
|
+
# TODO(drew): we may want to revisit this if we tracked default arg values,
|
|
1447
|
+
# or perhaps if they have been "modified".
|
|
1259
1448
|
ret = []
|
|
1260
1449
|
for k,v in self.args.items():
|
|
1261
1450
|
|
|
1262
1451
|
# Some args cannot be extracted and work, so omit these:
|
|
1263
1452
|
if k in ['top-path'] + remove_args:
|
|
1264
1453
|
continue
|
|
1265
|
-
if any(
|
|
1454
|
+
if any(k.startswith(x) for x in remove_args_startswith):
|
|
1266
1455
|
continue
|
|
1267
1456
|
|
|
1268
|
-
|
|
1457
|
+
is_modified = self.modified_args.get(k, False)
|
|
1458
|
+
|
|
1459
|
+
if isinstance(v, bool) and v:
|
|
1269
1460
|
ret.append(f'--{k}')
|
|
1270
|
-
elif
|
|
1461
|
+
elif isinstance(v, int) and (bool(v) or is_modified):
|
|
1271
1462
|
ret.append(f'--{k}={v}')
|
|
1272
|
-
elif
|
|
1463
|
+
elif isinstance(v, str) and v:
|
|
1273
1464
|
ret.append(f'--{k}={v}')
|
|
1274
|
-
elif
|
|
1465
|
+
elif isinstance(v, list):
|
|
1275
1466
|
for item in v:
|
|
1276
|
-
if item or
|
|
1467
|
+
if item or isinstance(item, (bool, str)):
|
|
1277
1468
|
# don't print bool/str that are blank.
|
|
1278
1469
|
ret.append(f'--{k}={item}') # lists append
|
|
1279
1470
|
|
|
1280
1471
|
return ret
|
|
1281
1472
|
|
|
1282
1473
|
|
|
1283
|
-
|
|
1284
|
-
|
|
1474
|
+
#_THREADS_START = 0
|
|
1475
|
+
#_THREADS_DONE = 0
|
|
1476
|
+
|
|
1477
|
+
class ThreadStats:
|
|
1478
|
+
'''To avoid globals for two ints, keep a class holder so CommandParallel and
|
|
1479
|
+
CommandParallelWorker can share values'''
|
|
1480
|
+
|
|
1481
|
+
done = 0
|
|
1482
|
+
started = 0
|
|
1483
|
+
|
|
1484
|
+
def all_done(self) -> bool:
|
|
1485
|
+
'''Returns True if all started jobs are done'''
|
|
1486
|
+
return self.started == self.done
|
|
1487
|
+
|
|
1488
|
+
def get_remaining(self) -> int:
|
|
1489
|
+
'''Returns remaining number of jobs'''
|
|
1490
|
+
return max(self.started - self.done, 0)
|
|
1491
|
+
|
|
1492
|
+
def __repr__(self) -> str:
|
|
1493
|
+
'''Pretty print DONE/STARTED'''
|
|
1494
|
+
return f'{self.done}/{self.started}'
|
|
1495
|
+
|
|
1285
1496
|
|
|
1286
1497
|
class CommandParallelWorker(threading.Thread):
|
|
1287
|
-
|
|
1498
|
+
'''Helper class for a single threaded job, often run by a CommandParallel command
|
|
1499
|
+
handler class'''
|
|
1500
|
+
|
|
1501
|
+
def __init__(
|
|
1502
|
+
self, n, work_queue: queue.Queue, done_queue: queue.Queue,
|
|
1503
|
+
threads_stats: ThreadStats
|
|
1504
|
+
):
|
|
1288
1505
|
threading.Thread.__init__(self)
|
|
1289
1506
|
self.n = n
|
|
1290
1507
|
self.work_queue = work_queue
|
|
@@ -1294,54 +1511,46 @@ class CommandParallelWorker(threading.Thread):
|
|
|
1294
1511
|
self.proc = None
|
|
1295
1512
|
self.pid = None
|
|
1296
1513
|
self.last_timer_debug = 0
|
|
1514
|
+
self.threads_stats = threads_stats # ref to shared object
|
|
1297
1515
|
util.debug(f"WORKER_{n}: START")
|
|
1298
1516
|
|
|
1299
1517
|
def run(self):
|
|
1300
|
-
|
|
1301
|
-
|
|
1518
|
+
'''Runs this single job via threading. This is typically created and called by
|
|
1519
|
+
|
|
1520
|
+
a CommandParallel class handler
|
|
1521
|
+
'''
|
|
1522
|
+
|
|
1302
1523
|
while True:
|
|
1303
1524
|
# Get the work from the queue and expand the tuple
|
|
1304
|
-
i, command_list, job_name,
|
|
1525
|
+
i, command_list, job_name, _ = self.work_queue.get()
|
|
1305
1526
|
self.job_name = job_name
|
|
1306
1527
|
try:
|
|
1307
1528
|
util.debug(f"WORKER_{self.n}: Running job {i}: {job_name}")
|
|
1308
|
-
PIPE=subprocess.PIPE
|
|
1309
|
-
STDOUT=subprocess.STDOUT
|
|
1310
1529
|
util.debug(f"WORKER_{self.n}: Calling Popen")
|
|
1311
|
-
proc = subprocess.Popen(
|
|
1530
|
+
proc = subprocess.Popen( # pylint: disable=consider-using-with
|
|
1531
|
+
command_list, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
|
|
1532
|
+
)
|
|
1312
1533
|
self.proc = proc
|
|
1313
1534
|
util.debug(f"WORKER_{self.n}: Opened process, PID={proc.pid}")
|
|
1314
1535
|
self.pid = proc.pid
|
|
1315
|
-
|
|
1316
|
-
while proc.returncode
|
|
1536
|
+
self.threads_stats.started += 1
|
|
1537
|
+
while proc.returncode is None:
|
|
1317
1538
|
try:
|
|
1318
1539
|
if (time.time() - self.last_timer_debug) > 10:
|
|
1319
1540
|
util.debug(f"WORKER_{self.n}: Calling proc.communicate")
|
|
1320
1541
|
stdout, stderr = proc.communicate(timeout=0.5)
|
|
1321
|
-
util.debug(f"WORKER_{self.n}: got: \n*** stdout:\n{stdout}\n***
|
|
1542
|
+
util.debug(f"WORKER_{self.n}: got: \n*** stdout:\n{stdout}\n***",
|
|
1543
|
+
f"stderr:{stderr}")
|
|
1322
1544
|
except subprocess.TimeoutExpired:
|
|
1323
1545
|
if (time.time() - self.last_timer_debug) > 10:
|
|
1324
|
-
util.debug(f"WORKER_{self.n}: Timer expired, stop_request=
|
|
1546
|
+
util.debug(f"WORKER_{self.n}: Timer expired, stop_request=",
|
|
1547
|
+
f"{self.stop_request}")
|
|
1325
1548
|
self.last_timer_debug = time.time()
|
|
1326
|
-
pass
|
|
1327
1549
|
if self.stop_request:
|
|
1328
1550
|
util.debug(f"WORKER_{self.n}: got stop request, issuing SIGINT")
|
|
1329
1551
|
proc.send_signal(signal.SIGINT)
|
|
1330
1552
|
util.debug(f"WORKER_{self.n}: got stop request, calling proc.wait")
|
|
1331
1553
|
proc.wait()
|
|
1332
|
-
if False and self.stop_request:
|
|
1333
|
-
util.debug(f"WORKER_{self.n}: got stop request, issuing proc.terminate")
|
|
1334
|
-
proc.terminate()
|
|
1335
|
-
util.debug(f"WORKER_{self.n}: proc poll returns is now {proc.poll()}")
|
|
1336
|
-
try:
|
|
1337
|
-
util.debug(f"WORKER_{self.n}: Calling proc.communicate")
|
|
1338
|
-
stdout, stderr = proc.communicate(timeout=0.2) # for completeness, in case we ever pipe/search stdout/stderr
|
|
1339
|
-
util.debug(f"WORKER_{self.n}: got: \n*** stdout:\n{stdout}\n*** stderr:{stderr}")
|
|
1340
|
-
except subprocess.TimeoutExpired:
|
|
1341
|
-
util.debug(f"WORKER_{self.n}: timeout waiting for comminicate after terminate")
|
|
1342
|
-
except:
|
|
1343
|
-
pass
|
|
1344
|
-
util.debug(f"WORKER_{self.n}: proc poll returns is now {proc.poll()}")
|
|
1345
1554
|
|
|
1346
1555
|
util.debug(f"WORKER_{self.n}: -- out of while loop")
|
|
1347
1556
|
self.pid = None
|
|
@@ -1350,99 +1559,113 @@ class CommandParallelWorker(threading.Thread):
|
|
|
1350
1559
|
util.debug(f"WORKER_{self.n}: proc poll returns is now {proc.poll()}")
|
|
1351
1560
|
try:
|
|
1352
1561
|
util.debug(f"WORKER_{self.n}: Calling proc.communicate one last time")
|
|
1353
|
-
stdout, stderr = proc.communicate(timeout=0.1)
|
|
1354
|
-
util.debug(f"WORKER_{self.n}: got: \n*** stdout:\n{stdout}\n***
|
|
1562
|
+
stdout, stderr = proc.communicate(timeout=0.1)
|
|
1563
|
+
util.debug(f"WORKER_{self.n}: got: \n*** stdout:\n{stdout}\n***",
|
|
1564
|
+
f"stderr:{stderr}")
|
|
1355
1565
|
except subprocess.TimeoutExpired:
|
|
1356
1566
|
util.debug(f"WORKER_{self.n}: timeout waiting for communicate after loop?")
|
|
1357
|
-
except:
|
|
1358
|
-
|
|
1567
|
+
except Exception as e:
|
|
1568
|
+
util.debug(f"WORKER_{self.n}: timeout with exception {e=}")
|
|
1569
|
+
|
|
1359
1570
|
return_code = proc.poll()
|
|
1360
|
-
util.debug(f"WORKER_{self.n}: Finished job {i}: {job_name} with return code
|
|
1571
|
+
util.debug(f"WORKER_{self.n}: Finished job {i}: {job_name} with return code",
|
|
1572
|
+
f"{return_code}")
|
|
1361
1573
|
self.done_queue.put((i, job_name, return_code))
|
|
1362
1574
|
finally:
|
|
1363
1575
|
util.debug(f"WORKER_{self.n}: -- in finally block")
|
|
1364
1576
|
self.work_queue.task_done()
|
|
1365
|
-
|
|
1577
|
+
self.threads_stats.done += 1
|
|
1366
1578
|
|
|
1367
1579
|
|
|
1368
1580
|
class CommandParallel(Command):
|
|
1581
|
+
'''Base class command handler for running multiple eda jobs, such as for child classes
|
|
1582
|
+
|
|
1583
|
+
in commands.multi or commands.sweep
|
|
1584
|
+
'''
|
|
1369
1585
|
def __init__(self, config: dict):
|
|
1370
1586
|
Command.__init__(self, config=config)
|
|
1371
1587
|
self.jobs = []
|
|
1372
1588
|
self.jobs_status = []
|
|
1373
1589
|
self.args['parallel'] = 1
|
|
1374
1590
|
self.worker_threads = []
|
|
1591
|
+
self.threads_stats = ThreadStats()
|
|
1375
1592
|
|
|
1376
|
-
def __del__(self):
|
|
1377
|
-
util.debug(f"In Command.__del__, threads done/started: {
|
|
1378
|
-
if
|
|
1593
|
+
def __del__(self): # pylint: disable=too-many-branches
|
|
1594
|
+
util.debug(f"In Command.__del__, threads done/started: {self.threads_stats}")
|
|
1595
|
+
if self.threads_stats.all_done():
|
|
1379
1596
|
return
|
|
1380
|
-
util.warning(f"Need to shut down {
|
|
1597
|
+
util.warning(f"Need to shut down {self.threads_stats.get_remaining()} worker threads...")
|
|
1381
1598
|
for w in self.worker_threads:
|
|
1382
1599
|
if w.proc:
|
|
1383
1600
|
util.warning(f"Requesting stop of PID {w.pid}: {w.job_name}")
|
|
1384
1601
|
w.stop_request = True
|
|
1385
|
-
for
|
|
1386
|
-
util.debug(f"Threads done/started: {
|
|
1387
|
-
if
|
|
1388
|
-
util.info(
|
|
1602
|
+
for _ in range(10):
|
|
1603
|
+
util.debug(f"Threads done/started: {self.threads_stats}")
|
|
1604
|
+
if self.threads_stats.started == self.threads_stats.done:
|
|
1605
|
+
util.info("All threads done")
|
|
1389
1606
|
return
|
|
1390
1607
|
time.sleep(1)
|
|
1391
|
-
subprocess.Popen(['stty', 'sane']).wait()
|
|
1392
|
-
util.debug(
|
|
1608
|
+
subprocess.Popen(['stty', 'sane']).wait() # pylint: disable=consider-using-with
|
|
1609
|
+
util.debug("Scanning workers again")
|
|
1393
1610
|
for w in self.worker_threads:
|
|
1394
1611
|
if w.proc:
|
|
1395
1612
|
util.info(f"need to SIGINT WORKER_{w.n}, may need manual cleanup, check 'ps'")
|
|
1396
1613
|
if w.pid:
|
|
1397
1614
|
os.kill(w.pid, signal.SIGINT)
|
|
1398
|
-
for
|
|
1399
|
-
util.debug(f"Threads done/started: {
|
|
1400
|
-
if
|
|
1401
|
-
util.info(
|
|
1615
|
+
for _ in range(5):
|
|
1616
|
+
util.debug(f"Threads done/started: {self.threads_stats}")
|
|
1617
|
+
if self.threads_stats.all_done():
|
|
1618
|
+
util.info("All threads done")
|
|
1402
1619
|
return
|
|
1403
1620
|
time.sleep(1)
|
|
1404
|
-
subprocess.Popen(['stty', 'sane']).wait()
|
|
1405
|
-
util.debug(
|
|
1621
|
+
subprocess.Popen(['stty', 'sane']).wait() # pylint: disable=consider-using-with
|
|
1622
|
+
util.debug("Scanning workers again")
|
|
1406
1623
|
for w in self.worker_threads:
|
|
1407
1624
|
if w.proc:
|
|
1408
1625
|
util.info(f"need to TERM WORKER_{w.n}, probably needs manual cleanup, check 'ps'")
|
|
1409
1626
|
if w.pid:
|
|
1410
1627
|
os.kill(w.pid, signal.SIGTERM)
|
|
1411
|
-
for
|
|
1412
|
-
util.debug(f"Threads done/started: {
|
|
1413
|
-
if
|
|
1414
|
-
util.info(
|
|
1628
|
+
for _ in range(5):
|
|
1629
|
+
util.debug(f"Threads done/started: {self.threads_stats}")
|
|
1630
|
+
if self.threads_stats.all_done():
|
|
1631
|
+
util.info("All threads done")
|
|
1415
1632
|
return
|
|
1416
1633
|
time.sleep(1)
|
|
1417
|
-
subprocess.Popen(['stty', 'sane']).wait()
|
|
1418
|
-
util.debug(
|
|
1634
|
+
subprocess.Popen(['stty', 'sane']).wait() # pylint: disable=consider-using-with
|
|
1635
|
+
util.debug("Scanning workers again")
|
|
1419
1636
|
for w in self.worker_threads:
|
|
1420
1637
|
if w.proc:
|
|
1421
1638
|
util.info(f"need to KILL WORKER_{w.n}, probably needs manual cleanup, check 'ps'")
|
|
1422
1639
|
if w.pid:
|
|
1423
1640
|
os.kill(w.pid, signal.SIGKILL)
|
|
1424
1641
|
util.stop_log()
|
|
1425
|
-
subprocess.Popen(['stty', 'sane']).wait()
|
|
1642
|
+
subprocess.Popen(['stty', 'sane']).wait() # pylint: disable=consider-using-with
|
|
1643
|
+
|
|
1644
|
+
def run_jobs( # pylint: disable=too-many-locals,too-many-branches,too-many-statements
|
|
1645
|
+
self, command: str
|
|
1646
|
+
) -> None:
|
|
1647
|
+
'''Runs all "jobs" in self.jobs list, either serially or in parallel'''
|
|
1426
1648
|
|
|
1427
|
-
|
|
1428
|
-
#
|
|
1429
|
-
#
|
|
1430
|
-
#
|
|
1431
|
-
# UI output functional and awesome sauce
|
|
1649
|
+
# this is where we actually run the jobs. it's a messy piece of code and prob could use
|
|
1650
|
+
# refactoring but the goal was to share as much as possible (job start, end, pass/fail
|
|
1651
|
+
# judgement, etc) while supporting various mode combinations (parallel mode, verbose mode,
|
|
1652
|
+
# fancy mode, etc) and keeping the UI output functional and awesome sauce
|
|
1432
1653
|
|
|
1433
1654
|
# walk targets to find the longest name, for display reasons
|
|
1434
1655
|
longest_job_name = 0
|
|
1435
1656
|
total_jobs = len(self.jobs)
|
|
1436
1657
|
self.jobs_status = [None] * total_jobs
|
|
1437
1658
|
for i in range(total_jobs):
|
|
1438
|
-
|
|
1439
|
-
if l>longest_job_name: longest_job_name = l
|
|
1659
|
+
longest_job_name = max(longest_job_name, len(self.jobs[i]['name']))
|
|
1440
1660
|
|
|
1441
1661
|
run_parallel = self.args['parallel'] > 1
|
|
1442
1662
|
|
|
1443
1663
|
# figure out the width to print various numbers
|
|
1444
1664
|
jobs_digits = len(f"{total_jobs}")
|
|
1445
|
-
|
|
1665
|
+
|
|
1666
|
+
job_done_return_code = None
|
|
1667
|
+
job_done_run_time = 0
|
|
1668
|
+
job_done_name = ''
|
|
1446
1669
|
|
|
1447
1670
|
# run the jobs!
|
|
1448
1671
|
running_jobs = {}
|
|
@@ -1453,9 +1676,10 @@ class CommandParallel(Command):
|
|
|
1453
1676
|
jobs_launched = 0
|
|
1454
1677
|
num_parallel = min(len(self.jobs), self.args['parallel'])
|
|
1455
1678
|
# 16 should really be the size of window or ?
|
|
1456
|
-
|
|
1457
|
-
# we will enter fancy mode if we are parallel and we can leave 6 lines of regular scrolling
|
|
1458
|
-
|
|
1679
|
+
_, lines = shutil.get_terminal_size()
|
|
1680
|
+
# we will enter fancy mode if we are parallel and we can leave 6 lines of regular scrolling
|
|
1681
|
+
# output
|
|
1682
|
+
fancy_mode = all([util.args['fancy'], num_parallel > 1, num_parallel <= (lines-6)])
|
|
1459
1683
|
multi_cwd = util.getcwd() + os.sep
|
|
1460
1684
|
|
|
1461
1685
|
self.patch_jobs_for_duplicate_target_names()
|
|
@@ -1466,8 +1690,12 @@ class CommandParallel(Command):
|
|
|
1466
1690
|
work_queue = queue.Queue()
|
|
1467
1691
|
done_queue = queue.Queue()
|
|
1468
1692
|
for x in range(num_parallel):
|
|
1469
|
-
worker = CommandParallelWorker(
|
|
1470
|
-
|
|
1693
|
+
worker = CommandParallelWorker(
|
|
1694
|
+
n=x, work_queue=work_queue, done_queue=done_queue,
|
|
1695
|
+
threads_stats=self.threads_stats
|
|
1696
|
+
)
|
|
1697
|
+
# Setting daemon to True will let the main thread exit even though the workers are
|
|
1698
|
+
# blocking
|
|
1471
1699
|
worker.daemon = True
|
|
1472
1700
|
worker.start()
|
|
1473
1701
|
self.worker_threads.append(worker)
|
|
@@ -1478,22 +1706,30 @@ class CommandParallel(Command):
|
|
|
1478
1706
|
for x in range(num_parallel):
|
|
1479
1707
|
util.fancy_print(f"Starting worker {x}", x)
|
|
1480
1708
|
|
|
1481
|
-
while
|
|
1709
|
+
while self.jobs or running_jobs:
|
|
1482
1710
|
job_done = False
|
|
1483
1711
|
job_done_quiet = False
|
|
1484
1712
|
anything_done = False
|
|
1485
1713
|
|
|
1486
1714
|
def sprint_job_line(job_number=0, job_name="", final=False, hide_stats=False):
|
|
1487
|
-
return (
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1715
|
+
return (
|
|
1716
|
+
"INFO: [EDA] " +
|
|
1717
|
+
string_or_space(
|
|
1718
|
+
f"[job {job_number:0{jobs_digits}d}/{total_jobs:0{jobs_digits}d} ",
|
|
1719
|
+
final) +
|
|
1720
|
+
string_or_space("| pass ", hide_stats or final) +
|
|
1721
|
+
string_or_space(
|
|
1722
|
+
f"{len(passed_jobs):0{jobs_digits}d}/{jobs_complete:0{jobs_digits}d} ",
|
|
1723
|
+
hide_stats) +
|
|
1724
|
+
string_or_space(f"@ {(100 * (jobs_complete)) / total_jobs : 5.1f}%",
|
|
1725
|
+
hide_stats or final) +
|
|
1726
|
+
string_or_space("] ", final) +
|
|
1727
|
+
f"{command} {(job_name+' ').ljust(longest_job_name+3,'.')}"
|
|
1728
|
+
)
|
|
1729
|
+
|
|
1730
|
+
# for any kind of run (parallel or not, fancy or not, verbose or not) ...
|
|
1731
|
+
# can we launch a job?
|
|
1732
|
+
if self.jobs and (len(running_jobs) < num_parallel):
|
|
1497
1733
|
# we are launching a job
|
|
1498
1734
|
jobs_launched += 1
|
|
1499
1735
|
anything_done = True
|
|
@@ -1501,15 +1737,17 @@ class CommandParallel(Command):
|
|
|
1501
1737
|
# TODO(drew): it might be nice to pass more items on 'job' dict, like
|
|
1502
1738
|
# logfile or job-name, so CommandSweep or CommandMulti don't have to set
|
|
1503
1739
|
# via args. on their command_list.
|
|
1504
|
-
if job['name'].startswith(multi_cwd):
|
|
1505
|
-
|
|
1740
|
+
if job['name'].startswith(multi_cwd):
|
|
1741
|
+
job['name'] = job['name'][len(multi_cwd):]
|
|
1742
|
+
# in all but fancy mode, we will print this text at the launch of a job. It may
|
|
1743
|
+
# get a newline below
|
|
1506
1744
|
job_text = sprint_job_line(jobs_launched, job['name'], hide_stats=run_parallel)
|
|
1507
1745
|
command_list = job['command_list']
|
|
1508
1746
|
cwd = util.getcwd()
|
|
1509
1747
|
|
|
1510
1748
|
if run_parallel:
|
|
1511
1749
|
# multithreaded job launch: add to queue
|
|
1512
|
-
worker = workers.pop(0)
|
|
1750
|
+
worker = workers.pop(0)
|
|
1513
1751
|
running_jobs[str(jobs_launched)] = { 'name' : job['name'],
|
|
1514
1752
|
'number' : jobs_launched,
|
|
1515
1753
|
'worker' : worker,
|
|
@@ -1519,15 +1757,20 @@ class CommandParallel(Command):
|
|
|
1519
1757
|
suffix = "<START>"
|
|
1520
1758
|
if fancy_mode:
|
|
1521
1759
|
util.fancy_print(job_text+suffix, worker)
|
|
1760
|
+
elif failed_jobs:
|
|
1761
|
+
# if we aren't in fancy mode, we will print a START line, periodic RUNNING
|
|
1762
|
+
# lines, and PASS/FAIL line per-job
|
|
1763
|
+
util.print_orange(job_text + Colors.yellow + suffix)
|
|
1522
1764
|
else:
|
|
1523
|
-
|
|
1524
|
-
if len(failed_jobs): util.print_orange(job_text + util.string_yellow + suffix)
|
|
1525
|
-
else: util.print_yellow(job_text + util.string_yellow + suffix)
|
|
1765
|
+
util.print_yellow(job_text + Colors.yellow + suffix)
|
|
1526
1766
|
else:
|
|
1527
|
-
# single-threaded job launch, we are going to print out job info as we start
|
|
1528
|
-
# since non-verbose silences the job and prints only
|
|
1529
|
-
|
|
1530
|
-
|
|
1767
|
+
# single-threaded job launch, we are going to print out job info as we start
|
|
1768
|
+
# each job... no newline. since non-verbose silences the job and prints only
|
|
1769
|
+
# <PASS>/<FAIL> after the trailing "..." we leave here
|
|
1770
|
+
if failed_jobs:
|
|
1771
|
+
util.print_orange(job_text, end="")
|
|
1772
|
+
else:
|
|
1773
|
+
util.print_yellow(job_text, end="")
|
|
1531
1774
|
job_done_number = jobs_launched
|
|
1532
1775
|
job_done_name = job['name']
|
|
1533
1776
|
job_start_time = time.time()
|
|
@@ -1539,13 +1782,16 @@ class CommandParallel(Command):
|
|
|
1539
1782
|
_, _, job_done_return_code = self.exec(
|
|
1540
1783
|
cwd, command_list, background=False, stop_on_error=False, quiet=False
|
|
1541
1784
|
)
|
|
1542
|
-
# reprint the job text previously printed before running job(and given
|
|
1785
|
+
# reprint the job text previously printed before running job (and given
|
|
1786
|
+
# "\n" after the trailing "...")
|
|
1543
1787
|
else:
|
|
1544
1788
|
# run job, swallowing output (hope you have a logfile)
|
|
1545
1789
|
_, _, job_done_return_code = self.exec(
|
|
1546
1790
|
cwd, command_list, background=True, stop_on_error=False, quiet=True
|
|
1547
1791
|
)
|
|
1548
|
-
|
|
1792
|
+
# in this case, we have the job start text (trailing "...", no newline)
|
|
1793
|
+
# printed
|
|
1794
|
+
job_done_quiet = True
|
|
1549
1795
|
job_done = True
|
|
1550
1796
|
job_done_run_time = time.time() - job_start_time
|
|
1551
1797
|
# Since we consumed the job, use the job['index'] to track the per-job status:
|
|
@@ -1553,15 +1799,16 @@ class CommandParallel(Command):
|
|
|
1553
1799
|
if run_parallel:
|
|
1554
1800
|
# parallel run, check for completed job
|
|
1555
1801
|
if done_queue.qsize():
|
|
1556
|
-
# we're collecting a finished job from a worker thread. note we will only
|
|
1557
|
-
# loop, so as to share job completion code
|
|
1802
|
+
# we're collecting a finished job from a worker thread. note we will only
|
|
1803
|
+
# reap one job per iter of the big loop, so as to share job completion code
|
|
1804
|
+
# at the bottom
|
|
1558
1805
|
anything_done = True
|
|
1559
1806
|
job_done = True
|
|
1560
1807
|
job_done_number, job_done_name, job_done_return_code = done_queue.get()
|
|
1561
1808
|
t = running_jobs[str(job_done_number)]
|
|
1562
1809
|
# in fancy mode, we need to clear the worker line related to this job.
|
|
1563
1810
|
if fancy_mode:
|
|
1564
|
-
util.fancy_print(
|
|
1811
|
+
util.fancy_print("INFO: [EDA] Parallel: Worker Idle ...", t['worker'])
|
|
1565
1812
|
job_done_run_time = time.time() - t['start_time']
|
|
1566
1813
|
util.debug(f"removing job #{job_done_number} from running jobs")
|
|
1567
1814
|
del running_jobs[str(job_done_number)]
|
|
@@ -1573,33 +1820,40 @@ class CommandParallel(Command):
|
|
|
1573
1820
|
if (fancy_mode or (time.time() - t['update_time']) > 30):
|
|
1574
1821
|
t['update_time'] = time.time()
|
|
1575
1822
|
job_text = sprint_job_line(t['number'], t['name'], hide_stats=True)
|
|
1576
|
-
suffix = f"<RUNNING: {
|
|
1823
|
+
suffix = f"<RUNNING: {sprint_time(time.time() - t['start_time'])}>"
|
|
1577
1824
|
if fancy_mode:
|
|
1578
1825
|
util.fancy_print(f"{job_text}{suffix}", t['worker'])
|
|
1826
|
+
elif failed_jobs:
|
|
1827
|
+
util.print_orange(job_text + Colors.yellow + suffix)
|
|
1579
1828
|
else:
|
|
1580
|
-
|
|
1581
|
-
else: util.print_yellow(job_text+util.string_yellow+suffix)
|
|
1829
|
+
util.print_yellow(job_text + Colors.yellow + suffix)
|
|
1582
1830
|
|
|
1583
1831
|
# shared job completion code
|
|
1584
|
-
# single or multi-threaded, we can arrive here to harvest <= 1 jobs, and need
|
|
1585
|
-
# we expect the start of a status line to have been
|
|
1832
|
+
# single or multi-threaded, we can arrive here to harvest <= 1 jobs, and need
|
|
1833
|
+
# {job, return_code} valid, and we expect the start of a status line to have been
|
|
1834
|
+
# printed, ready for pass/fail
|
|
1586
1835
|
if job_done:
|
|
1587
1836
|
jobs_complete += 1
|
|
1588
1837
|
if job_done_return_code is None or job_done_return_code:
|
|
1589
|
-
# embed the color code, to change color of pass/fail during the
|
|
1838
|
+
# embed the color code, to change color of pass/fail during the
|
|
1839
|
+
# util.print_orange/yellow below
|
|
1590
1840
|
if job_done_return_code == 124:
|
|
1591
|
-
# bash uses 124 for bash timeout errors, if that was preprended to the
|
|
1592
|
-
|
|
1841
|
+
# bash uses 124 for bash timeout errors, if that was preprended to the
|
|
1842
|
+
# command list.
|
|
1843
|
+
suffix = f"{Colors.red}<TOUT: {sprint_time(job_done_run_time)}>"
|
|
1593
1844
|
else:
|
|
1594
|
-
suffix = f"{
|
|
1845
|
+
suffix = f"{Colors.red}<FAIL: {sprint_time(job_done_run_time)}>"
|
|
1595
1846
|
failed_jobs.append(job_done_name)
|
|
1596
1847
|
else:
|
|
1597
|
-
suffix = f"{
|
|
1848
|
+
suffix = f"{Colors.green}<PASS: {sprint_time(job_done_run_time)}>"
|
|
1598
1849
|
passed_jobs.append(job_done_name)
|
|
1599
1850
|
# we want to print in one shot, because in fancy modes that's all that we're allowed
|
|
1600
|
-
job_done_text = "" if job_done_quiet else sprint_job_line(job_done_number,
|
|
1601
|
-
|
|
1602
|
-
|
|
1851
|
+
job_done_text = "" if job_done_quiet else sprint_job_line(job_done_number,
|
|
1852
|
+
job_done_name)
|
|
1853
|
+
if failed_jobs:
|
|
1854
|
+
util.print_orange(f"{job_done_text}{suffix}")
|
|
1855
|
+
else:
|
|
1856
|
+
util.print_yellow(f"{job_done_text}{suffix}")
|
|
1603
1857
|
self.jobs_status[job_done_number-1] = job_done_return_code
|
|
1604
1858
|
|
|
1605
1859
|
if not anything_done:
|
|
@@ -1607,23 +1861,26 @@ class CommandParallel(Command):
|
|
|
1607
1861
|
|
|
1608
1862
|
if total_jobs:
|
|
1609
1863
|
emoji = "< :) >" if (len(passed_jobs) == total_jobs) else "< :( >"
|
|
1610
|
-
util.info(sprint_job_line(final=True,job_name="jobs passed")+emoji, start="")
|
|
1864
|
+
util.info(sprint_job_line(final=True, job_name="jobs passed") + emoji, start="")
|
|
1611
1865
|
else:
|
|
1612
|
-
util.info(
|
|
1866
|
+
util.info("Parallel: <No jobs found>")
|
|
1613
1867
|
# Make sure all jobs have a set status:
|
|
1614
|
-
for i,rc in enumerate(self.jobs_status):
|
|
1615
|
-
if rc is None or
|
|
1868
|
+
for i, rc in enumerate(self.jobs_status):
|
|
1869
|
+
if rc is None or not isinstance(rc, int):
|
|
1616
1870
|
self.error(f'job {i=} {rc=} did not return a proper return code')
|
|
1617
|
-
jobs_status[i] =
|
|
1871
|
+
self.jobs_status[i] = 2
|
|
1618
1872
|
|
|
1619
1873
|
# if self.status > 0, then keep it non-zero, else set it if we still have running jobs.
|
|
1620
1874
|
if self.status == 0:
|
|
1621
|
-
|
|
1875
|
+
if self.jobs_status:
|
|
1876
|
+
self.status = max(self.jobs_status)
|
|
1877
|
+
# else keep at 0, empty list.
|
|
1622
1878
|
util.fancy_stop()
|
|
1623
1879
|
|
|
1624
1880
|
@staticmethod
|
|
1625
1881
|
def get_name_from_target(target: str) -> str:
|
|
1626
|
-
|
|
1882
|
+
'''Given a target path, strip leftmost path separators to get a shorter string name'''
|
|
1883
|
+
return target.replace('../', '').lstrip('./').lstrip(os.path.sep)
|
|
1627
1884
|
|
|
1628
1885
|
|
|
1629
1886
|
def update_args_list(self, args: list, tool: str) -> None:
|
|
@@ -1667,7 +1924,7 @@ class CommandParallel(Command):
|
|
|
1667
1924
|
tokens=tokens.copy(),
|
|
1668
1925
|
apply_parsed_args=False,
|
|
1669
1926
|
)
|
|
1670
|
-
util.debug(f'{self.command_name}: {single_cmd_unparsed=}')
|
|
1927
|
+
util.debug(f'{self.command_name}: {single_cmd_parsed=}, {single_cmd_unparsed=}')
|
|
1671
1928
|
|
|
1672
1929
|
# There should not be any single_cmd_unparsed args starting with '-'
|
|
1673
1930
|
bad_remaining_args = [x for x in single_cmd_unparsed if x.startswith('-')]
|
|
@@ -1694,7 +1951,7 @@ class CommandParallel(Command):
|
|
|
1694
1951
|
if item.startswith(f'--{arg_name}='):
|
|
1695
1952
|
_, name = item.split(f'--{arg_name}=')
|
|
1696
1953
|
return name
|
|
1697
|
-
|
|
1954
|
+
if item == f'--{arg_name}':
|
|
1698
1955
|
return job_dict['command_list'][i + 1]
|
|
1699
1956
|
return ''
|
|
1700
1957
|
|
|
@@ -1704,7 +1961,7 @@ class CommandParallel(Command):
|
|
|
1704
1961
|
if item.startswith(f'--{arg_name}='):
|
|
1705
1962
|
job_dict['command_list'][i] = f'--{arg_name}=' + new_value
|
|
1706
1963
|
return True
|
|
1707
|
-
|
|
1964
|
+
if item == f'--{arg_name}':
|
|
1708
1965
|
job_dict['command_list'][i + 1] = new_value
|
|
1709
1966
|
return True
|
|
1710
1967
|
return False
|
|
@@ -1729,7 +1986,6 @@ class CommandParallel(Command):
|
|
|
1729
1986
|
tpath, _ = os.path.split(job_dict['target'])
|
|
1730
1987
|
|
|
1731
1988
|
# prepend path information to job-name:
|
|
1732
|
-
patched_sub_work_dir = False
|
|
1733
1989
|
patched_target_path = os.path.relpath(tpath).replace(os.sep, '_')
|
|
1734
1990
|
new_job_name = f'{patched_target_path}.{key}'
|
|
1735
1991
|
replace_job_arg(job_dict, arg_name='job-name', new_value=new_job_name)
|