opencos-eda 0.2.47__py3-none-any.whl → 0.2.49__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- opencos/__init__.py +4 -2
- opencos/_version.py +10 -7
- opencos/commands/flist.py +8 -7
- opencos/commands/multi.py +35 -18
- opencos/commands/sweep.py +9 -4
- opencos/commands/waves.py +1 -1
- opencos/deps/__init__.py +0 -0
- opencos/deps/defaults.py +69 -0
- opencos/deps/deps_commands.py +419 -0
- opencos/deps/deps_file.py +326 -0
- opencos/deps/deps_processor.py +670 -0
- opencos/deps_schema.py +7 -8
- opencos/eda.py +92 -67
- opencos/eda_base.py +625 -332
- opencos/eda_config.py +80 -14
- opencos/eda_extract_targets.py +22 -14
- opencos/eda_tool_helper.py +33 -7
- opencos/export_helper.py +166 -86
- opencos/export_json_convert.py +31 -23
- opencos/files.py +2 -1
- opencos/hw/__init__.py +0 -0
- opencos/{oc_cli.py → hw/oc_cli.py} +9 -4
- opencos/names.py +0 -4
- opencos/peakrdl_cleanup.py +13 -7
- opencos/seed.py +19 -11
- opencos/tests/helpers.py +27 -14
- opencos/tests/test_deps_helpers.py +35 -32
- opencos/tests/test_eda.py +47 -41
- opencos/tests/test_eda_elab.py +5 -3
- opencos/tests/test_eda_synth.py +1 -1
- opencos/tests/test_oc_cli.py +1 -1
- opencos/tests/test_tools.py +3 -2
- opencos/tools/iverilog.py +2 -2
- opencos/tools/modelsim_ase.py +2 -2
- opencos/tools/riviera.py +1 -1
- opencos/tools/slang.py +1 -1
- opencos/tools/surelog.py +1 -1
- opencos/tools/verilator.py +1 -1
- opencos/tools/vivado.py +1 -1
- opencos/tools/yosys.py +4 -3
- opencos/util.py +440 -483
- opencos/utils/__init__.py +0 -0
- opencos/utils/markup_helpers.py +98 -0
- opencos/utils/str_helpers.py +111 -0
- opencos/utils/subprocess_helpers.py +108 -0
- {opencos_eda-0.2.47.dist-info → opencos_eda-0.2.49.dist-info}/METADATA +1 -1
- opencos_eda-0.2.49.dist-info/RECORD +88 -0
- {opencos_eda-0.2.47.dist-info → opencos_eda-0.2.49.dist-info}/entry_points.txt +1 -1
- opencos/deps_helpers.py +0 -1346
- opencos_eda-0.2.47.dist-info/RECORD +0 -79
- /opencos/{pcie.py → hw/pcie.py} +0 -0
- {opencos_eda-0.2.47.dist-info → opencos_eda-0.2.49.dist-info}/WHEEL +0 -0
- {opencos_eda-0.2.47.dist-info → opencos_eda-0.2.49.dist-info}/licenses/LICENSE +0 -0
- {opencos_eda-0.2.47.dist-info → opencos_eda-0.2.49.dist-info}/licenses/LICENSE.spdx +0 -0
- {opencos_eda-0.2.47.dist-info → opencos_eda-0.2.49.dist-info}/top_level.txt +0 -0
opencos/eda_base.py
CHANGED
|
@@ -1,22 +1,36 @@
|
|
|
1
|
+
'''opencos.eda_base holds base classes for opencos.tools and opencos.commands
|
|
2
|
+
|
|
3
|
+
Note that opencos.Command.process_tokens (on a derived class) is the main entrypoint
|
|
4
|
+
from eda.py, it will find a command handler class for a command + tool, and call
|
|
5
|
+
its obj.process_tokens(args)
|
|
6
|
+
'''
|
|
7
|
+
|
|
8
|
+
# TODO(drew): Clean up the following pylint
|
|
9
|
+
# pylint: disable=too-many-lines
|
|
10
|
+
|
|
1
11
|
import argparse
|
|
2
12
|
import copy
|
|
3
|
-
import glob
|
|
4
13
|
import subprocess
|
|
5
14
|
import os
|
|
6
15
|
import queue
|
|
7
16
|
import re
|
|
8
17
|
import shutil
|
|
9
|
-
import shlex
|
|
10
18
|
import signal
|
|
11
19
|
import sys
|
|
12
20
|
import threading
|
|
13
21
|
import time
|
|
14
22
|
from pathlib import Path
|
|
15
23
|
|
|
16
|
-
import
|
|
17
|
-
from opencos import seed, deps_helpers, util, files
|
|
24
|
+
from opencos import seed, util, files
|
|
18
25
|
from opencos import eda_config
|
|
19
|
-
|
|
26
|
+
|
|
27
|
+
from opencos.util import Colors
|
|
28
|
+
from opencos.utils.str_helpers import sprint_time, strip_outer_quotes, string_or_space, \
|
|
29
|
+
indent_wrap_long_text
|
|
30
|
+
from opencos.utils.subprocess_helpers import subprocess_run_background
|
|
31
|
+
|
|
32
|
+
from opencos.deps.deps_file import DepsFile, deps_data_get_all_targets
|
|
33
|
+
from opencos.deps.deps_processor import DepsProcessor
|
|
20
34
|
|
|
21
35
|
|
|
22
36
|
def print_base_help() -> None:
|
|
@@ -28,9 +42,12 @@ def print_base_help() -> None:
|
|
|
28
42
|
|
|
29
43
|
|
|
30
44
|
def get_argparser() -> argparse.ArgumentParser:
|
|
45
|
+
'''Returns the ArgumentParser for general eda CLI'''
|
|
31
46
|
parser = argparse.ArgumentParser(prog='eda options', add_help=False, allow_abbrev=False)
|
|
32
47
|
parser.add_argument('-q', '--quit', action='store_true',
|
|
33
|
-
help=
|
|
48
|
+
help=(
|
|
49
|
+
'For interactive mode (eda called with no options, command, or'
|
|
50
|
+
' targets)'))
|
|
34
51
|
parser.add_argument('--exit', action='store_true', help='same as --quit')
|
|
35
52
|
parser.add_argument('-h', '--help', action='store_true')
|
|
36
53
|
|
|
@@ -40,22 +57,25 @@ def get_argparser() -> argparse.ArgumentParser:
|
|
|
40
57
|
help='Tool to use for this command, such as: modelsim_ase, verilator,' \
|
|
41
58
|
+ ' modelsim_ase=/path/to/bin/vsim, verilator=/path/to/bin/verilator')
|
|
42
59
|
parser.add_argument('--eda-safe', action='store_true',
|
|
43
|
-
help='disable all DEPS file deps shell commands, overrides values from
|
|
60
|
+
help=('disable all DEPS file deps shell commands, overrides values from'
|
|
61
|
+
' --config-yml'))
|
|
44
62
|
return parser
|
|
45
63
|
|
|
46
64
|
|
|
47
65
|
def get_argparser_short_help() -> str:
|
|
66
|
+
'''Returns str for the eda_base argparser (--help, --tool, etc)'''
|
|
48
67
|
return util.get_argparser_short_help(parser=get_argparser())
|
|
49
68
|
|
|
50
69
|
|
|
51
|
-
def get_eda_exec(command:str=''):
|
|
70
|
+
def get_eda_exec(command: str = '') -> str:
|
|
71
|
+
'''Returns the full path of `eda` executable to be used for a given eda <command>'''
|
|
52
72
|
# NOTE(drew): This is kind of flaky. 'eda multi' reinvokes 'eda'. But the executable for 'eda'
|
|
53
73
|
# is one of:
|
|
54
74
|
# 1. pip3 install opencos-eda
|
|
55
75
|
# -- script 'eda', installed from PyPi
|
|
56
76
|
# 2. pip3 uninstall .; python3 -m build; pip3 install
|
|
57
77
|
# -- script 'eda' but installed from local.
|
|
58
|
-
# 2. (opencos repo)/bin/eda - a python wrapper to link to (opencos repo)/opencos/eda.py
|
|
78
|
+
# 2. (opencos repo)/bin/eda - a python wrapper to link to (opencos repo)/opencos/eda.py, but
|
|
59
79
|
# packages cannot be run standalone, they need to be called as: python3 -m opencos.eda,
|
|
60
80
|
# and do not work with relative paths. This only works if env OC_ROOT is set or can be found.
|
|
61
81
|
# 3. If you ran 'source bin/addpath' then you are always using the local (opencos repo)/bin/eda
|
|
@@ -64,11 +84,13 @@ def get_eda_exec(command:str=''):
|
|
|
64
84
|
# Can we run from OC_ROOT/bin/eda?
|
|
65
85
|
oc_root = util.get_oc_root()
|
|
66
86
|
if not oc_root:
|
|
67
|
-
util.error(f"Need 'eda' in our path to run 'eda {command}', could not find env
|
|
87
|
+
util.error(f"Need 'eda' in our path to run 'eda {command}', could not find env",
|
|
88
|
+
f"OC_ROOT, {eda_path=}, {oc_root=}")
|
|
68
89
|
else:
|
|
69
90
|
bin_eda = os.path.join(oc_root, 'bin', 'eda')
|
|
70
91
|
if not os.path.exists(bin_eda):
|
|
71
|
-
util.error(f"Need 'eda' in our path to run 'eda {command}', cound not find
|
|
92
|
+
util.error(f"Need 'eda' in our path to run 'eda {command}', cound not find",
|
|
93
|
+
f"bin/, {eda_path=}, {oc_root=}, {bin_eda=}")
|
|
72
94
|
else:
|
|
73
95
|
util.info(f"'eda' not in path, using {bin_eda=} for 'eda' {command} executable")
|
|
74
96
|
eda_path = os.path.abspath(bin_eda)
|
|
@@ -76,9 +98,10 @@ def get_eda_exec(command:str=''):
|
|
|
76
98
|
return eda_path
|
|
77
99
|
|
|
78
100
|
|
|
79
|
-
def which_tool(command, config):
|
|
80
|
-
'''Returns which tool will be used for a command, given the command_handlers in
|
|
81
|
-
|
|
101
|
+
def which_tool(command: str, config: dict) -> str:
|
|
102
|
+
'''Returns which tool (str) will be used for a command, given the command_handlers in
|
|
103
|
+
|
|
104
|
+
config dict.'''
|
|
82
105
|
if config is None:
|
|
83
106
|
util.error(f'which_tool({command=}) called w/out config')
|
|
84
107
|
if not command in config.get('command_handler', {}):
|
|
@@ -136,9 +159,10 @@ class Tool:
|
|
|
136
159
|
self.set_tool_config_from_config()
|
|
137
160
|
|
|
138
161
|
def set_exe(self, config: dict) -> None:
|
|
162
|
+
'''Sets self._EXE based on config'''
|
|
139
163
|
if self._TOOL and self._TOOL in config.get('auto_tools_order', [{}])[0]:
|
|
140
164
|
exe = config.get('auto_tools_order', [{}])[0][self._TOOL].get('exe', '')
|
|
141
|
-
if exe and
|
|
165
|
+
if exe and isinstance(exe, list):
|
|
142
166
|
exe = exe[0] # pick first
|
|
143
167
|
if exe and exe != self._EXE:
|
|
144
168
|
util.info(f'Override for {self._TOOL} using exe {exe}')
|
|
@@ -155,9 +179,16 @@ class Tool:
|
|
|
155
179
|
return self._VERSION
|
|
156
180
|
|
|
157
181
|
def set_tool_defines(self) -> None:
|
|
158
|
-
|
|
182
|
+
'''Derived classes may override, sets any additional defines based on tool.'''
|
|
183
|
+
return
|
|
184
|
+
|
|
159
185
|
|
|
160
186
|
class Command:
|
|
187
|
+
'''Base class for all: eda <command>
|
|
188
|
+
|
|
189
|
+
The Command class should be used when you don't require files, otherwise consider
|
|
190
|
+
CommandDesign.
|
|
191
|
+
'''
|
|
161
192
|
|
|
162
193
|
command_name: str = ''
|
|
163
194
|
|
|
@@ -170,20 +201,20 @@ class Command:
|
|
|
170
201
|
"keep" : False,
|
|
171
202
|
"force" : False,
|
|
172
203
|
"fake" : False,
|
|
173
|
-
"stop-before-compile": False,
|
|
204
|
+
"stop-before-compile": False,
|
|
174
205
|
"stop-after-compile": False,
|
|
175
|
-
"stop-after-elaborate": False,
|
|
206
|
+
"stop-after-elaborate": False,
|
|
176
207
|
"lint": False, # Same as stop-after-elaborate
|
|
177
208
|
"eda-dir" : "eda.work", # all eda jobs go in here
|
|
178
209
|
"job-name" : "", # this is used to create a certain dir under "eda_dir"
|
|
179
|
-
"work-dir" : "", #
|
|
180
|
-
"sub-work-dir" : "", # this can be used to name the dir built under <eda-dir
|
|
210
|
+
"work-dir" : "", # default is usually <eda-dir>/<job-name> or <eda-dir>/<target>.<cmd>
|
|
211
|
+
"sub-work-dir" : "", # this can be used to name the dir built under <eda-dir>
|
|
181
212
|
"work-dir-use-target-dir": False,
|
|
182
213
|
"suffix" : "",
|
|
183
214
|
"design" : "", # not sure how this relates to top
|
|
184
215
|
'export': False,
|
|
185
|
-
'export-run': False,
|
|
186
|
-
'export-json': False,
|
|
216
|
+
'export-run': False, # run from the exported location if possible
|
|
217
|
+
'export-json': False, # generates an export.json suitable for a testrunner
|
|
187
218
|
'enable-tags': [],
|
|
188
219
|
'disable-tags': [],
|
|
189
220
|
'test-mode': False,
|
|
@@ -212,12 +243,32 @@ class Command:
|
|
|
212
243
|
self.target = ""
|
|
213
244
|
self.target_path = ""
|
|
214
245
|
self.status = 0
|
|
246
|
+
self.errors_log_f = None
|
|
247
|
+
|
|
215
248
|
|
|
216
|
-
def error(self, *args, **kwargs):
|
|
217
|
-
'''Returns None, child classes can call self.error(..) instead of util.error,
|
|
249
|
+
def error(self, *args, **kwargs) -> None:
|
|
250
|
+
'''Returns None, child classes can call self.error(..) instead of util.error,
|
|
218
251
|
|
|
219
|
-
|
|
252
|
+
which updates their self.status. Also will write out to eda.errors.log if the work-dir
|
|
253
|
+
exists. Please consider using Command.error(..) (or self.error(..)) in place of util.error
|
|
254
|
+
so self.status is updated.
|
|
220
255
|
'''
|
|
256
|
+
|
|
257
|
+
if self.args['work-dir']:
|
|
258
|
+
if not self.errors_log_f:
|
|
259
|
+
try:
|
|
260
|
+
self.errors_log_f = open( # pylint: disable=consider-using-with
|
|
261
|
+
os.path.join(self.args['work-dir'], 'eda.errors.log'), 'w',
|
|
262
|
+
encoding='utf-8'
|
|
263
|
+
)
|
|
264
|
+
except FileNotFoundError:
|
|
265
|
+
pass
|
|
266
|
+
if self.errors_log_f:
|
|
267
|
+
print(
|
|
268
|
+
f'ERROR: [eda] ({self.command_name}) {" ".join(list(args))}',
|
|
269
|
+
file=self.errors_log_f
|
|
270
|
+
)
|
|
271
|
+
|
|
221
272
|
self.status = util.error(*args, **kwargs)
|
|
222
273
|
|
|
223
274
|
def status_any_error(self, report=True) -> bool:
|
|
@@ -227,17 +278,29 @@ class Command:
|
|
|
227
278
|
util.error(f"command '{self.command_name}' has previous errors")
|
|
228
279
|
return self.status > 0
|
|
229
280
|
|
|
230
|
-
def which_tool(self, command:str):
|
|
281
|
+
def which_tool(self, command:str) -> str:
|
|
282
|
+
'''Returns a str for the tool name used for the requested command'''
|
|
231
283
|
return which_tool(command, config=self.config)
|
|
232
284
|
|
|
233
|
-
def create_work_dir(
|
|
285
|
+
def create_work_dir( # pylint: disable=too-many-branches,too-many-statements
|
|
286
|
+
self
|
|
287
|
+
) -> str:
|
|
288
|
+
'''Creates the working directory and populates self.args['work-dir']
|
|
289
|
+
|
|
290
|
+
Generally uses ./ self.args['eda-dir'] / <target-name>.<command> /
|
|
291
|
+
however, self.args['job-name'] or ['sub-work-dir'] can override that.
|
|
292
|
+
|
|
293
|
+
Additionally, the work-dir is attempted to be deleted if it already exists
|
|
294
|
+
AND it is beneath the caller's original working directory. We also will
|
|
295
|
+
not delete 'work-dir' that is ./ or $PWD.
|
|
296
|
+
'''
|
|
234
297
|
util.debug(f"create_work_dir: {self.args['eda-dir']=} {self.args['work-dir']=}")
|
|
235
298
|
if self.args['work-dir-use-target-dir']:
|
|
236
299
|
if not self.target_path:
|
|
237
300
|
self.target_path = '.'
|
|
238
|
-
util.info(
|
|
239
|
-
|
|
240
|
-
|
|
301
|
+
util.info("create_work_dir: --work-dir-use-target-dir: using:",
|
|
302
|
+
f"{os.path.abspath(self.target_path)}",
|
|
303
|
+
f'target={self.target}')
|
|
241
304
|
self.args['work-dir'] = self.target_path
|
|
242
305
|
self.args['sub-work-dir'] = ''
|
|
243
306
|
return self.args['work-dir']
|
|
@@ -245,28 +308,37 @@ class Command:
|
|
|
245
308
|
if not os.path.exists(self.args['eda-dir']):
|
|
246
309
|
util.safe_mkdir(self.args['eda-dir'])
|
|
247
310
|
util.info(f"create_work_dir: created {self.args['eda-dir']}")
|
|
311
|
+
|
|
248
312
|
if self.args['design'] == "":
|
|
249
313
|
if ('top' in self.args) and (self.args['top'] != ""):
|
|
250
314
|
self.args['design'] = self.args['top']
|
|
251
|
-
util.debug(f"create_work_dir: set {self.args['design']=} from
|
|
315
|
+
util.debug(f"create_work_dir: set {self.args['design']=} from",
|
|
316
|
+
f"{self.args['top']=}, since it was empty")
|
|
252
317
|
else:
|
|
253
318
|
self.args['design'] = "design" # generic, i.e. to create work dir "design_upload"
|
|
254
|
-
util.debug(f"create_work_dir: set {self.args['design']=} to 'design', since it
|
|
319
|
+
util.debug(f"create_work_dir: set {self.args['design']=} to 'design', since it",
|
|
320
|
+
"was empty and we have no top")
|
|
321
|
+
|
|
255
322
|
if self.target == "":
|
|
256
323
|
self.target = self.args['design']
|
|
257
324
|
util.debug(f"create_work_dir: set {self.target=} from design name, since it was empty")
|
|
325
|
+
|
|
258
326
|
if self.args['work-dir'] == '':
|
|
259
327
|
if self.args['sub-work-dir'] == '':
|
|
260
328
|
if self.args['job-name'] != '':
|
|
261
329
|
self.args['sub-work-dir'] = self.args['job-name']
|
|
262
|
-
util.debug(f"create_work_dir: set {self.args['sub-work-dir']=} from
|
|
330
|
+
util.debug(f"create_work_dir: set {self.args['sub-work-dir']=} from",
|
|
331
|
+
f"{self.args['job-name']=}, since it was empty")
|
|
263
332
|
else:
|
|
264
333
|
self.args['sub-work-dir'] = f'{self.target}.{self.command_name}'
|
|
265
|
-
util.debug(f"create_work_dir: set {self.args['sub-work-dir']=} from
|
|
334
|
+
util.debug(f"create_work_dir: set {self.args['sub-work-dir']=} from",
|
|
335
|
+
f"{self.target=} and {self.command_name=}, since it was empty and",
|
|
336
|
+
"we have no job-name")
|
|
266
337
|
self.args['work-dir'] = os.path.join(self.args['eda-dir'], self.args['sub-work-dir'])
|
|
267
338
|
util.debug(f"create_work_dir: set {self.args['work-dir']=}")
|
|
339
|
+
|
|
268
340
|
keep_file = os.path.join(self.args['work-dir'], "eda.keep")
|
|
269
|
-
if
|
|
341
|
+
if os.path.exists(self.args['work-dir']):
|
|
270
342
|
if os.path.exists(keep_file) and not self.args['force']:
|
|
271
343
|
self.error(f"Cannot remove old work dir due to '{keep_file}'")
|
|
272
344
|
elif os.path.abspath(self.args['work-dir']) in os.getcwd():
|
|
@@ -275,16 +347,19 @@ class Command:
|
|
|
275
347
|
# --work-dir=$PWD
|
|
276
348
|
# --work-dir=/some/path/almost/here
|
|
277
349
|
# Allow it, but preserve the existing directory, we don't want to blow away
|
|
278
|
-
# files up-
|
|
350
|
+
# files up-hier from us.
|
|
279
351
|
# Enables support for --work-dir=.
|
|
280
|
-
util.info(f"Not removing existing work-dir: '{self.args['work-dir']}' is within
|
|
352
|
+
util.info(f"Not removing existing work-dir: '{self.args['work-dir']}' is within",
|
|
353
|
+
f"{os.getcwd()=}")
|
|
281
354
|
elif str(Path(self.args['work-dir'])).startswith(str(Path('/'))):
|
|
282
355
|
# Do not allow other absolute path work dirs if it already exists.
|
|
283
356
|
# This prevents you from --work-dir=~ and eda wipes out your home dir.
|
|
284
|
-
self.error(f'Cannot use work-dir={self.args["work-dir"]} starting with
|
|
357
|
+
self.error(f'Cannot use work-dir={self.args["work-dir"]} starting with',
|
|
358
|
+
'fabsolute path "/"')
|
|
285
359
|
elif str(Path('..')) in str(Path(self.args['work-dir'])):
|
|
286
360
|
# Do not allow other ../ work dirs if it already exists.
|
|
287
|
-
self.error(f'Cannot use work-dir={self.args["work-dir"]} with up-hierarchy
|
|
361
|
+
self.error(f'Cannot use work-dir={self.args["work-dir"]} with up-hierarchy'
|
|
362
|
+
'../ paths')
|
|
288
363
|
else:
|
|
289
364
|
# If we made it this far, on a directory that exists, that appears safe
|
|
290
365
|
# to delete and re-create:
|
|
@@ -295,22 +370,32 @@ class Command:
|
|
|
295
370
|
else:
|
|
296
371
|
util.safe_mkdir(self.args['work-dir'])
|
|
297
372
|
util.debug(f'create_work_dir: created {self.args["work-dir"]}')
|
|
298
|
-
|
|
299
|
-
|
|
373
|
+
|
|
374
|
+
if self.args['keep']:
|
|
375
|
+
open(keep_file, 'w', encoding='utf-8').close() # pylint: disable=consider-using-with
|
|
300
376
|
util.debug(f'create_work_dir: created {keep_file=}')
|
|
301
377
|
return self.args['work-dir']
|
|
302
378
|
|
|
303
|
-
|
|
304
|
-
|
|
379
|
+
|
|
380
|
+
def exec(self, work_dir: str, command_list: list, background: bool = False,
|
|
381
|
+
stop_on_error: bool = True, quiet: bool = False, tee_fpath: str = '',
|
|
382
|
+
shell: bool = False) -> (str, str, int):
|
|
383
|
+
'''Runs a command via subprocess and returns a tuple of (stderr, stdout, rc)
|
|
384
|
+
|
|
385
|
+
- work_dir: is passed to the subprocess call, so os.chdir(..) is not invoked.
|
|
386
|
+
- command_list: should be a list of individual strings w/out spaces.
|
|
387
|
+
- background: if False, does not print to stdout (but will print to any
|
|
388
|
+
tee_fpath or the global log
|
|
389
|
+
- shell: arg passed to subprocess, defaults to False.
|
|
390
|
+
'''
|
|
391
|
+
|
|
305
392
|
if not tee_fpath and getattr(command_list, 'tee_fpath', None):
|
|
306
393
|
tee_fpath = getattr(command_list, 'tee_fpath', '')
|
|
307
394
|
if not quiet:
|
|
308
395
|
util.info(f"exec: {' '.join(command_list)} (in {work_dir}, {tee_fpath=})")
|
|
309
|
-
original_cwd = util.getcwd()
|
|
310
|
-
os.chdir(work_dir)
|
|
311
396
|
|
|
312
|
-
stdout, stderr, return_code =
|
|
313
|
-
work_dir=
|
|
397
|
+
stdout, stderr, return_code = subprocess_run_background(
|
|
398
|
+
work_dir=work_dir,
|
|
314
399
|
command_list=command_list,
|
|
315
400
|
background=background,
|
|
316
401
|
fake=self.args.get('fake', False),
|
|
@@ -318,16 +403,23 @@ class Command:
|
|
|
318
403
|
shell=shell
|
|
319
404
|
)
|
|
320
405
|
|
|
321
|
-
os.chdir(original_cwd)
|
|
322
406
|
if return_code:
|
|
323
407
|
self.status += return_code
|
|
324
|
-
if stop_on_error:
|
|
325
|
-
|
|
408
|
+
if stop_on_error:
|
|
409
|
+
self.error(f"exec: returned with error (return code: {return_code})")
|
|
410
|
+
else:
|
|
411
|
+
util.debug(f"exec: returned with error (return code: {return_code})")
|
|
326
412
|
else:
|
|
327
413
|
util.debug(f"exec: returned without error (return code: {return_code})")
|
|
328
414
|
return stderr, stdout, return_code
|
|
329
415
|
|
|
330
|
-
def set_arg(
|
|
416
|
+
def set_arg( # pylint: disable=too-many-branches
|
|
417
|
+
self, key, value
|
|
418
|
+
) -> None:
|
|
419
|
+
'''Sets self.args[key] with value, and self.modified_args[key]=True
|
|
420
|
+
|
|
421
|
+
Does type checking, handles list type append
|
|
422
|
+
'''
|
|
331
423
|
|
|
332
424
|
# Do some minimal type handling, preserving the type(self.args[key])
|
|
333
425
|
if key not in self.args:
|
|
@@ -360,14 +452,14 @@ class Command:
|
|
|
360
452
|
else:
|
|
361
453
|
self.args[key] = True
|
|
362
454
|
else:
|
|
363
|
-
|
|
455
|
+
assert False, f'set_arg({key=}, {value=}) bool {type(cur_value)=} {type(value)=}'
|
|
364
456
|
|
|
365
457
|
elif isinstance(cur_value, int):
|
|
366
458
|
# if int, attempt to convert string or bool --> int
|
|
367
459
|
if isinstance(value, (bool, int, str)):
|
|
368
460
|
self.args[key] = int(value)
|
|
369
461
|
else:
|
|
370
|
-
|
|
462
|
+
assert False, f'set_arg({key=}, {value=}) int {type(cur_value)=} {type(value)=}'
|
|
371
463
|
|
|
372
464
|
|
|
373
465
|
else:
|
|
@@ -378,7 +470,9 @@ class Command:
|
|
|
378
470
|
util.debug(f'Set arg["{key}"]="{self.args[key]}"')
|
|
379
471
|
|
|
380
472
|
|
|
381
|
-
def get_argparser(
|
|
473
|
+
def get_argparser( # pylint: disable=too-many-branches
|
|
474
|
+
self, parser_arg_list=None
|
|
475
|
+
) -> argparse.ArgumentParser:
|
|
382
476
|
''' Returns an argparse.ArgumentParser() based on self.args (dict)
|
|
383
477
|
|
|
384
478
|
If parser_arg_list is not None, the ArgumentParser() is created using only the keys in
|
|
@@ -386,8 +480,8 @@ class Command:
|
|
|
386
480
|
'''
|
|
387
481
|
|
|
388
482
|
# Preference is --args-with-dashes, which then become parsed.args_with_dashes, b/c
|
|
389
|
-
# parsed.args-with-dashes is not legal python. Some of self.args.keys() still have - or _,
|
|
390
|
-
# this will handle both.
|
|
483
|
+
# parsed.args-with-dashes is not legal python. Some of self.args.keys() still have - or _,
|
|
484
|
+
# so this will handle both.
|
|
391
485
|
# Also, preference is for self.args.keys(), to be str with - dashes
|
|
392
486
|
parser = argparse.ArgumentParser(prog='eda', add_help=False, allow_abbrev=False)
|
|
393
487
|
bool_action_kwargs = util.get_argparse_bool_action_kwargs()
|
|
@@ -421,19 +515,19 @@ class Command:
|
|
|
421
515
|
help_kwargs = {'help': f'{type(value).__name__} default={value}'}
|
|
422
516
|
|
|
423
517
|
|
|
424
|
-
# It's important to set the default=None on these, except for list types where default
|
|
425
|
-
# If the parsed Namespace has values set to None or [], we do not update. This
|
|
426
|
-
# are processed that have args set, they cannot override the top
|
|
427
|
-
# nor be overriden by defaults.
|
|
428
|
-
if
|
|
429
|
-
# For bool, support --key and --no-key with
|
|
430
|
-
# Note, this means you cannot use --some-bool=True, or --some-bool=False, has to
|
|
431
|
-
# or --no-some-bool.
|
|
518
|
+
# It's important to set the default=None on these, except for list types where default
|
|
519
|
+
# is []. If the parsed Namespace has values set to None or [], we do not update. This
|
|
520
|
+
# means that as deps are processed that have args set, they cannot override the top
|
|
521
|
+
# level args that were already set, nor be overriden by defaults.
|
|
522
|
+
if isinstance(value, bool):
|
|
523
|
+
# For bool, support --key and --no-key with action=argparse.BooleanOptionalAction.
|
|
524
|
+
# Note, this means you cannot use --some-bool=True, or --some-bool=False, has to
|
|
525
|
+
# be --some-bool or --no-some-bool.
|
|
432
526
|
parser.add_argument(
|
|
433
527
|
*arguments, default=None, **bool_action_kwargs, **help_kwargs)
|
|
434
|
-
elif
|
|
528
|
+
elif isinstance(value, list):
|
|
435
529
|
parser.add_argument(*arguments, default=value, action='append', **help_kwargs)
|
|
436
|
-
elif
|
|
530
|
+
elif isinstance(value, (int, str)):
|
|
437
531
|
parser.add_argument(*arguments, default=value, type=type(value), **help_kwargs)
|
|
438
532
|
elif value is None:
|
|
439
533
|
parser.add_argument(*arguments, default=None, **help_kwargs)
|
|
@@ -444,10 +538,10 @@ class Command:
|
|
|
444
538
|
|
|
445
539
|
|
|
446
540
|
def run_argparser_on_list(
|
|
447
|
-
self, tokens: list, parser_arg_list=None, apply_parsed_args:bool=True
|
|
541
|
+
self, tokens: list, parser_arg_list=None, apply_parsed_args: bool = True
|
|
448
542
|
) -> (dict, list):
|
|
449
|
-
''' Creates an argparse.ArgumentParser() for all the keys in self.args, and attempts to
|
|
450
|
-
from the provided list. Parsed args are applied to self.args.
|
|
543
|
+
''' Creates an argparse.ArgumentParser() for all the keys in self.args, and attempts to
|
|
544
|
+
parse from the provided list. Parsed args are applied to self.args.
|
|
451
545
|
|
|
452
546
|
Returns a tuple of (parsed argparse.Namespace obj, list of unparsed args)
|
|
453
547
|
|
|
@@ -461,17 +555,16 @@ class Command:
|
|
|
461
555
|
parsed, unparsed = parser.parse_known_args(tokens + [''])
|
|
462
556
|
unparsed = list(filter(None, unparsed))
|
|
463
557
|
except argparse.ArgumentError:
|
|
464
|
-
self.error(f'problem {command_name=} attempting to parse_known_args for {tokens=}')
|
|
558
|
+
self.error(f'problem {self.command_name=} attempting to parse_known_args for {tokens=}')
|
|
465
559
|
|
|
466
560
|
parsed_as_dict = vars(parsed)
|
|
467
561
|
|
|
468
562
|
args_to_be_applied = {}
|
|
469
563
|
|
|
470
|
-
for key,value in parsed_as_dict.items():
|
|
471
|
-
# key
|
|
472
|
-
# etc.
|
|
564
|
+
for key, value in parsed_as_dict.items():
|
|
565
|
+
# key currently has _ instead of POSIX dashes, so we convert to dashes, for
|
|
566
|
+
# most self.args, like self.args['build-file'], etc.
|
|
473
567
|
if key not in self.args and '_' in key:
|
|
474
|
-
# try with dashes instead of _
|
|
475
568
|
key = key.replace('_', '-')
|
|
476
569
|
assert key in self.args, f'{key=} not in {self.args=}'
|
|
477
570
|
|
|
@@ -483,7 +576,8 @@ class Command:
|
|
|
483
576
|
return parsed, unparsed
|
|
484
577
|
|
|
485
578
|
|
|
486
|
-
def apply_args_from_dict(self, args_to_be_applied:dict) -> list:
|
|
579
|
+
def apply_args_from_dict(self, args_to_be_applied: dict) -> list:
|
|
580
|
+
'''Given a dict of key/value for self.args, apply them to self.args'''
|
|
487
581
|
util.debug('apply_args_from_dict() -- called by:',
|
|
488
582
|
f'{self.command_name=}, {self.__class__.__name__=},',
|
|
489
583
|
f'{args_to_be_applied=}')
|
|
@@ -505,7 +599,7 @@ class Command:
|
|
|
505
599
|
f"with different cur value (cur value={self.args.get(key, None)})")
|
|
506
600
|
continue
|
|
507
601
|
if self.args[key] != value:
|
|
508
|
-
util.debug(
|
|
602
|
+
util.debug("Command.run_argparser_on_list - setting set_arg b/c",
|
|
509
603
|
f" argparse -- {key=} {value=} (cur value={self.args[key]})")
|
|
510
604
|
self.set_arg(key, value) # Note this has special handling for lists already.
|
|
511
605
|
self.modified_args[key] = True
|
|
@@ -523,7 +617,7 @@ class Command:
|
|
|
523
617
|
_, unparsed = self.run_argparser_on_list(tokens)
|
|
524
618
|
if process_all and len(unparsed) > 0:
|
|
525
619
|
self.error(f"Didn't understand argument: '{unparsed=}' in",
|
|
526
|
-
f" {self.command_name=} context")
|
|
620
|
+
f" {self.command_name=} context, {pwd=}")
|
|
527
621
|
|
|
528
622
|
return unparsed
|
|
529
623
|
|
|
@@ -556,27 +650,33 @@ class Command:
|
|
|
556
650
|
|
|
557
651
|
Allows command handlers to make thier own customizations from --config-yaml=YAML.
|
|
558
652
|
'''
|
|
559
|
-
|
|
653
|
+
return
|
|
560
654
|
|
|
561
655
|
def update_tool_config(self) -> None:
|
|
562
656
|
'''Returns None. Hook for classes like CommandSim to make tool specific overrides.'''
|
|
563
|
-
|
|
657
|
+
return
|
|
564
658
|
|
|
565
659
|
def write_eda_config_and_args(self):
|
|
660
|
+
'''Attempts to write eda_output_config.yml to our work-dir'''
|
|
566
661
|
if not self.args.get('work-dir', None):
|
|
567
662
|
util.warning(f'Ouput work-dir not set, saving ouput eda_config to {os.getcwd()}')
|
|
568
|
-
|
|
569
|
-
|
|
663
|
+
eda_config.write_eda_config_and_args(
|
|
664
|
+
dirpath=self.args.get('work-dir', os.getcwd()), command_obj_ref=self
|
|
665
|
+
)
|
|
570
666
|
|
|
571
667
|
def is_export_enabled(self) -> bool:
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
return any(
|
|
668
|
+
'''Returns True if any self.args['export'] is set in any way (but not set to False
|
|
669
|
+
or empty list)'''
|
|
670
|
+
return any(arg.startswith('export') and v for arg,v in self.args.items())
|
|
575
671
|
|
|
576
672
|
def run(self) -> None:
|
|
673
|
+
'''Alias for do_it(self)'''
|
|
577
674
|
self.do_it()
|
|
578
675
|
|
|
579
676
|
def do_it(self) -> None:
|
|
677
|
+
'''Main entrypoint of running a command, usually called by process_tokens.
|
|
678
|
+
|
|
679
|
+
process_tokens(..) is the starting entrypoint from eda.py'''
|
|
580
680
|
self.write_eda_config_and_args()
|
|
581
681
|
self.error(f"No tool bound to command '{self.command_name}', you",
|
|
582
682
|
" probably need to setup tool, or use '--tool <name>'")
|
|
@@ -592,7 +692,9 @@ class Command:
|
|
|
592
692
|
self.set_tool_defines()
|
|
593
693
|
|
|
594
694
|
|
|
595
|
-
def help(
|
|
695
|
+
def help( # pylint: disable=dangerous-default-value,too-many-branches
|
|
696
|
+
self, tokens: list = []
|
|
697
|
+
) -> None:
|
|
596
698
|
'''Since we don't quite follow standard argparger help()/usage(), we'll format our own
|
|
597
699
|
|
|
598
700
|
if self.args_help has additional help information.
|
|
@@ -600,7 +702,7 @@ class Command:
|
|
|
600
702
|
|
|
601
703
|
# Indent long lines (>100) to indent=56 (this is where we leave off w/ {vstr:12} below.
|
|
602
704
|
def indent_me(text:str):
|
|
603
|
-
return
|
|
705
|
+
return indent_wrap_long_text(text, width=100, indent=56)
|
|
604
706
|
|
|
605
707
|
util.info('Help:')
|
|
606
708
|
# using bare 'print' here, since help was requested, avoids --color and --quiet
|
|
@@ -619,14 +721,14 @@ class Command:
|
|
|
619
721
|
lines.append(f"Generic help for command='{self.command_name}'"
|
|
620
722
|
f" (using '{self.__class__.__name__}')")
|
|
621
723
|
else:
|
|
622
|
-
lines.append(
|
|
724
|
+
lines.append("Generic help (from class Command):")
|
|
623
725
|
|
|
624
726
|
# Attempt to run argparser on args, but don't error if it fails.
|
|
625
727
|
unparsed = []
|
|
626
728
|
if tokens:
|
|
627
729
|
try:
|
|
628
730
|
_, unparsed = self.run_argparser_on_list(tokens=tokens)
|
|
629
|
-
except:
|
|
731
|
+
except Exception:
|
|
630
732
|
pass
|
|
631
733
|
|
|
632
734
|
for k in sorted(self.args.keys()):
|
|
@@ -635,13 +737,13 @@ class Command:
|
|
|
635
737
|
khelp = self.args_help.get(k, '')
|
|
636
738
|
if khelp:
|
|
637
739
|
khelp = f' - {khelp}'
|
|
638
|
-
if
|
|
740
|
+
if isinstance(v, bool):
|
|
639
741
|
lines.append(indent_me(f" --{k:20} : boolean : {vstr:12}{khelp}"))
|
|
640
|
-
elif
|
|
742
|
+
elif isinstance(v, int):
|
|
641
743
|
lines.append(indent_me(f" --{k:20} : integer : {vstr:12}{khelp}"))
|
|
642
|
-
elif
|
|
744
|
+
elif isinstance(v, list):
|
|
643
745
|
lines.append(indent_me(f" --{k:20} : list : {vstr:12}{khelp}"))
|
|
644
|
-
elif
|
|
746
|
+
elif isinstance(v, str):
|
|
645
747
|
vstr = "'" + v + "'"
|
|
646
748
|
lines.append(indent_me(f" --{k:20} : string : {vstr:12}{khelp}"))
|
|
647
749
|
else:
|
|
@@ -654,7 +756,10 @@ class Command:
|
|
|
654
756
|
print(f'Unparsed args: {unparsed}')
|
|
655
757
|
|
|
656
758
|
|
|
657
|
-
class CommandDesign(Command):
|
|
759
|
+
class CommandDesign(Command): # pylint: disable=too-many-instance-attributes
|
|
760
|
+
'''CommandDesign is the eda base class for command handlers that need to track files.
|
|
761
|
+
|
|
762
|
+
This is the base class for CommandSim, CommandSynth, and others.'''
|
|
658
763
|
|
|
659
764
|
# Used by for DEPS work_dir_add_srcs@ commands, by class methods:
|
|
660
765
|
# update_file_lists_for_work_dir(..), and resolve_target(..)
|
|
@@ -675,11 +780,16 @@ class CommandDesign(Command):
|
|
|
675
780
|
self.args_help.update({
|
|
676
781
|
'seed': 'design seed, default is 31-bit non-zero urandom',
|
|
677
782
|
'top': 'TOP level verilog/SV module or VHDL entity for this target',
|
|
678
|
-
'all-sv': (
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
783
|
+
'all-sv': (
|
|
784
|
+
'Maintain .sv and .v in single file list.'
|
|
785
|
+
' False: .sv flist separate from .v flist and separate compile(s)'
|
|
786
|
+
' True: .sv and .v files compiled together if possible'
|
|
787
|
+
),
|
|
788
|
+
'unprocessed-plusargs': (
|
|
789
|
+
'Args that began with +, but were not +define+ or +incdir+, +<name>, '
|
|
790
|
+
' or +<name>=<value>. These become tool dependent, for example "sim" commands will'
|
|
791
|
+
' treat as sim-plusargs'
|
|
792
|
+
),
|
|
683
793
|
})
|
|
684
794
|
self.defines = {}
|
|
685
795
|
self.incdirs = []
|
|
@@ -697,12 +807,18 @@ class CommandDesign(Command):
|
|
|
697
807
|
for (d,v) in self.config.get('defines', {}).items():
|
|
698
808
|
self.defines[d] = v
|
|
699
809
|
|
|
700
|
-
|
|
810
|
+
# cached_deps: key = abspath of DEPS markup file, value is a dict with
|
|
811
|
+
# keys 'data' and 'line_numbers'
|
|
812
|
+
self.cached_deps = {}
|
|
701
813
|
self.targets_dict = {} # key = targets that we've already processed in DEPS files
|
|
702
814
|
self.last_added_source_file_inferred_top = ''
|
|
703
815
|
|
|
704
|
-
|
|
705
|
-
|
|
816
|
+
|
|
817
|
+
def run_dep_commands(self) -> None:
|
|
818
|
+
'''Run shell/peakrdl style commands from DEPS files
|
|
819
|
+
|
|
820
|
+
These are deferred to maintain the deps ordering, and run in that order.
|
|
821
|
+
'''
|
|
706
822
|
self.run_dep_shell_commands()
|
|
707
823
|
# Update any work_dir_add_srcs@ in our self.files, self.files_v, etc, b/c
|
|
708
824
|
# self.args['work-dir'] now exists.
|
|
@@ -710,7 +826,10 @@ class CommandDesign(Command):
|
|
|
710
826
|
# Link any non-sources to our work-dir:
|
|
711
827
|
self.update_non_source_files_in_work_dir()
|
|
712
828
|
|
|
713
|
-
|
|
829
|
+
|
|
830
|
+
def run_dep_shell_commands(self) -> None:
|
|
831
|
+
'''Specifically runs shell command from DEPS files'''
|
|
832
|
+
|
|
714
833
|
# Runs from self.args['work-dir']
|
|
715
834
|
all_cmds_lists = []
|
|
716
835
|
|
|
@@ -729,7 +848,8 @@ class CommandDesign(Command):
|
|
|
729
848
|
log_fnames_count.update({target_node: lognum + 1})
|
|
730
849
|
all_cmds_lists += [
|
|
731
850
|
[], # blank line
|
|
732
|
-
# comment, where it came from, log to {node}__shell_{lognum}.log
|
|
851
|
+
# comment, where it came from, log to {node}__shell_{lognum}.log
|
|
852
|
+
# (or tee name from DEPS.yml)
|
|
733
853
|
[f'# command {i}: target: {d["target_path"]} : {target_node} --> {log}'],
|
|
734
854
|
]
|
|
735
855
|
if not run_from_work_dir:
|
|
@@ -743,10 +863,12 @@ class CommandDesign(Command):
|
|
|
743
863
|
|
|
744
864
|
d['exec_list'] = clist # update to tee_fpath is set.
|
|
745
865
|
|
|
746
|
-
util.write_shell_command_file(
|
|
747
|
-
|
|
866
|
+
util.write_shell_command_file(
|
|
867
|
+
dirpath=self.args['work-dir'], filename='pre_compile_dep_shell_commands.sh',
|
|
868
|
+
command_lists=all_cmds_lists
|
|
869
|
+
)
|
|
748
870
|
|
|
749
|
-
for i,d in enumerate(self.dep_shell_commands):
|
|
871
|
+
for i, d in enumerate(self.dep_shell_commands):
|
|
750
872
|
util.info(f'run_dep_shell_commands {i=}: {d=}')
|
|
751
873
|
clist = util.ShellCommandList(d['exec_list'])
|
|
752
874
|
tee_fpath=clist.tee_fpath
|
|
@@ -764,16 +886,23 @@ class CommandDesign(Command):
|
|
|
764
886
|
self.exec(run_from_dir, clist, tee_fpath=tee_fpath,
|
|
765
887
|
shell=self.config.get('deps_subprocess_shell', False))
|
|
766
888
|
|
|
767
|
-
|
|
768
|
-
|
|
889
|
+
|
|
890
|
+
def update_file_lists_for_work_dir(self) -> None:
|
|
891
|
+
'''Handles any source files that were creating by "shell" style commands in the
|
|
892
|
+
|
|
893
|
+
work-dir, these need to be added via self.add_file(..) in the correct order
|
|
894
|
+
of the dependencies. Basically, any files that were left with @EDA-WORK-DIR@ prefix
|
|
895
|
+
need to be patched and added.
|
|
896
|
+
'''
|
|
897
|
+
if not self.dep_work_dir_add_srcs:
|
|
769
898
|
return
|
|
770
899
|
|
|
771
|
-
# If we encounter any @EDA-WORK_DIR@some_file.v in self.files, self.files_v, etc,
|
|
772
|
-
# self.args['work-dir'] / some_file.v:
|
|
900
|
+
# If we encounter any @EDA-WORK_DIR@some_file.v in self.files, self.files_v, etc,
|
|
901
|
+
# then replace it with: self.args['work-dir'] / some_file.v:
|
|
773
902
|
_work_dir_add_srcs_path_string_len = len(self._work_dir_add_srcs_path_string)
|
|
774
903
|
work_dir_abspath = os.path.abspath(self.args['work-dir'])
|
|
775
|
-
for key in list(self.files.keys()): # list so it's not an iterator,
|
|
776
|
-
if
|
|
904
|
+
for key in list(self.files.keys()): # list so it's not an iterator, updates self.files.
|
|
905
|
+
if isinstance(key, str) and key.startswith(self._work_dir_add_srcs_path_string):
|
|
777
906
|
new_key = os.path.join(work_dir_abspath, key[_work_dir_add_srcs_path_string_len :])
|
|
778
907
|
self.files.pop(key)
|
|
779
908
|
self.files[new_key] = True
|
|
@@ -782,12 +911,23 @@ class CommandDesign(Command):
|
|
|
782
911
|
self.files_sdc]
|
|
783
912
|
for my_file_list in my_file_lists_list:
|
|
784
913
|
for i,value in enumerate(my_file_list):
|
|
785
|
-
if value and
|
|
786
|
-
|
|
914
|
+
if value and isinstance(value, str) and \
|
|
915
|
+
value.startswith(self._work_dir_add_srcs_path_string):
|
|
916
|
+
new_value = os.path.join(
|
|
917
|
+
work_dir_abspath, value[_work_dir_add_srcs_path_string_len :]
|
|
918
|
+
)
|
|
787
919
|
my_file_list[i] = new_value
|
|
788
920
|
util.debug(f"file lists: replaced {value} with {new_value}")
|
|
789
921
|
|
|
790
|
-
|
|
922
|
+
|
|
923
|
+
def update_non_source_files_in_work_dir(self) -> None:
|
|
924
|
+
'''For non-source files (that are tracked as 'reqs' in DEPS markup files) these
|
|
925
|
+
|
|
926
|
+
need to be copied or linked to the work-dir. For example, if some SV assumes it
|
|
927
|
+
can $readmemh('file_that_is_here.txt') but we're running out of work-dir. Linking
|
|
928
|
+
is the easy work-around vs trying to run-in-place of all SV files.
|
|
929
|
+
'''
|
|
930
|
+
|
|
791
931
|
for fname in self.files_non_source:
|
|
792
932
|
_, leaf_fname = os.path.split(fname)
|
|
793
933
|
destfile = os.path.join(self.args['work-dir'], leaf_fname)
|
|
@@ -797,16 +937,25 @@ class CommandDesign(Command):
|
|
|
797
937
|
util.info(f'{fname=} {self.files_caller_info=}')
|
|
798
938
|
self.error(f'Non-source file (reqs?) {relfname=} does not exist from {caller_info}')
|
|
799
939
|
elif not os.path.exists(destfile):
|
|
800
|
-
util.debug(f'updating non-source file to work-dir: Linked {fname=} to {destfile=},
|
|
940
|
+
util.debug(f'updating non-source file to work-dir: Linked {fname=} to {destfile=},',
|
|
941
|
+
f'from {caller_info}')
|
|
801
942
|
if sys.platform == "win32":
|
|
802
943
|
shutil.copyfile(fname, destfile) # On Windows, fall back to copying
|
|
803
944
|
else:
|
|
804
945
|
os.symlink(src=fname, dst=destfile)
|
|
805
946
|
|
|
806
|
-
|
|
947
|
+
@staticmethod
|
|
948
|
+
def get_top_name(name: str) -> str:
|
|
949
|
+
'''Attempt to get the 'top' module name from a file, such as path/to/mine.sv will
|
|
950
|
+
|
|
951
|
+
return "mine"'''
|
|
952
|
+
# TODO(drew): Use the helper method in util for this instead to peek in file contents?
|
|
807
953
|
return os.path.splitext(os.path.basename(name))[0]
|
|
808
954
|
|
|
809
|
-
|
|
955
|
+
|
|
956
|
+
def process_plusarg( # pylint: disable=too-many-branches
|
|
957
|
+
self, plusarg: str, pwd: str = os.getcwd()
|
|
958
|
+
) -> None:
|
|
810
959
|
'''Retuns str, parses a +define+, +incdir+, +key=value str; adds to internal.
|
|
811
960
|
|
|
812
961
|
Adds to self.defines, self.incdirs,
|
|
@@ -818,12 +967,12 @@ class CommandDesign(Command):
|
|
|
818
967
|
# args that come from shlex.quote(token), such as:
|
|
819
968
|
# token = '\'+define+OC_ROOT="/foo/bar/opencos"\''
|
|
820
969
|
# So we strip all outer ' or " on the plusarg:
|
|
821
|
-
plusarg =
|
|
970
|
+
plusarg = strip_outer_quotes(plusarg)
|
|
822
971
|
if not pwd:
|
|
823
972
|
pwd = ''
|
|
824
973
|
|
|
825
974
|
if plusarg.startswith('+define+'):
|
|
826
|
-
plusarg = plusarg
|
|
975
|
+
plusarg = plusarg[len('+define+'):]
|
|
827
976
|
m = re.match(r'^(\w+)$', plusarg)
|
|
828
977
|
if m:
|
|
829
978
|
k = m.group(1)
|
|
@@ -831,11 +980,12 @@ class CommandDesign(Command):
|
|
|
831
980
|
util.debug(f"Defined {k}")
|
|
832
981
|
return None
|
|
833
982
|
m = re.match(r'^(\w+)\=(\S+)$', plusarg)
|
|
834
|
-
if not m:
|
|
983
|
+
if not m:
|
|
984
|
+
m = re.match(r'^(\w+)\=(\"[^\"]*\")$', plusarg)
|
|
835
985
|
if m:
|
|
836
986
|
k = m.group(1)
|
|
837
987
|
v = m.group(2)
|
|
838
|
-
if v and
|
|
988
|
+
if v and isinstance(v, str):
|
|
839
989
|
if v.startswith('%PWD%/'):
|
|
840
990
|
v = v.replace('%PWD%', os.path.abspath(pwd))
|
|
841
991
|
if v.startswith('%SEED%'):
|
|
@@ -847,7 +997,7 @@ class CommandDesign(Command):
|
|
|
847
997
|
return None
|
|
848
998
|
|
|
849
999
|
if plusarg.startswith('+incdir+'):
|
|
850
|
-
plusarg = plusarg
|
|
1000
|
+
plusarg = plusarg[len('+incdir+'):]
|
|
851
1001
|
m = re.match(r'^(\S+)$', plusarg)
|
|
852
1002
|
if m:
|
|
853
1003
|
incdir = m.group(1)
|
|
@@ -863,40 +1013,59 @@ class CommandDesign(Command):
|
|
|
863
1013
|
if not self.config.get('bare_plusarg_supported', False):
|
|
864
1014
|
self.error(f"bare plusarg(s) are not supported: {plusarg}'")
|
|
865
1015
|
return None
|
|
866
|
-
|
|
1016
|
+
if plusarg not in self.args['unprocessed-plusargs']:
|
|
867
1017
|
self.args['unprocessed-plusargs'].append(plusarg)
|
|
868
1018
|
# For anything added to unprocessed-plusarg, we have to return it, to let
|
|
869
1019
|
# derived classes have the option to handle it
|
|
870
1020
|
return plusarg
|
|
871
|
-
|
|
872
|
-
|
|
1021
|
+
|
|
1022
|
+
self.error(f"Didn't understand +plusarg: '{plusarg}'")
|
|
873
1023
|
return None
|
|
874
1024
|
|
|
875
1025
|
|
|
876
|
-
def append_shell_commands(self, cmds : list):
|
|
877
|
-
|
|
1026
|
+
def append_shell_commands(self, cmds : list) -> None:
|
|
1027
|
+
''' Given a cmds (list), where each item is a dict with:
|
|
1028
|
+
|
|
1029
|
+
{ 'target_node': str,
|
|
1030
|
+
'target_path': str,
|
|
1031
|
+
'exec_list': list
|
|
1032
|
+
}
|
|
1033
|
+
add to this class's dep_shell_commands for deferred processing
|
|
1034
|
+
'''
|
|
1035
|
+
|
|
878
1036
|
for entry in cmds:
|
|
879
|
-
if entry is None or
|
|
1037
|
+
if entry is None or not isinstance(entry, dict):
|
|
880
1038
|
continue
|
|
881
1039
|
if entry in self.dep_shell_commands:
|
|
882
|
-
# we've already run this exact command (target node, target path, exec list),
|
|
883
|
-
# again
|
|
1040
|
+
# we've already run this exact command (target node, target path, exec list),
|
|
1041
|
+
# don't run it again
|
|
884
1042
|
continue
|
|
885
1043
|
|
|
886
1044
|
assert 'exec_list' in entry, f'{entry=}'
|
|
887
1045
|
util.debug(f'adding - dep_shell_command: {entry=}')
|
|
888
1046
|
self.dep_shell_commands.append(entry)
|
|
889
1047
|
|
|
890
|
-
|
|
891
|
-
|
|
1048
|
+
|
|
1049
|
+
def append_work_dir_add_srcs(self, add_srcs: list, caller_info: str) -> None:
|
|
1050
|
+
'''Given add_srcs (list), where each item is a dict with:
|
|
1051
|
+
|
|
1052
|
+
{ 'target_node': str,
|
|
1053
|
+
'target_path': str,
|
|
1054
|
+
'file_list': list
|
|
1055
|
+
}
|
|
1056
|
+
|
|
1057
|
+
adds files to set self.dep_work_dir_add_src, and call add_file on it, basically
|
|
1058
|
+
resolving adding source files that were generated and residing in the work-dir.
|
|
1059
|
+
'''
|
|
892
1060
|
for entry in add_srcs:
|
|
893
|
-
if entry is None or
|
|
1061
|
+
if entry is None or not isinstance(entry, dict):
|
|
894
1062
|
continue
|
|
895
1063
|
|
|
896
1064
|
work_dir_files = entry['file_list']
|
|
897
1065
|
for filename in work_dir_files:
|
|
898
|
-
# Unfortunately, self.args['work-dir'] doesn't exist yet and hasn't been set,
|
|
899
|
-
# files as '@EDA-WORK_DIR@' + filename, and have to replace
|
|
1066
|
+
# Unfortunately, self.args['work-dir'] doesn't exist yet and hasn't been set,
|
|
1067
|
+
# so we'll add these files as '@EDA-WORK_DIR@' + filename, and have to replace
|
|
1068
|
+
# the EDA-WORK_DIR@ string later in our flow.
|
|
900
1069
|
filename_use = self._work_dir_add_srcs_path_string + filename
|
|
901
1070
|
dep_key_tuple = (
|
|
902
1071
|
entry['target_path'],
|
|
@@ -911,14 +1080,25 @@ class CommandDesign(Command):
|
|
|
911
1080
|
self.dep_work_dir_add_srcs.add(dep_key_tuple) # add to set()
|
|
912
1081
|
elif dep_key_tuple not in self.dep_work_dir_add_srcs:
|
|
913
1082
|
# we've already added the file so this dep was skipped for this one file.
|
|
914
|
-
util.warning(f'work_dir_add_srcs@ {dep_key_tuple=} but {filename_use=}'
|
|
915
|
-
|
|
1083
|
+
util.warning(f'work_dir_add_srcs@ {dep_key_tuple=} but {filename_use=}',
|
|
1084
|
+
'is already in self.files (duplicate dependency on generated',
|
|
1085
|
+
'file?)')
|
|
1086
|
+
|
|
916
1087
|
|
|
1088
|
+
def resolve_target(
|
|
1089
|
+
self, target: str, no_recursion: bool = False, caller_info: str = ''
|
|
1090
|
+
) -> bool:
|
|
1091
|
+
'''Returns True if target is found. Entry point for resolving a CLI path based target name.
|
|
917
1092
|
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
1093
|
+
Will recursively call resolve_target_core, looking for a filename existing
|
|
1094
|
+
or DEPS markup file in that path and a target (key) in the markup table.
|
|
1095
|
+
|
|
1096
|
+
This will populate all source file dependencies, commands, and other DEPS markup
|
|
1097
|
+
features as the target is resolved.
|
|
1098
|
+
'''
|
|
1099
|
+
util.debug(f"Entered resolve_target({target})")
|
|
1100
|
+
# self.target is a name we grab for the job (i.e. for naming work dir etc). we don't want
|
|
1101
|
+
# the path prefix.
|
|
922
1102
|
|
|
923
1103
|
self.target_path, self.target = os.path.split(target)
|
|
924
1104
|
|
|
@@ -932,7 +1112,7 @@ class CommandDesign(Command):
|
|
|
932
1112
|
# If the target is a file (we're at the root here processing CLI arg tokens)
|
|
933
1113
|
# and that file exists and has an extension, then there's no reason to go looking
|
|
934
1114
|
# in DEPS files, add the file and return True.
|
|
935
|
-
|
|
1115
|
+
_, file_ext = os.path.splitext(fpath)
|
|
936
1116
|
if forced_extension or file_ext:
|
|
937
1117
|
self.add_file(fpath, caller_info=caller_info,
|
|
938
1118
|
forced_extension=forced_extension)
|
|
@@ -940,23 +1120,25 @@ class CommandDesign(Command):
|
|
|
940
1120
|
|
|
941
1121
|
return self.resolve_target_core(target, no_recursion, caller_info)
|
|
942
1122
|
|
|
943
|
-
def resolve_target_core(
|
|
944
|
-
|
|
1123
|
+
def resolve_target_core( # pylint: disable=too-many-locals,too-many-branches
|
|
1124
|
+
self, target: str, no_recursion: bool, caller_info: str = ''
|
|
1125
|
+
) -> bool:
|
|
1126
|
+
'''Returns True if target is found. recursive point for resolving path or DEPS markup
|
|
1127
|
+
target names.'''
|
|
1128
|
+
|
|
1129
|
+
util.debug(f"Entered resolve_target_core({target=})")
|
|
945
1130
|
found_target = False
|
|
946
|
-
util.debug("Starting to resolve target '%s'" % (target))
|
|
947
1131
|
target_path, target_node = os.path.split(target)
|
|
948
1132
|
|
|
949
|
-
deps = None
|
|
950
|
-
data = None
|
|
1133
|
+
deps, data, deps_file = None, None, None
|
|
951
1134
|
found_deps_file = False
|
|
952
1135
|
|
|
953
1136
|
if self.config['deps_markup_supported']:
|
|
954
|
-
deps =
|
|
1137
|
+
deps = DepsFile(
|
|
955
1138
|
command_design_ref=self, target_path=target_path, cache=self.cached_deps
|
|
956
1139
|
)
|
|
957
1140
|
deps_file = deps.deps_file
|
|
958
1141
|
data = deps.data
|
|
959
|
-
data_only_lines = deps.line_numbers
|
|
960
1142
|
|
|
961
1143
|
# Continue if we have data, otherwise look for files other than DEPS.<yml|yaml>
|
|
962
1144
|
if data is not None:
|
|
@@ -969,7 +1151,7 @@ class CommandDesign(Command):
|
|
|
969
1151
|
|
|
970
1152
|
# For convenience, use an external class for this DEPS.yml table/dict
|
|
971
1153
|
# This could be re-used for any markup DEPS.json, DEPS.toml, DEPS.py, etc.
|
|
972
|
-
deps_processor =
|
|
1154
|
+
deps_processor = DepsProcessor(
|
|
973
1155
|
command_design_ref = self,
|
|
974
1156
|
deps_entry = entry,
|
|
975
1157
|
target = target,
|
|
@@ -990,10 +1172,11 @@ class CommandDesign(Command):
|
|
|
990
1172
|
# Recurse on the returned deps (ordered list), if they haven't already been traversed.
|
|
991
1173
|
for x in deps_targets_to_resolve:
|
|
992
1174
|
caller_info = deps.gen_caller_info(target_node)
|
|
993
|
-
if x and
|
|
1175
|
+
if x and isinstance(x, tuple):
|
|
994
1176
|
# if deps_processor.process_deps_entry() gave us a tuple, it's an
|
|
995
1177
|
# unprocessed 'command' that we kept in order until now. Append it.
|
|
996
|
-
assert len(x) == 2,
|
|
1178
|
+
assert len(x) == 2, \
|
|
1179
|
+
f'command tuple {x=} must be len 2, {target_node=} {deps_file=}'
|
|
997
1180
|
shell_commands_list, work_dir_add_srcs_list = x
|
|
998
1181
|
self.append_shell_commands( cmds=shell_commands_list )
|
|
999
1182
|
self.append_work_dir_add_srcs( add_srcs=work_dir_add_srcs_list,
|
|
@@ -1007,32 +1190,40 @@ class CommandDesign(Command):
|
|
|
1007
1190
|
forced_extension=forced_extension)
|
|
1008
1191
|
else:
|
|
1009
1192
|
util.debug(f' ... Calling resolve_target_core({x=})')
|
|
1010
|
-
found_target |= self.resolve_target_core(
|
|
1193
|
+
found_target |= self.resolve_target_core(
|
|
1194
|
+
x, no_recursion, caller_info=caller_info
|
|
1195
|
+
)
|
|
1011
1196
|
|
|
1012
1197
|
|
|
1013
1198
|
# Done with DEPS.yml if it existed.
|
|
1014
1199
|
|
|
1015
1200
|
if not found_target:
|
|
1016
|
-
util.debug("Haven't been able to resolve
|
|
1201
|
+
util.debug(f"Haven't been able to resolve {target=} via DEPS")
|
|
1017
1202
|
known_file_extensions_for_source = []
|
|
1018
|
-
for x in
|
|
1019
|
-
known_file_extensions_for_source += self.config.get(
|
|
1203
|
+
for x in ('verilog', 'systemverilog', 'vhdl', 'cpp'):
|
|
1204
|
+
known_file_extensions_for_source += self.config.get(
|
|
1205
|
+
'file_extensions', {}).get(x, [])
|
|
1020
1206
|
for e in known_file_extensions_for_source:
|
|
1021
1207
|
try_file = target + e
|
|
1022
|
-
util.debug("Looking for
|
|
1208
|
+
util.debug(f"Looking for file {try_file}")
|
|
1023
1209
|
if os.path.exists(try_file):
|
|
1024
|
-
self.add_file(try_file, caller_info=
|
|
1210
|
+
self.add_file(try_file, caller_info=f'n/a::{target}::n/a')
|
|
1025
1211
|
found_target = True
|
|
1026
1212
|
break # move on to the next target
|
|
1027
1213
|
if not found_target: # if STILL not found_this_target...
|
|
1028
|
-
self.error("Unable to resolve target
|
|
1214
|
+
self.error(f"Unable to resolve {target=}")
|
|
1029
1215
|
|
|
1030
1216
|
# if we've found any target since being called, it means we found the one we were called for
|
|
1031
1217
|
return found_target
|
|
1032
1218
|
|
|
1033
|
-
def add_file(
|
|
1034
|
-
|
|
1035
|
-
|
|
1219
|
+
def add_file(
|
|
1220
|
+
self, filename: str, use_abspath: bool = True, add_to_non_sources: bool = False,
|
|
1221
|
+
caller_info: str = '', forced_extension: str = ''
|
|
1222
|
+
) -> str:
|
|
1223
|
+
'''Given a filename, add it to one of self.files_sv or similar lists
|
|
1224
|
+
|
|
1225
|
+
based on file extension or prefix directive.'''
|
|
1226
|
+
_, file_ext = os.path.splitext(filename)
|
|
1036
1227
|
if use_abspath:
|
|
1037
1228
|
file_abspath = os.path.abspath(filename)
|
|
1038
1229
|
else:
|
|
@@ -1040,8 +1231,8 @@ class CommandDesign(Command):
|
|
|
1040
1231
|
|
|
1041
1232
|
|
|
1042
1233
|
if file_abspath in self.files:
|
|
1043
|
-
util.debug("Not adding file
|
|
1044
|
-
return
|
|
1234
|
+
util.debug(f"Not adding file {file_abspath}, already have it")
|
|
1235
|
+
return ''
|
|
1045
1236
|
|
|
1046
1237
|
known_file_ext_dict = self.config.get('file_extensions', {})
|
|
1047
1238
|
v_file_ext_list = known_file_ext_dict.get('verilog', [])
|
|
@@ -1064,34 +1255,43 @@ class CommandDesign(Command):
|
|
|
1064
1255
|
|
|
1065
1256
|
if add_to_non_sources:
|
|
1066
1257
|
self.files_non_source.append(file_abspath)
|
|
1067
|
-
util.debug("Added non-source file file
|
|
1258
|
+
util.debug(f"Added non-source file file {filename} as {file_abspath}")
|
|
1068
1259
|
elif file_ext in v_file_ext_list and not self.args['all-sv']:
|
|
1069
1260
|
self.files_v.append(file_abspath)
|
|
1070
|
-
util.debug("Added Verilog file
|
|
1071
|
-
elif file_ext in sv_file_ext_list or
|
|
1261
|
+
util.debug(f"Added Verilog file {filename} as {file_abspath}")
|
|
1262
|
+
elif file_ext in sv_file_ext_list or \
|
|
1263
|
+
((file_ext in v_file_ext_list) and self.args['all-sv']):
|
|
1072
1264
|
self.files_sv.append(file_abspath)
|
|
1073
|
-
util.debug("Added SystemVerilog file
|
|
1265
|
+
util.debug(f"Added SystemVerilog file {filename} as {file_abspath}")
|
|
1074
1266
|
elif file_ext in vhdl_file_ext_list:
|
|
1075
1267
|
self.files_vhd.append(file_abspath)
|
|
1076
|
-
util.debug("Added VHDL file
|
|
1268
|
+
util.debug(f"Added VHDL file {filename} as {file_abspath}")
|
|
1077
1269
|
elif file_ext in cpp_file_ext_list:
|
|
1078
1270
|
self.files_cpp.append(file_abspath)
|
|
1079
|
-
util.debug("Added C++ file
|
|
1271
|
+
util.debug(f"Added C++ file {filename} as {file_abspath}")
|
|
1080
1272
|
elif file_ext in sdc_file_ext_list:
|
|
1081
1273
|
self.files_sdc.append(file_abspath)
|
|
1082
|
-
util.debug("Added SDC file
|
|
1274
|
+
util.debug(f"Added SDC file {filename} as {file_abspath}")
|
|
1083
1275
|
else:
|
|
1084
1276
|
# unknown file extension. In these cases we link the file to the working directory
|
|
1085
|
-
# so it is available (for example, a .mem file that is expected to exist with relative
|
|
1277
|
+
# so it is available (for example, a .mem file that is expected to exist with relative
|
|
1278
|
+
# path)
|
|
1086
1279
|
self.files_non_source.append(file_abspath)
|
|
1087
|
-
util.debug("Added non-source file
|
|
1280
|
+
util.debug(f"Added non-source file {filename} as {file_abspath}")
|
|
1088
1281
|
|
|
1089
1282
|
self.files[file_abspath] = True
|
|
1090
1283
|
self.files_caller_info[file_abspath] = caller_info
|
|
1091
1284
|
return file_abspath
|
|
1092
1285
|
|
|
1093
|
-
def process_tokens(
|
|
1094
|
-
|
|
1286
|
+
def process_tokens( # pylint: disable=too-many-locals,too-many-branches,too-many-statements
|
|
1287
|
+
self, tokens: list, process_all: bool = True,
|
|
1288
|
+
pwd: str = os.getcwd()
|
|
1289
|
+
) -> list:
|
|
1290
|
+
'''Returns a list of unparsed args (self.args did not have these keys present)
|
|
1291
|
+
|
|
1292
|
+
Entrypoint from eda.py calling a command handler obj.process_tokens(args).
|
|
1293
|
+
Dervied classes are expected to call this parent process_tokens to parse and handle
|
|
1294
|
+
their self.args dict'''
|
|
1095
1295
|
|
|
1096
1296
|
util.debug(f'CommandDesign - process_tokens start - {tokens=}')
|
|
1097
1297
|
|
|
@@ -1105,19 +1305,21 @@ class CommandDesign(Command):
|
|
|
1105
1305
|
# walk the list, remove all items after we're done.
|
|
1106
1306
|
remove_list = []
|
|
1107
1307
|
for token in unparsed:
|
|
1108
|
-
# Since this is a raw argparser, we may have args that come from shlex.quote(token),
|
|
1109
|
-
#
|
|
1110
|
-
#
|
|
1308
|
+
# Since this is a raw argparser, we may have args that come from shlex.quote(token),
|
|
1309
|
+
# such as:
|
|
1310
|
+
# token = '\'+define+OC_ROOT="/foo/bar/opencos"\''
|
|
1311
|
+
# So we have to check for strings that have been escaped for shell with extra single
|
|
1312
|
+
# quotes.
|
|
1111
1313
|
m = re.match(r"^\'?\+\w+", token)
|
|
1112
1314
|
if m:
|
|
1113
1315
|
# Copy and strip all outer ' or " on the plusarg:
|
|
1114
|
-
plusarg =
|
|
1316
|
+
plusarg = strip_outer_quotes(token)
|
|
1115
1317
|
self.process_plusarg(plusarg, pwd=pwd)
|
|
1116
1318
|
remove_list.append(token)
|
|
1117
1319
|
for x in remove_list:
|
|
1118
1320
|
unparsed.remove(x)
|
|
1119
1321
|
|
|
1120
|
-
if
|
|
1322
|
+
if not unparsed and self.error_on_no_files_or_targets:
|
|
1121
1323
|
# derived classes can set error_on_no_files_or_targets=True
|
|
1122
1324
|
# For example: CommandSim will error (requires files/targets),
|
|
1123
1325
|
# CommandWaves does not (files/targets not required)
|
|
@@ -1125,11 +1327,11 @@ class CommandDesign(Command):
|
|
|
1125
1327
|
# check the DEPS markup file for a single target, and if so run it
|
|
1126
1328
|
# on the one and only target.
|
|
1127
1329
|
if self.config['deps_markup_supported']:
|
|
1128
|
-
deps =
|
|
1330
|
+
deps = DepsFile(
|
|
1129
1331
|
command_design_ref=self, target_path=os.getcwd(), cache=self.cached_deps
|
|
1130
1332
|
)
|
|
1131
1333
|
if deps.deps_file and deps.data:
|
|
1132
|
-
all_targets =
|
|
1334
|
+
all_targets = deps_data_get_all_targets(deps.data)
|
|
1133
1335
|
if all_targets:
|
|
1134
1336
|
target = all_targets[-1]
|
|
1135
1337
|
unparsed.append(target)
|
|
@@ -1151,7 +1353,7 @@ class CommandDesign(Command):
|
|
|
1151
1353
|
file_exists, fpath, forced_extension = files.get_source_file(token)
|
|
1152
1354
|
if file_exists:
|
|
1153
1355
|
file_abspath = os.path.abspath(fpath)
|
|
1154
|
-
|
|
1356
|
+
_, file_ext = os.path.splitext(file_abspath)
|
|
1155
1357
|
if not forced_extension and not file_ext:
|
|
1156
1358
|
# This probably isn't a file we want to use
|
|
1157
1359
|
util.warning(f'looking for deps {token=}, found {file_abspath=}' \
|
|
@@ -1171,11 +1373,12 @@ class CommandDesign(Command):
|
|
|
1171
1373
|
remove_list.append(token)
|
|
1172
1374
|
continue # done with token, consume it, we added the file.
|
|
1173
1375
|
|
|
1174
|
-
# we appear to be dealing with a target name which needs to be resolved (usually
|
|
1376
|
+
# we appear to be dealing with a target name which needs to be resolved (usually
|
|
1377
|
+
# recursively)
|
|
1175
1378
|
if token.startswith(os.sep):
|
|
1176
1379
|
target_name = token # if it's absolute path, don't prepend anything
|
|
1177
1380
|
else:
|
|
1178
|
-
target_name = os.path.join(".", token) # prepend ./
|
|
1381
|
+
target_name = os.path.join(".", token) # prepend ./ to make it like: <path>/<file>
|
|
1179
1382
|
|
|
1180
1383
|
util.debug(f'Calling self.resolve_target on {target_name=} ({token=})')
|
|
1181
1384
|
if self.resolve_target(target_name, caller_info=caller_info):
|
|
@@ -1224,49 +1427,81 @@ class CommandDesign(Command):
|
|
|
1224
1427
|
self.target = self.args['top']
|
|
1225
1428
|
|
|
1226
1429
|
if self.error_on_missing_top and not self.args.get('top', ''):
|
|
1227
|
-
self.error(
|
|
1430
|
+
self.error("Did not get a --top or DEPS top, required to run command",
|
|
1228
1431
|
f"'{self.command_name}' for tool={self.args.get('tool', None)}")
|
|
1229
1432
|
|
|
1230
1433
|
return unparsed
|
|
1231
1434
|
|
|
1232
1435
|
|
|
1233
|
-
def get_command_line_args(
|
|
1436
|
+
def get_command_line_args( # pylint: disable=dangerous-default-value
|
|
1437
|
+
self, remove_args: list = [], remove_args_startswith: list = []
|
|
1438
|
+
) -> list:
|
|
1234
1439
|
'''Returns a list of all the args if you wanted to re-run this command
|
|
1235
1440
|
(excludes eda, command, target).'''
|
|
1236
1441
|
|
|
1237
1442
|
# This will not set bool's that are False, does not add --no-<somearg>
|
|
1238
|
-
# nor --<somearg>=False
|
|
1239
|
-
# This will not set str's that are empty
|
|
1240
|
-
# this will not set ints that are 0
|
|
1443
|
+
# nor --<somearg>=False (it's unclear which are 'store_true' v/ bool-action-kwargs)
|
|
1444
|
+
# This will not set str's that are empty
|
|
1445
|
+
# this will not set ints that are 0 (unless modified)
|
|
1446
|
+
# TODO(drew): we may want to revisit this if we tracked default arg values,
|
|
1447
|
+
# or perhaps if they have been "modified".
|
|
1241
1448
|
ret = []
|
|
1242
1449
|
for k,v in self.args.items():
|
|
1243
1450
|
|
|
1244
1451
|
# Some args cannot be extracted and work, so omit these:
|
|
1245
1452
|
if k in ['top-path'] + remove_args:
|
|
1246
1453
|
continue
|
|
1247
|
-
if any(
|
|
1454
|
+
if any(k.startswith(x) for x in remove_args_startswith):
|
|
1248
1455
|
continue
|
|
1249
1456
|
|
|
1250
|
-
|
|
1457
|
+
is_modified = self.modified_args.get(k, False)
|
|
1458
|
+
|
|
1459
|
+
if isinstance(v, bool) and v:
|
|
1251
1460
|
ret.append(f'--{k}')
|
|
1252
|
-
elif
|
|
1461
|
+
elif isinstance(v, int) and (bool(v) or is_modified):
|
|
1253
1462
|
ret.append(f'--{k}={v}')
|
|
1254
|
-
elif
|
|
1463
|
+
elif isinstance(v, str) and v:
|
|
1255
1464
|
ret.append(f'--{k}={v}')
|
|
1256
|
-
elif
|
|
1465
|
+
elif isinstance(v, list):
|
|
1257
1466
|
for item in v:
|
|
1258
|
-
if item or
|
|
1467
|
+
if item or isinstance(item, (bool, str)):
|
|
1259
1468
|
# don't print bool/str that are blank.
|
|
1260
1469
|
ret.append(f'--{k}={item}') # lists append
|
|
1261
1470
|
|
|
1262
1471
|
return ret
|
|
1263
1472
|
|
|
1264
1473
|
|
|
1265
|
-
|
|
1266
|
-
|
|
1474
|
+
#_THREADS_START = 0
|
|
1475
|
+
#_THREADS_DONE = 0
|
|
1476
|
+
|
|
1477
|
+
class ThreadStats:
|
|
1478
|
+
'''To avoid globals for two ints, keep a class holder so CommandParallel and
|
|
1479
|
+
CommandParallelWorker can share values'''
|
|
1480
|
+
|
|
1481
|
+
done = 0
|
|
1482
|
+
started = 0
|
|
1483
|
+
|
|
1484
|
+
def all_done(self) -> bool:
|
|
1485
|
+
'''Returns True if all started jobs are done'''
|
|
1486
|
+
return self.started == self.done
|
|
1487
|
+
|
|
1488
|
+
def get_remaining(self) -> int:
|
|
1489
|
+
'''Returns remaining number of jobs'''
|
|
1490
|
+
return max(self.started - self.done, 0)
|
|
1491
|
+
|
|
1492
|
+
def __repr__(self) -> str:
|
|
1493
|
+
'''Pretty print DONE/STARTED'''
|
|
1494
|
+
return f'{self.done}/{self.started}'
|
|
1495
|
+
|
|
1267
1496
|
|
|
1268
1497
|
class CommandParallelWorker(threading.Thread):
|
|
1269
|
-
|
|
1498
|
+
'''Helper class for a single threaded job, often run by a CommandParallel command
|
|
1499
|
+
handler class'''
|
|
1500
|
+
|
|
1501
|
+
def __init__(
|
|
1502
|
+
self, n, work_queue: queue.Queue, done_queue: queue.Queue,
|
|
1503
|
+
threads_stats: ThreadStats
|
|
1504
|
+
):
|
|
1270
1505
|
threading.Thread.__init__(self)
|
|
1271
1506
|
self.n = n
|
|
1272
1507
|
self.work_queue = work_queue
|
|
@@ -1276,54 +1511,46 @@ class CommandParallelWorker(threading.Thread):
|
|
|
1276
1511
|
self.proc = None
|
|
1277
1512
|
self.pid = None
|
|
1278
1513
|
self.last_timer_debug = 0
|
|
1514
|
+
self.threads_stats = threads_stats # ref to shared object
|
|
1279
1515
|
util.debug(f"WORKER_{n}: START")
|
|
1280
1516
|
|
|
1281
1517
|
def run(self):
|
|
1282
|
-
|
|
1283
|
-
|
|
1518
|
+
'''Runs this single job via threading. This is typically created and called by
|
|
1519
|
+
|
|
1520
|
+
a CommandParallel class handler
|
|
1521
|
+
'''
|
|
1522
|
+
|
|
1284
1523
|
while True:
|
|
1285
1524
|
# Get the work from the queue and expand the tuple
|
|
1286
|
-
i, command_list, job_name,
|
|
1525
|
+
i, command_list, job_name, _ = self.work_queue.get()
|
|
1287
1526
|
self.job_name = job_name
|
|
1288
1527
|
try:
|
|
1289
1528
|
util.debug(f"WORKER_{self.n}: Running job {i}: {job_name}")
|
|
1290
|
-
PIPE=subprocess.PIPE
|
|
1291
|
-
STDOUT=subprocess.STDOUT
|
|
1292
1529
|
util.debug(f"WORKER_{self.n}: Calling Popen")
|
|
1293
|
-
proc = subprocess.Popen(
|
|
1530
|
+
proc = subprocess.Popen( # pylint: disable=consider-using-with
|
|
1531
|
+
command_list, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
|
|
1532
|
+
)
|
|
1294
1533
|
self.proc = proc
|
|
1295
1534
|
util.debug(f"WORKER_{self.n}: Opened process, PID={proc.pid}")
|
|
1296
1535
|
self.pid = proc.pid
|
|
1297
|
-
|
|
1298
|
-
while proc.returncode
|
|
1536
|
+
self.threads_stats.started += 1
|
|
1537
|
+
while proc.returncode is None:
|
|
1299
1538
|
try:
|
|
1300
1539
|
if (time.time() - self.last_timer_debug) > 10:
|
|
1301
1540
|
util.debug(f"WORKER_{self.n}: Calling proc.communicate")
|
|
1302
1541
|
stdout, stderr = proc.communicate(timeout=0.5)
|
|
1303
|
-
util.debug(f"WORKER_{self.n}: got: \n*** stdout:\n{stdout}\n***
|
|
1542
|
+
util.debug(f"WORKER_{self.n}: got: \n*** stdout:\n{stdout}\n***",
|
|
1543
|
+
f"stderr:{stderr}")
|
|
1304
1544
|
except subprocess.TimeoutExpired:
|
|
1305
1545
|
if (time.time() - self.last_timer_debug) > 10:
|
|
1306
|
-
util.debug(f"WORKER_{self.n}: Timer expired, stop_request=
|
|
1546
|
+
util.debug(f"WORKER_{self.n}: Timer expired, stop_request=",
|
|
1547
|
+
f"{self.stop_request}")
|
|
1307
1548
|
self.last_timer_debug = time.time()
|
|
1308
|
-
pass
|
|
1309
1549
|
if self.stop_request:
|
|
1310
1550
|
util.debug(f"WORKER_{self.n}: got stop request, issuing SIGINT")
|
|
1311
1551
|
proc.send_signal(signal.SIGINT)
|
|
1312
1552
|
util.debug(f"WORKER_{self.n}: got stop request, calling proc.wait")
|
|
1313
1553
|
proc.wait()
|
|
1314
|
-
if False and self.stop_request:
|
|
1315
|
-
util.debug(f"WORKER_{self.n}: got stop request, issuing proc.terminate")
|
|
1316
|
-
proc.terminate()
|
|
1317
|
-
util.debug(f"WORKER_{self.n}: proc poll returns is now {proc.poll()}")
|
|
1318
|
-
try:
|
|
1319
|
-
util.debug(f"WORKER_{self.n}: Calling proc.communicate")
|
|
1320
|
-
stdout, stderr = proc.communicate(timeout=0.2) # for completeness, in case we ever pipe/search stdout/stderr
|
|
1321
|
-
util.debug(f"WORKER_{self.n}: got: \n*** stdout:\n{stdout}\n*** stderr:{stderr}")
|
|
1322
|
-
except subprocess.TimeoutExpired:
|
|
1323
|
-
util.debug(f"WORKER_{self.n}: timeout waiting for comminicate after terminate")
|
|
1324
|
-
except:
|
|
1325
|
-
pass
|
|
1326
|
-
util.debug(f"WORKER_{self.n}: proc poll returns is now {proc.poll()}")
|
|
1327
1554
|
|
|
1328
1555
|
util.debug(f"WORKER_{self.n}: -- out of while loop")
|
|
1329
1556
|
self.pid = None
|
|
@@ -1332,99 +1559,113 @@ class CommandParallelWorker(threading.Thread):
|
|
|
1332
1559
|
util.debug(f"WORKER_{self.n}: proc poll returns is now {proc.poll()}")
|
|
1333
1560
|
try:
|
|
1334
1561
|
util.debug(f"WORKER_{self.n}: Calling proc.communicate one last time")
|
|
1335
|
-
stdout, stderr = proc.communicate(timeout=0.1)
|
|
1336
|
-
util.debug(f"WORKER_{self.n}: got: \n*** stdout:\n{stdout}\n***
|
|
1562
|
+
stdout, stderr = proc.communicate(timeout=0.1)
|
|
1563
|
+
util.debug(f"WORKER_{self.n}: got: \n*** stdout:\n{stdout}\n***",
|
|
1564
|
+
f"stderr:{stderr}")
|
|
1337
1565
|
except subprocess.TimeoutExpired:
|
|
1338
1566
|
util.debug(f"WORKER_{self.n}: timeout waiting for communicate after loop?")
|
|
1339
|
-
except:
|
|
1340
|
-
|
|
1567
|
+
except Exception as e:
|
|
1568
|
+
util.debug(f"WORKER_{self.n}: timeout with exception {e=}")
|
|
1569
|
+
|
|
1341
1570
|
return_code = proc.poll()
|
|
1342
|
-
util.debug(f"WORKER_{self.n}: Finished job {i}: {job_name} with return code
|
|
1571
|
+
util.debug(f"WORKER_{self.n}: Finished job {i}: {job_name} with return code",
|
|
1572
|
+
f"{return_code}")
|
|
1343
1573
|
self.done_queue.put((i, job_name, return_code))
|
|
1344
1574
|
finally:
|
|
1345
1575
|
util.debug(f"WORKER_{self.n}: -- in finally block")
|
|
1346
1576
|
self.work_queue.task_done()
|
|
1347
|
-
|
|
1577
|
+
self.threads_stats.done += 1
|
|
1348
1578
|
|
|
1349
1579
|
|
|
1350
1580
|
class CommandParallel(Command):
|
|
1581
|
+
'''Base class command handler for running multiple eda jobs, such as for child classes
|
|
1582
|
+
|
|
1583
|
+
in commands.multi or commands.sweep
|
|
1584
|
+
'''
|
|
1351
1585
|
def __init__(self, config: dict):
|
|
1352
1586
|
Command.__init__(self, config=config)
|
|
1353
1587
|
self.jobs = []
|
|
1354
1588
|
self.jobs_status = []
|
|
1355
1589
|
self.args['parallel'] = 1
|
|
1356
1590
|
self.worker_threads = []
|
|
1591
|
+
self.threads_stats = ThreadStats()
|
|
1357
1592
|
|
|
1358
|
-
def __del__(self):
|
|
1359
|
-
util.debug(f"In Command.__del__, threads done/started: {
|
|
1360
|
-
if
|
|
1593
|
+
def __del__(self): # pylint: disable=too-many-branches
|
|
1594
|
+
util.debug(f"In Command.__del__, threads done/started: {self.threads_stats}")
|
|
1595
|
+
if self.threads_stats.all_done():
|
|
1361
1596
|
return
|
|
1362
|
-
util.warning(f"Need to shut down {
|
|
1597
|
+
util.warning(f"Need to shut down {self.threads_stats.get_remaining()} worker threads...")
|
|
1363
1598
|
for w in self.worker_threads:
|
|
1364
1599
|
if w.proc:
|
|
1365
1600
|
util.warning(f"Requesting stop of PID {w.pid}: {w.job_name}")
|
|
1366
1601
|
w.stop_request = True
|
|
1367
|
-
for
|
|
1368
|
-
util.debug(f"Threads done/started: {
|
|
1369
|
-
if
|
|
1370
|
-
util.info(
|
|
1602
|
+
for _ in range(10):
|
|
1603
|
+
util.debug(f"Threads done/started: {self.threads_stats}")
|
|
1604
|
+
if self.threads_stats.started == self.threads_stats.done:
|
|
1605
|
+
util.info("All threads done")
|
|
1371
1606
|
return
|
|
1372
1607
|
time.sleep(1)
|
|
1373
|
-
subprocess.Popen(['stty', 'sane']).wait()
|
|
1374
|
-
util.debug(
|
|
1608
|
+
subprocess.Popen(['stty', 'sane']).wait() # pylint: disable=consider-using-with
|
|
1609
|
+
util.debug("Scanning workers again")
|
|
1375
1610
|
for w in self.worker_threads:
|
|
1376
1611
|
if w.proc:
|
|
1377
1612
|
util.info(f"need to SIGINT WORKER_{w.n}, may need manual cleanup, check 'ps'")
|
|
1378
1613
|
if w.pid:
|
|
1379
1614
|
os.kill(w.pid, signal.SIGINT)
|
|
1380
|
-
for
|
|
1381
|
-
util.debug(f"Threads done/started: {
|
|
1382
|
-
if
|
|
1383
|
-
util.info(
|
|
1615
|
+
for _ in range(5):
|
|
1616
|
+
util.debug(f"Threads done/started: {self.threads_stats}")
|
|
1617
|
+
if self.threads_stats.all_done():
|
|
1618
|
+
util.info("All threads done")
|
|
1384
1619
|
return
|
|
1385
1620
|
time.sleep(1)
|
|
1386
|
-
subprocess.Popen(['stty', 'sane']).wait()
|
|
1387
|
-
util.debug(
|
|
1621
|
+
subprocess.Popen(['stty', 'sane']).wait() # pylint: disable=consider-using-with
|
|
1622
|
+
util.debug("Scanning workers again")
|
|
1388
1623
|
for w in self.worker_threads:
|
|
1389
1624
|
if w.proc:
|
|
1390
1625
|
util.info(f"need to TERM WORKER_{w.n}, probably needs manual cleanup, check 'ps'")
|
|
1391
1626
|
if w.pid:
|
|
1392
1627
|
os.kill(w.pid, signal.SIGTERM)
|
|
1393
|
-
for
|
|
1394
|
-
util.debug(f"Threads done/started: {
|
|
1395
|
-
if
|
|
1396
|
-
util.info(
|
|
1628
|
+
for _ in range(5):
|
|
1629
|
+
util.debug(f"Threads done/started: {self.threads_stats}")
|
|
1630
|
+
if self.threads_stats.all_done():
|
|
1631
|
+
util.info("All threads done")
|
|
1397
1632
|
return
|
|
1398
1633
|
time.sleep(1)
|
|
1399
|
-
subprocess.Popen(['stty', 'sane']).wait()
|
|
1400
|
-
util.debug(
|
|
1634
|
+
subprocess.Popen(['stty', 'sane']).wait() # pylint: disable=consider-using-with
|
|
1635
|
+
util.debug("Scanning workers again")
|
|
1401
1636
|
for w in self.worker_threads:
|
|
1402
1637
|
if w.proc:
|
|
1403
1638
|
util.info(f"need to KILL WORKER_{w.n}, probably needs manual cleanup, check 'ps'")
|
|
1404
1639
|
if w.pid:
|
|
1405
1640
|
os.kill(w.pid, signal.SIGKILL)
|
|
1406
1641
|
util.stop_log()
|
|
1407
|
-
subprocess.Popen(['stty', 'sane']).wait()
|
|
1642
|
+
subprocess.Popen(['stty', 'sane']).wait() # pylint: disable=consider-using-with
|
|
1408
1643
|
|
|
1409
|
-
def run_jobs(
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1644
|
+
def run_jobs( # pylint: disable=too-many-locals,too-many-branches,too-many-statements
|
|
1645
|
+
self, command: str
|
|
1646
|
+
) -> None:
|
|
1647
|
+
'''Runs all "jobs" in self.jobs list, either serially or in parallel'''
|
|
1648
|
+
|
|
1649
|
+
# this is where we actually run the jobs. it's a messy piece of code and prob could use
|
|
1650
|
+
# refactoring but the goal was to share as much as possible (job start, end, pass/fail
|
|
1651
|
+
# judgement, etc) while supporting various mode combinations (parallel mode, verbose mode,
|
|
1652
|
+
# fancy mode, etc) and keeping the UI output functional and awesome sauce
|
|
1414
1653
|
|
|
1415
1654
|
# walk targets to find the longest name, for display reasons
|
|
1416
1655
|
longest_job_name = 0
|
|
1417
1656
|
total_jobs = len(self.jobs)
|
|
1418
1657
|
self.jobs_status = [None] * total_jobs
|
|
1419
1658
|
for i in range(total_jobs):
|
|
1420
|
-
|
|
1421
|
-
if l>longest_job_name: longest_job_name = l
|
|
1659
|
+
longest_job_name = max(longest_job_name, len(self.jobs[i]['name']))
|
|
1422
1660
|
|
|
1423
1661
|
run_parallel = self.args['parallel'] > 1
|
|
1424
1662
|
|
|
1425
1663
|
# figure out the width to print various numbers
|
|
1426
1664
|
jobs_digits = len(f"{total_jobs}")
|
|
1427
|
-
|
|
1665
|
+
|
|
1666
|
+
job_done_return_code = None
|
|
1667
|
+
job_done_run_time = 0
|
|
1668
|
+
job_done_name = ''
|
|
1428
1669
|
|
|
1429
1670
|
# run the jobs!
|
|
1430
1671
|
running_jobs = {}
|
|
@@ -1435,9 +1676,10 @@ class CommandParallel(Command):
|
|
|
1435
1676
|
jobs_launched = 0
|
|
1436
1677
|
num_parallel = min(len(self.jobs), self.args['parallel'])
|
|
1437
1678
|
# 16 should really be the size of window or ?
|
|
1438
|
-
|
|
1439
|
-
# we will enter fancy mode if we are parallel and we can leave 6 lines of regular scrolling
|
|
1440
|
-
|
|
1679
|
+
_, lines = shutil.get_terminal_size()
|
|
1680
|
+
# we will enter fancy mode if we are parallel and we can leave 6 lines of regular scrolling
|
|
1681
|
+
# output
|
|
1682
|
+
fancy_mode = all([util.args['fancy'], num_parallel > 1, num_parallel <= (lines-6)])
|
|
1441
1683
|
multi_cwd = util.getcwd() + os.sep
|
|
1442
1684
|
|
|
1443
1685
|
self.patch_jobs_for_duplicate_target_names()
|
|
@@ -1448,8 +1690,12 @@ class CommandParallel(Command):
|
|
|
1448
1690
|
work_queue = queue.Queue()
|
|
1449
1691
|
done_queue = queue.Queue()
|
|
1450
1692
|
for x in range(num_parallel):
|
|
1451
|
-
worker = CommandParallelWorker(
|
|
1452
|
-
|
|
1693
|
+
worker = CommandParallelWorker(
|
|
1694
|
+
n=x, work_queue=work_queue, done_queue=done_queue,
|
|
1695
|
+
threads_stats=self.threads_stats
|
|
1696
|
+
)
|
|
1697
|
+
# Setting daemon to True will let the main thread exit even though the workers are
|
|
1698
|
+
# blocking
|
|
1453
1699
|
worker.daemon = True
|
|
1454
1700
|
worker.start()
|
|
1455
1701
|
self.worker_threads.append(worker)
|
|
@@ -1460,35 +1706,48 @@ class CommandParallel(Command):
|
|
|
1460
1706
|
for x in range(num_parallel):
|
|
1461
1707
|
util.fancy_print(f"Starting worker {x}", x)
|
|
1462
1708
|
|
|
1463
|
-
while
|
|
1709
|
+
while self.jobs or running_jobs:
|
|
1464
1710
|
job_done = False
|
|
1465
1711
|
job_done_quiet = False
|
|
1466
1712
|
anything_done = False
|
|
1467
1713
|
|
|
1468
1714
|
def sprint_job_line(job_number=0, job_name="", final=False, hide_stats=False):
|
|
1469
|
-
return (
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1715
|
+
return (
|
|
1716
|
+
"INFO: [EDA] " +
|
|
1717
|
+
string_or_space(
|
|
1718
|
+
f"[job {job_number:0{jobs_digits}d}/{total_jobs:0{jobs_digits}d} ",
|
|
1719
|
+
final) +
|
|
1720
|
+
string_or_space("| pass ", hide_stats or final) +
|
|
1721
|
+
string_or_space(
|
|
1722
|
+
f"{len(passed_jobs):0{jobs_digits}d}/{jobs_complete:0{jobs_digits}d} ",
|
|
1723
|
+
hide_stats) +
|
|
1724
|
+
string_or_space(f"@ {(100 * (jobs_complete)) / total_jobs : 5.1f}%",
|
|
1725
|
+
hide_stats or final) +
|
|
1726
|
+
string_or_space("] ", final) +
|
|
1727
|
+
f"{command} {(job_name+' ').ljust(longest_job_name+3,'.')}"
|
|
1728
|
+
)
|
|
1729
|
+
|
|
1730
|
+
# for any kind of run (parallel or not, fancy or not, verbose or not) ...
|
|
1731
|
+
# can we launch a job?
|
|
1732
|
+
if self.jobs and (len(running_jobs) < num_parallel):
|
|
1479
1733
|
# we are launching a job
|
|
1480
1734
|
jobs_launched += 1
|
|
1481
1735
|
anything_done = True
|
|
1482
1736
|
job = self.jobs.pop(0)
|
|
1483
|
-
|
|
1484
|
-
#
|
|
1737
|
+
# TODO(drew): it might be nice to pass more items on 'job' dict, like
|
|
1738
|
+
# logfile or job-name, so CommandSweep or CommandMulti don't have to set
|
|
1739
|
+
# via args. on their command_list.
|
|
1740
|
+
if job['name'].startswith(multi_cwd):
|
|
1741
|
+
job['name'] = job['name'][len(multi_cwd):]
|
|
1742
|
+
# in all but fancy mode, we will print this text at the launch of a job. It may
|
|
1743
|
+
# get a newline below
|
|
1485
1744
|
job_text = sprint_job_line(jobs_launched, job['name'], hide_stats=run_parallel)
|
|
1486
1745
|
command_list = job['command_list']
|
|
1487
1746
|
cwd = util.getcwd()
|
|
1488
1747
|
|
|
1489
1748
|
if run_parallel:
|
|
1490
1749
|
# multithreaded job launch: add to queue
|
|
1491
|
-
worker = workers.pop(0)
|
|
1750
|
+
worker = workers.pop(0)
|
|
1492
1751
|
running_jobs[str(jobs_launched)] = { 'name' : job['name'],
|
|
1493
1752
|
'number' : jobs_launched,
|
|
1494
1753
|
'worker' : worker,
|
|
@@ -1498,15 +1757,20 @@ class CommandParallel(Command):
|
|
|
1498
1757
|
suffix = "<START>"
|
|
1499
1758
|
if fancy_mode:
|
|
1500
1759
|
util.fancy_print(job_text+suffix, worker)
|
|
1760
|
+
elif failed_jobs:
|
|
1761
|
+
# if we aren't in fancy mode, we will print a START line, periodic RUNNING
|
|
1762
|
+
# lines, and PASS/FAIL line per-job
|
|
1763
|
+
util.print_orange(job_text + Colors.yellow + suffix)
|
|
1501
1764
|
else:
|
|
1502
|
-
|
|
1503
|
-
if len(failed_jobs): util.print_orange(job_text + util.string_yellow + suffix)
|
|
1504
|
-
else: util.print_yellow(job_text + util.string_yellow + suffix)
|
|
1765
|
+
util.print_yellow(job_text + Colors.yellow + suffix)
|
|
1505
1766
|
else:
|
|
1506
|
-
# single-threaded job launch, we are going to print out job info as we start
|
|
1507
|
-
# since non-verbose silences the job and prints only
|
|
1508
|
-
|
|
1509
|
-
|
|
1767
|
+
# single-threaded job launch, we are going to print out job info as we start
|
|
1768
|
+
# each job... no newline. since non-verbose silences the job and prints only
|
|
1769
|
+
# <PASS>/<FAIL> after the trailing "..." we leave here
|
|
1770
|
+
if failed_jobs:
|
|
1771
|
+
util.print_orange(job_text, end="")
|
|
1772
|
+
else:
|
|
1773
|
+
util.print_yellow(job_text, end="")
|
|
1510
1774
|
job_done_number = jobs_launched
|
|
1511
1775
|
job_done_name = job['name']
|
|
1512
1776
|
job_start_time = time.time()
|
|
@@ -1518,13 +1782,16 @@ class CommandParallel(Command):
|
|
|
1518
1782
|
_, _, job_done_return_code = self.exec(
|
|
1519
1783
|
cwd, command_list, background=False, stop_on_error=False, quiet=False
|
|
1520
1784
|
)
|
|
1521
|
-
# reprint the job text previously printed before running job(and given
|
|
1785
|
+
# reprint the job text previously printed before running job (and given
|
|
1786
|
+
# "\n" after the trailing "...")
|
|
1522
1787
|
else:
|
|
1523
1788
|
# run job, swallowing output (hope you have a logfile)
|
|
1524
1789
|
_, _, job_done_return_code = self.exec(
|
|
1525
1790
|
cwd, command_list, background=True, stop_on_error=False, quiet=True
|
|
1526
1791
|
)
|
|
1527
|
-
|
|
1792
|
+
# in this case, we have the job start text (trailing "...", no newline)
|
|
1793
|
+
# printed
|
|
1794
|
+
job_done_quiet = True
|
|
1528
1795
|
job_done = True
|
|
1529
1796
|
job_done_run_time = time.time() - job_start_time
|
|
1530
1797
|
# Since we consumed the job, use the job['index'] to track the per-job status:
|
|
@@ -1532,15 +1799,16 @@ class CommandParallel(Command):
|
|
|
1532
1799
|
if run_parallel:
|
|
1533
1800
|
# parallel run, check for completed job
|
|
1534
1801
|
if done_queue.qsize():
|
|
1535
|
-
# we're collecting a finished job from a worker thread. note we will only
|
|
1536
|
-
# loop, so as to share job completion code
|
|
1802
|
+
# we're collecting a finished job from a worker thread. note we will only
|
|
1803
|
+
# reap one job per iter of the big loop, so as to share job completion code
|
|
1804
|
+
# at the bottom
|
|
1537
1805
|
anything_done = True
|
|
1538
1806
|
job_done = True
|
|
1539
1807
|
job_done_number, job_done_name, job_done_return_code = done_queue.get()
|
|
1540
1808
|
t = running_jobs[str(job_done_number)]
|
|
1541
1809
|
# in fancy mode, we need to clear the worker line related to this job.
|
|
1542
1810
|
if fancy_mode:
|
|
1543
|
-
util.fancy_print(
|
|
1811
|
+
util.fancy_print("INFO: [EDA] Parallel: Worker Idle ...", t['worker'])
|
|
1544
1812
|
job_done_run_time = time.time() - t['start_time']
|
|
1545
1813
|
util.debug(f"removing job #{job_done_number} from running jobs")
|
|
1546
1814
|
del running_jobs[str(job_done_number)]
|
|
@@ -1552,33 +1820,40 @@ class CommandParallel(Command):
|
|
|
1552
1820
|
if (fancy_mode or (time.time() - t['update_time']) > 30):
|
|
1553
1821
|
t['update_time'] = time.time()
|
|
1554
1822
|
job_text = sprint_job_line(t['number'], t['name'], hide_stats=True)
|
|
1555
|
-
suffix = f"<RUNNING: {
|
|
1823
|
+
suffix = f"<RUNNING: {sprint_time(time.time() - t['start_time'])}>"
|
|
1556
1824
|
if fancy_mode:
|
|
1557
1825
|
util.fancy_print(f"{job_text}{suffix}", t['worker'])
|
|
1826
|
+
elif failed_jobs:
|
|
1827
|
+
util.print_orange(job_text + Colors.yellow + suffix)
|
|
1558
1828
|
else:
|
|
1559
|
-
|
|
1560
|
-
else: util.print_yellow(job_text+util.string_yellow+suffix)
|
|
1829
|
+
util.print_yellow(job_text + Colors.yellow + suffix)
|
|
1561
1830
|
|
|
1562
1831
|
# shared job completion code
|
|
1563
|
-
# single or multi-threaded, we can arrive here to harvest <= 1 jobs, and need
|
|
1564
|
-
# we expect the start of a status line to have been
|
|
1832
|
+
# single or multi-threaded, we can arrive here to harvest <= 1 jobs, and need
|
|
1833
|
+
# {job, return_code} valid, and we expect the start of a status line to have been
|
|
1834
|
+
# printed, ready for pass/fail
|
|
1565
1835
|
if job_done:
|
|
1566
1836
|
jobs_complete += 1
|
|
1567
1837
|
if job_done_return_code is None or job_done_return_code:
|
|
1568
|
-
# embed the color code, to change color of pass/fail during the
|
|
1838
|
+
# embed the color code, to change color of pass/fail during the
|
|
1839
|
+
# util.print_orange/yellow below
|
|
1569
1840
|
if job_done_return_code == 124:
|
|
1570
|
-
# bash uses 124 for bash timeout errors, if that was preprended to the
|
|
1571
|
-
|
|
1841
|
+
# bash uses 124 for bash timeout errors, if that was preprended to the
|
|
1842
|
+
# command list.
|
|
1843
|
+
suffix = f"{Colors.red}<TOUT: {sprint_time(job_done_run_time)}>"
|
|
1572
1844
|
else:
|
|
1573
|
-
suffix = f"{
|
|
1845
|
+
suffix = f"{Colors.red}<FAIL: {sprint_time(job_done_run_time)}>"
|
|
1574
1846
|
failed_jobs.append(job_done_name)
|
|
1575
1847
|
else:
|
|
1576
|
-
suffix = f"{
|
|
1848
|
+
suffix = f"{Colors.green}<PASS: {sprint_time(job_done_run_time)}>"
|
|
1577
1849
|
passed_jobs.append(job_done_name)
|
|
1578
1850
|
# we want to print in one shot, because in fancy modes that's all that we're allowed
|
|
1579
|
-
job_done_text = "" if job_done_quiet else sprint_job_line(job_done_number,
|
|
1580
|
-
|
|
1581
|
-
|
|
1851
|
+
job_done_text = "" if job_done_quiet else sprint_job_line(job_done_number,
|
|
1852
|
+
job_done_name)
|
|
1853
|
+
if failed_jobs:
|
|
1854
|
+
util.print_orange(f"{job_done_text}{suffix}")
|
|
1855
|
+
else:
|
|
1856
|
+
util.print_yellow(f"{job_done_text}{suffix}")
|
|
1582
1857
|
self.jobs_status[job_done_number-1] = job_done_return_code
|
|
1583
1858
|
|
|
1584
1859
|
if not anything_done:
|
|
@@ -1586,23 +1861,26 @@ class CommandParallel(Command):
|
|
|
1586
1861
|
|
|
1587
1862
|
if total_jobs:
|
|
1588
1863
|
emoji = "< :) >" if (len(passed_jobs) == total_jobs) else "< :( >"
|
|
1589
|
-
util.info(sprint_job_line(final=True,job_name="jobs passed")+emoji, start="")
|
|
1864
|
+
util.info(sprint_job_line(final=True, job_name="jobs passed") + emoji, start="")
|
|
1590
1865
|
else:
|
|
1591
|
-
util.info(
|
|
1866
|
+
util.info("Parallel: <No jobs found>")
|
|
1592
1867
|
# Make sure all jobs have a set status:
|
|
1593
|
-
for i,rc in enumerate(self.jobs_status):
|
|
1594
|
-
if rc is None or
|
|
1868
|
+
for i, rc in enumerate(self.jobs_status):
|
|
1869
|
+
if rc is None or not isinstance(rc, int):
|
|
1595
1870
|
self.error(f'job {i=} {rc=} did not return a proper return code')
|
|
1596
|
-
jobs_status[i] =
|
|
1871
|
+
self.jobs_status[i] = 2
|
|
1597
1872
|
|
|
1598
1873
|
# if self.status > 0, then keep it non-zero, else set it if we still have running jobs.
|
|
1599
1874
|
if self.status == 0:
|
|
1600
|
-
|
|
1875
|
+
if self.jobs_status:
|
|
1876
|
+
self.status = max(self.jobs_status)
|
|
1877
|
+
# else keep at 0, empty list.
|
|
1601
1878
|
util.fancy_stop()
|
|
1602
1879
|
|
|
1603
1880
|
@staticmethod
|
|
1604
1881
|
def get_name_from_target(target: str) -> str:
|
|
1605
|
-
|
|
1882
|
+
'''Given a target path, strip leftmost path separators to get a shorter string name'''
|
|
1883
|
+
return target.replace('../', '').lstrip('./').lstrip(os.path.sep)
|
|
1606
1884
|
|
|
1607
1885
|
|
|
1608
1886
|
def update_args_list(self, args: list, tool: str) -> None:
|
|
@@ -1646,7 +1924,7 @@ class CommandParallel(Command):
|
|
|
1646
1924
|
tokens=tokens.copy(),
|
|
1647
1925
|
apply_parsed_args=False,
|
|
1648
1926
|
)
|
|
1649
|
-
util.debug(f'{self.command_name}: {single_cmd_unparsed=}')
|
|
1927
|
+
util.debug(f'{self.command_name}: {single_cmd_parsed=}, {single_cmd_unparsed=}')
|
|
1650
1928
|
|
|
1651
1929
|
# There should not be any single_cmd_unparsed args starting with '-'
|
|
1652
1930
|
bad_remaining_args = [x for x in single_cmd_unparsed if x.startswith('-')]
|
|
@@ -1663,34 +1941,36 @@ class CommandParallel(Command):
|
|
|
1663
1941
|
'''Examines list self.jobs, and if leaf target names are duplicate will
|
|
1664
1942
|
patch each command's job-name to:
|
|
1665
1943
|
--job-name=path.leaf.command[.tool]
|
|
1944
|
+
|
|
1945
|
+
Also do for --force-logfile
|
|
1666
1946
|
'''
|
|
1667
1947
|
|
|
1668
|
-
def
|
|
1669
|
-
'''Fishes the
|
|
1948
|
+
def get_job_arg(job_dict: dict, arg_name: str) -> str:
|
|
1949
|
+
'''Fishes the arg_name out of an entry in self.jobs'''
|
|
1670
1950
|
for i, item in enumerate(job_dict['command_list']):
|
|
1671
|
-
if item.startswith('--
|
|
1672
|
-
_, name = item.split('--
|
|
1951
|
+
if item.startswith(f'--{arg_name}='):
|
|
1952
|
+
_, name = item.split(f'--{arg_name}=')
|
|
1673
1953
|
return name
|
|
1674
|
-
|
|
1954
|
+
if item == f'--{arg_name}':
|
|
1675
1955
|
return job_dict['command_list'][i + 1]
|
|
1676
1956
|
return ''
|
|
1677
1957
|
|
|
1678
|
-
def
|
|
1679
|
-
'''Replaces the
|
|
1958
|
+
def replace_job_arg(job_dict: dict, arg_name: str, new_value: str) -> bool:
|
|
1959
|
+
'''Replaces the arg_name's value in an entry in self.jobs'''
|
|
1680
1960
|
for i, item in enumerate(job_dict['command_list']):
|
|
1681
|
-
if item.startswith('--
|
|
1682
|
-
job_dict['command_list'][i] = '--
|
|
1683
|
-
return
|
|
1684
|
-
|
|
1685
|
-
job_dict['command_list'][i + 1] =
|
|
1686
|
-
return
|
|
1687
|
-
return
|
|
1961
|
+
if item.startswith(f'--{arg_name}='):
|
|
1962
|
+
job_dict['command_list'][i] = f'--{arg_name}=' + new_value
|
|
1963
|
+
return True
|
|
1964
|
+
if item == f'--{arg_name}':
|
|
1965
|
+
job_dict['command_list'][i + 1] = new_value
|
|
1966
|
+
return True
|
|
1967
|
+
return False
|
|
1688
1968
|
|
|
1689
1969
|
|
|
1690
1970
|
job_names_count_dict = {}
|
|
1691
1971
|
for job_dict in self.jobs:
|
|
1692
1972
|
|
|
1693
|
-
key =
|
|
1973
|
+
key = get_job_arg(job_dict, arg_name='job-name')
|
|
1694
1974
|
if not key:
|
|
1695
1975
|
self.error(f'{job_dict=} needs to have a --job-name= arg attached')
|
|
1696
1976
|
if key not in job_names_count_dict:
|
|
@@ -1699,16 +1979,29 @@ class CommandParallel(Command):
|
|
|
1699
1979
|
job_names_count_dict[key] += 1
|
|
1700
1980
|
|
|
1701
1981
|
for i, job_dict in enumerate(self.jobs):
|
|
1702
|
-
key =
|
|
1982
|
+
key = get_job_arg(job_dict, 'job-name')
|
|
1703
1983
|
if job_names_count_dict[key] < 2:
|
|
1704
1984
|
continue
|
|
1705
1985
|
|
|
1706
1986
|
tpath, _ = os.path.split(job_dict['target'])
|
|
1707
1987
|
|
|
1708
1988
|
# prepend path information to job-name:
|
|
1709
|
-
patched_sub_work_dir = False
|
|
1710
1989
|
patched_target_path = os.path.relpath(tpath).replace(os.sep, '_')
|
|
1711
1990
|
new_job_name = f'{patched_target_path}.{key}'
|
|
1712
|
-
|
|
1713
|
-
|
|
1991
|
+
replace_job_arg(job_dict, arg_name='job-name', new_value=new_job_name)
|
|
1992
|
+
|
|
1993
|
+
# prepend path information to force-logfile (if present):
|
|
1994
|
+
force_logfile = get_job_arg(job_dict, arg_name='force-logfile')
|
|
1995
|
+
if force_logfile:
|
|
1996
|
+
left, right = os.path.split(force_logfile)
|
|
1997
|
+
new_force_logfile = os.path.join(left, f'{patched_target_path}.{right}')
|
|
1998
|
+
replace_job_arg(
|
|
1999
|
+
job_dict, arg_name='force-logfile', new_value=new_force_logfile
|
|
2000
|
+
)
|
|
2001
|
+
util.debug(
|
|
2002
|
+
f'Patched job {job_dict["name"]}: --force-logfile={new_force_logfile}'
|
|
2003
|
+
)
|
|
2004
|
+
|
|
2005
|
+
|
|
2006
|
+
self.jobs[i] = job_dict
|
|
1714
2007
|
util.debug(f'Patched job {job_dict["name"]}: --job-name={new_job_name}')
|