opencos-eda 0.2.48__py3-none-any.whl → 0.2.50__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- opencos/__init__.py +4 -2
- opencos/_version.py +10 -7
- opencos/commands/flist.py +8 -7
- opencos/commands/multi.py +14 -15
- opencos/commands/sim.py +5 -0
- opencos/commands/sweep.py +3 -2
- opencos/deps/__init__.py +0 -0
- opencos/deps/defaults.py +69 -0
- opencos/deps/deps_commands.py +419 -0
- opencos/deps/deps_file.py +326 -0
- opencos/deps/deps_processor.py +670 -0
- opencos/deps_schema.py +7 -8
- opencos/eda.py +84 -64
- opencos/eda_base.py +585 -316
- opencos/eda_config.py +85 -14
- opencos/eda_config_defaults.yml +36 -4
- opencos/eda_extract_targets.py +22 -14
- opencos/eda_tool_helper.py +33 -7
- opencos/export_helper.py +166 -86
- opencos/export_json_convert.py +31 -23
- opencos/files.py +2 -1
- opencos/hw/__init__.py +0 -0
- opencos/{oc_cli.py → hw/oc_cli.py} +9 -4
- opencos/names.py +0 -4
- opencos/peakrdl_cleanup.py +13 -7
- opencos/seed.py +19 -11
- opencos/tests/helpers.py +3 -2
- opencos/tests/test_deps_helpers.py +35 -32
- opencos/tests/test_eda.py +36 -29
- opencos/tests/test_eda_elab.py +7 -4
- opencos/tests/test_eda_synth.py +1 -1
- opencos/tests/test_oc_cli.py +1 -1
- opencos/tests/test_tools.py +4 -2
- opencos/tools/iverilog.py +2 -2
- opencos/tools/modelsim_ase.py +24 -2
- opencos/tools/questa.py +5 -3
- opencos/tools/questa_fse.py +57 -0
- opencos/tools/riviera.py +1 -1
- opencos/tools/slang.py +9 -3
- opencos/tools/surelog.py +1 -1
- opencos/tools/verilator.py +26 -1
- opencos/tools/vivado.py +34 -27
- opencos/tools/yosys.py +4 -3
- opencos/util.py +532 -474
- opencos/utils/__init__.py +0 -0
- opencos/utils/markup_helpers.py +98 -0
- opencos/utils/str_helpers.py +111 -0
- opencos/utils/subprocess_helpers.py +108 -0
- {opencos_eda-0.2.48.dist-info → opencos_eda-0.2.50.dist-info}/METADATA +1 -1
- opencos_eda-0.2.50.dist-info/RECORD +89 -0
- {opencos_eda-0.2.48.dist-info → opencos_eda-0.2.50.dist-info}/entry_points.txt +1 -1
- opencos/deps_helpers.py +0 -1346
- opencos_eda-0.2.48.dist-info/RECORD +0 -79
- /opencos/{pcie.py → hw/pcie.py} +0 -0
- {opencos_eda-0.2.48.dist-info → opencos_eda-0.2.50.dist-info}/WHEEL +0 -0
- {opencos_eda-0.2.48.dist-info → opencos_eda-0.2.50.dist-info}/licenses/LICENSE +0 -0
- {opencos_eda-0.2.48.dist-info → opencos_eda-0.2.50.dist-info}/licenses/LICENSE.spdx +0 -0
- {opencos_eda-0.2.48.dist-info → opencos_eda-0.2.50.dist-info}/top_level.txt +0 -0
opencos/eda_base.py
CHANGED
|
@@ -1,22 +1,36 @@
|
|
|
1
|
+
'''opencos.eda_base holds base classes for opencos.tools and opencos.commands
|
|
2
|
+
|
|
3
|
+
Note that opencos.Command.process_tokens (on a derived class) is the main entrypoint
|
|
4
|
+
from eda.py, it will find a command handler class for a command + tool, and call
|
|
5
|
+
its obj.process_tokens(args)
|
|
6
|
+
'''
|
|
7
|
+
|
|
8
|
+
# TODO(drew): Clean up the following pylint
|
|
9
|
+
# pylint: disable=too-many-lines
|
|
10
|
+
|
|
1
11
|
import argparse
|
|
2
12
|
import copy
|
|
3
|
-
import glob
|
|
4
13
|
import subprocess
|
|
5
14
|
import os
|
|
6
15
|
import queue
|
|
7
16
|
import re
|
|
8
17
|
import shutil
|
|
9
|
-
import shlex
|
|
10
18
|
import signal
|
|
11
19
|
import sys
|
|
12
20
|
import threading
|
|
13
21
|
import time
|
|
14
22
|
from pathlib import Path
|
|
15
23
|
|
|
16
|
-
import
|
|
17
|
-
from opencos import seed, deps_helpers, util, files
|
|
24
|
+
from opencos import seed, util, files
|
|
18
25
|
from opencos import eda_config
|
|
19
|
-
|
|
26
|
+
|
|
27
|
+
from opencos.util import Colors
|
|
28
|
+
from opencos.utils.str_helpers import sprint_time, strip_outer_quotes, string_or_space, \
|
|
29
|
+
indent_wrap_long_text
|
|
30
|
+
from opencos.utils.subprocess_helpers import subprocess_run_background
|
|
31
|
+
|
|
32
|
+
from opencos.deps.deps_file import DepsFile, deps_data_get_all_targets
|
|
33
|
+
from opencos.deps.deps_processor import DepsProcessor
|
|
20
34
|
|
|
21
35
|
|
|
22
36
|
def print_base_help() -> None:
|
|
@@ -28,9 +42,12 @@ def print_base_help() -> None:
|
|
|
28
42
|
|
|
29
43
|
|
|
30
44
|
def get_argparser() -> argparse.ArgumentParser:
|
|
45
|
+
'''Returns the ArgumentParser for general eda CLI'''
|
|
31
46
|
parser = argparse.ArgumentParser(prog='eda options', add_help=False, allow_abbrev=False)
|
|
32
47
|
parser.add_argument('-q', '--quit', action='store_true',
|
|
33
|
-
help=
|
|
48
|
+
help=(
|
|
49
|
+
'For interactive mode (eda called with no options, command, or'
|
|
50
|
+
' targets)'))
|
|
34
51
|
parser.add_argument('--exit', action='store_true', help='same as --quit')
|
|
35
52
|
parser.add_argument('-h', '--help', action='store_true')
|
|
36
53
|
|
|
@@ -40,22 +57,25 @@ def get_argparser() -> argparse.ArgumentParser:
|
|
|
40
57
|
help='Tool to use for this command, such as: modelsim_ase, verilator,' \
|
|
41
58
|
+ ' modelsim_ase=/path/to/bin/vsim, verilator=/path/to/bin/verilator')
|
|
42
59
|
parser.add_argument('--eda-safe', action='store_true',
|
|
43
|
-
help='disable all DEPS file deps shell commands, overrides values from
|
|
60
|
+
help=('disable all DEPS file deps shell commands, overrides values from'
|
|
61
|
+
' --config-yml'))
|
|
44
62
|
return parser
|
|
45
63
|
|
|
46
64
|
|
|
47
65
|
def get_argparser_short_help() -> str:
|
|
66
|
+
'''Returns str for the eda_base argparser (--help, --tool, etc)'''
|
|
48
67
|
return util.get_argparser_short_help(parser=get_argparser())
|
|
49
68
|
|
|
50
69
|
|
|
51
|
-
def get_eda_exec(command:str=''):
|
|
70
|
+
def get_eda_exec(command: str = '') -> str:
|
|
71
|
+
'''Returns the full path of `eda` executable to be used for a given eda <command>'''
|
|
52
72
|
# NOTE(drew): This is kind of flaky. 'eda multi' reinvokes 'eda'. But the executable for 'eda'
|
|
53
73
|
# is one of:
|
|
54
74
|
# 1. pip3 install opencos-eda
|
|
55
75
|
# -- script 'eda', installed from PyPi
|
|
56
76
|
# 2. pip3 uninstall .; python3 -m build; pip3 install
|
|
57
77
|
# -- script 'eda' but installed from local.
|
|
58
|
-
# 2. (opencos repo)/bin/eda - a python wrapper to link to (opencos repo)/opencos/eda.py
|
|
78
|
+
# 2. (opencos repo)/bin/eda - a python wrapper to link to (opencos repo)/opencos/eda.py, but
|
|
59
79
|
# packages cannot be run standalone, they need to be called as: python3 -m opencos.eda,
|
|
60
80
|
# and do not work with relative paths. This only works if env OC_ROOT is set or can be found.
|
|
61
81
|
# 3. If you ran 'source bin/addpath' then you are always using the local (opencos repo)/bin/eda
|
|
@@ -64,11 +84,13 @@ def get_eda_exec(command:str=''):
|
|
|
64
84
|
# Can we run from OC_ROOT/bin/eda?
|
|
65
85
|
oc_root = util.get_oc_root()
|
|
66
86
|
if not oc_root:
|
|
67
|
-
util.error(f"Need 'eda' in our path to run 'eda {command}', could not find env
|
|
87
|
+
util.error(f"Need 'eda' in our path to run 'eda {command}', could not find env",
|
|
88
|
+
f"OC_ROOT, {eda_path=}, {oc_root=}")
|
|
68
89
|
else:
|
|
69
90
|
bin_eda = os.path.join(oc_root, 'bin', 'eda')
|
|
70
91
|
if not os.path.exists(bin_eda):
|
|
71
|
-
util.error(f"Need 'eda' in our path to run 'eda {command}', cound not find
|
|
92
|
+
util.error(f"Need 'eda' in our path to run 'eda {command}', cound not find",
|
|
93
|
+
f"bin/, {eda_path=}, {oc_root=}, {bin_eda=}")
|
|
72
94
|
else:
|
|
73
95
|
util.info(f"'eda' not in path, using {bin_eda=} for 'eda' {command} executable")
|
|
74
96
|
eda_path = os.path.abspath(bin_eda)
|
|
@@ -76,9 +98,10 @@ def get_eda_exec(command:str=''):
|
|
|
76
98
|
return eda_path
|
|
77
99
|
|
|
78
100
|
|
|
79
|
-
def which_tool(command, config):
|
|
80
|
-
'''Returns which tool will be used for a command, given the command_handlers in
|
|
81
|
-
|
|
101
|
+
def which_tool(command: str, config: dict) -> str:
|
|
102
|
+
'''Returns which tool (str) will be used for a command, given the command_handlers in
|
|
103
|
+
|
|
104
|
+
config dict.'''
|
|
82
105
|
if config is None:
|
|
83
106
|
util.error(f'which_tool({command=}) called w/out config')
|
|
84
107
|
if not command in config.get('command_handler', {}):
|
|
@@ -136,9 +159,10 @@ class Tool:
|
|
|
136
159
|
self.set_tool_config_from_config()
|
|
137
160
|
|
|
138
161
|
def set_exe(self, config: dict) -> None:
|
|
162
|
+
'''Sets self._EXE based on config'''
|
|
139
163
|
if self._TOOL and self._TOOL in config.get('auto_tools_order', [{}])[0]:
|
|
140
164
|
exe = config.get('auto_tools_order', [{}])[0][self._TOOL].get('exe', '')
|
|
141
|
-
if exe and
|
|
165
|
+
if exe and isinstance(exe, list):
|
|
142
166
|
exe = exe[0] # pick first
|
|
143
167
|
if exe and exe != self._EXE:
|
|
144
168
|
util.info(f'Override for {self._TOOL} using exe {exe}')
|
|
@@ -155,9 +179,16 @@ class Tool:
|
|
|
155
179
|
return self._VERSION
|
|
156
180
|
|
|
157
181
|
def set_tool_defines(self) -> None:
|
|
158
|
-
|
|
182
|
+
'''Derived classes may override, sets any additional defines based on tool.'''
|
|
183
|
+
return
|
|
184
|
+
|
|
159
185
|
|
|
160
186
|
class Command:
|
|
187
|
+
'''Base class for all: eda <command>
|
|
188
|
+
|
|
189
|
+
The Command class should be used when you don't require files, otherwise consider
|
|
190
|
+
CommandDesign.
|
|
191
|
+
'''
|
|
161
192
|
|
|
162
193
|
command_name: str = ''
|
|
163
194
|
|
|
@@ -170,20 +201,20 @@ class Command:
|
|
|
170
201
|
"keep" : False,
|
|
171
202
|
"force" : False,
|
|
172
203
|
"fake" : False,
|
|
173
|
-
"stop-before-compile": False,
|
|
204
|
+
"stop-before-compile": False,
|
|
174
205
|
"stop-after-compile": False,
|
|
175
|
-
"stop-after-elaborate": False,
|
|
206
|
+
"stop-after-elaborate": False,
|
|
176
207
|
"lint": False, # Same as stop-after-elaborate
|
|
177
208
|
"eda-dir" : "eda.work", # all eda jobs go in here
|
|
178
209
|
"job-name" : "", # this is used to create a certain dir under "eda_dir"
|
|
179
|
-
"work-dir" : "", #
|
|
180
|
-
"sub-work-dir" : "", # this can be used to name the dir built under <eda-dir
|
|
210
|
+
"work-dir" : "", # default is usually <eda-dir>/<job-name> or <eda-dir>/<target>.<cmd>
|
|
211
|
+
"sub-work-dir" : "", # this can be used to name the dir built under <eda-dir>
|
|
181
212
|
"work-dir-use-target-dir": False,
|
|
182
213
|
"suffix" : "",
|
|
183
214
|
"design" : "", # not sure how this relates to top
|
|
184
215
|
'export': False,
|
|
185
|
-
'export-run': False,
|
|
186
|
-
'export-json': False,
|
|
216
|
+
'export-run': False, # run from the exported location if possible
|
|
217
|
+
'export-json': False, # generates an export.json suitable for a testrunner
|
|
187
218
|
'enable-tags': [],
|
|
188
219
|
'disable-tags': [],
|
|
189
220
|
'test-mode': False,
|
|
@@ -214,6 +245,7 @@ class Command:
|
|
|
214
245
|
self.status = 0
|
|
215
246
|
self.errors_log_f = None
|
|
216
247
|
|
|
248
|
+
|
|
217
249
|
def error(self, *args, **kwargs) -> None:
|
|
218
250
|
'''Returns None, child classes can call self.error(..) instead of util.error,
|
|
219
251
|
|
|
@@ -225,10 +257,16 @@ class Command:
|
|
|
225
257
|
if self.args['work-dir']:
|
|
226
258
|
if not self.errors_log_f:
|
|
227
259
|
try:
|
|
228
|
-
|
|
229
|
-
|
|
260
|
+
fullpath = os.path.join(self.args['work-dir'], 'eda.errors.log')
|
|
261
|
+
self.errors_log_f = open( # pylint: disable=consider-using-with
|
|
262
|
+
fullpath, 'w', encoding='utf-8'
|
|
230
263
|
)
|
|
231
|
-
|
|
264
|
+
util.artifacts.add(
|
|
265
|
+
name=fullpath,
|
|
266
|
+
typ='text', description='EDA reported errors'
|
|
267
|
+
)
|
|
268
|
+
|
|
269
|
+
except FileNotFoundError:
|
|
232
270
|
pass
|
|
233
271
|
if self.errors_log_f:
|
|
234
272
|
print(
|
|
@@ -245,17 +283,29 @@ class Command:
|
|
|
245
283
|
util.error(f"command '{self.command_name}' has previous errors")
|
|
246
284
|
return self.status > 0
|
|
247
285
|
|
|
248
|
-
def which_tool(self, command:str):
|
|
286
|
+
def which_tool(self, command:str) -> str:
|
|
287
|
+
'''Returns a str for the tool name used for the requested command'''
|
|
249
288
|
return which_tool(command, config=self.config)
|
|
250
289
|
|
|
251
|
-
def create_work_dir(
|
|
290
|
+
def create_work_dir( # pylint: disable=too-many-branches,too-many-statements
|
|
291
|
+
self
|
|
292
|
+
) -> str:
|
|
293
|
+
'''Creates the working directory and populates self.args['work-dir']
|
|
294
|
+
|
|
295
|
+
Generally uses ./ self.args['eda-dir'] / <target-name>.<command> /
|
|
296
|
+
however, self.args['job-name'] or ['sub-work-dir'] can override that.
|
|
297
|
+
|
|
298
|
+
Additionally, the work-dir is attempted to be deleted if it already exists
|
|
299
|
+
AND it is beneath the caller's original working directory. We also will
|
|
300
|
+
not delete 'work-dir' that is ./ or $PWD.
|
|
301
|
+
'''
|
|
252
302
|
util.debug(f"create_work_dir: {self.args['eda-dir']=} {self.args['work-dir']=}")
|
|
253
303
|
if self.args['work-dir-use-target-dir']:
|
|
254
304
|
if not self.target_path:
|
|
255
305
|
self.target_path = '.'
|
|
256
|
-
util.info(
|
|
257
|
-
|
|
258
|
-
|
|
306
|
+
util.info("create_work_dir: --work-dir-use-target-dir: using:",
|
|
307
|
+
f"{os.path.abspath(self.target_path)}",
|
|
308
|
+
f'target={self.target}')
|
|
259
309
|
self.args['work-dir'] = self.target_path
|
|
260
310
|
self.args['sub-work-dir'] = ''
|
|
261
311
|
return self.args['work-dir']
|
|
@@ -263,28 +313,37 @@ class Command:
|
|
|
263
313
|
if not os.path.exists(self.args['eda-dir']):
|
|
264
314
|
util.safe_mkdir(self.args['eda-dir'])
|
|
265
315
|
util.info(f"create_work_dir: created {self.args['eda-dir']}")
|
|
316
|
+
|
|
266
317
|
if self.args['design'] == "":
|
|
267
318
|
if ('top' in self.args) and (self.args['top'] != ""):
|
|
268
319
|
self.args['design'] = self.args['top']
|
|
269
|
-
util.debug(f"create_work_dir: set {self.args['design']=} from
|
|
320
|
+
util.debug(f"create_work_dir: set {self.args['design']=} from",
|
|
321
|
+
f"{self.args['top']=}, since it was empty")
|
|
270
322
|
else:
|
|
271
323
|
self.args['design'] = "design" # generic, i.e. to create work dir "design_upload"
|
|
272
|
-
util.debug(f"create_work_dir: set {self.args['design']=} to 'design', since it
|
|
324
|
+
util.debug(f"create_work_dir: set {self.args['design']=} to 'design', since it",
|
|
325
|
+
"was empty and we have no top")
|
|
326
|
+
|
|
273
327
|
if self.target == "":
|
|
274
328
|
self.target = self.args['design']
|
|
275
329
|
util.debug(f"create_work_dir: set {self.target=} from design name, since it was empty")
|
|
330
|
+
|
|
276
331
|
if self.args['work-dir'] == '':
|
|
277
332
|
if self.args['sub-work-dir'] == '':
|
|
278
333
|
if self.args['job-name'] != '':
|
|
279
334
|
self.args['sub-work-dir'] = self.args['job-name']
|
|
280
|
-
util.debug(f"create_work_dir: set {self.args['sub-work-dir']=} from
|
|
335
|
+
util.debug(f"create_work_dir: set {self.args['sub-work-dir']=} from",
|
|
336
|
+
f"{self.args['job-name']=}, since it was empty")
|
|
281
337
|
else:
|
|
282
338
|
self.args['sub-work-dir'] = f'{self.target}.{self.command_name}'
|
|
283
|
-
util.debug(f"create_work_dir: set {self.args['sub-work-dir']=} from
|
|
339
|
+
util.debug(f"create_work_dir: set {self.args['sub-work-dir']=} from",
|
|
340
|
+
f"{self.target=} and {self.command_name=}, since it was empty and",
|
|
341
|
+
"we have no job-name")
|
|
284
342
|
self.args['work-dir'] = os.path.join(self.args['eda-dir'], self.args['sub-work-dir'])
|
|
285
343
|
util.debug(f"create_work_dir: set {self.args['work-dir']=}")
|
|
344
|
+
|
|
286
345
|
keep_file = os.path.join(self.args['work-dir'], "eda.keep")
|
|
287
|
-
if
|
|
346
|
+
if os.path.exists(self.args['work-dir']):
|
|
288
347
|
if os.path.exists(keep_file) and not self.args['force']:
|
|
289
348
|
self.error(f"Cannot remove old work dir due to '{keep_file}'")
|
|
290
349
|
elif os.path.abspath(self.args['work-dir']) in os.getcwd():
|
|
@@ -293,16 +352,19 @@ class Command:
|
|
|
293
352
|
# --work-dir=$PWD
|
|
294
353
|
# --work-dir=/some/path/almost/here
|
|
295
354
|
# Allow it, but preserve the existing directory, we don't want to blow away
|
|
296
|
-
# files up-
|
|
355
|
+
# files up-hier from us.
|
|
297
356
|
# Enables support for --work-dir=.
|
|
298
|
-
util.info(f"Not removing existing work-dir: '{self.args['work-dir']}' is within
|
|
357
|
+
util.info(f"Not removing existing work-dir: '{self.args['work-dir']}' is within",
|
|
358
|
+
f"{os.getcwd()=}")
|
|
299
359
|
elif str(Path(self.args['work-dir'])).startswith(str(Path('/'))):
|
|
300
360
|
# Do not allow other absolute path work dirs if it already exists.
|
|
301
361
|
# This prevents you from --work-dir=~ and eda wipes out your home dir.
|
|
302
|
-
self.error(f'Cannot use work-dir={self.args["work-dir"]} starting with
|
|
362
|
+
self.error(f'Cannot use work-dir={self.args["work-dir"]} starting with',
|
|
363
|
+
'fabsolute path "/"')
|
|
303
364
|
elif str(Path('..')) in str(Path(self.args['work-dir'])):
|
|
304
365
|
# Do not allow other ../ work dirs if it already exists.
|
|
305
|
-
self.error(f'Cannot use work-dir={self.args["work-dir"]} with up-hierarchy
|
|
366
|
+
self.error(f'Cannot use work-dir={self.args["work-dir"]} with up-hierarchy'
|
|
367
|
+
'../ paths')
|
|
306
368
|
else:
|
|
307
369
|
# If we made it this far, on a directory that exists, that appears safe
|
|
308
370
|
# to delete and re-create:
|
|
@@ -313,22 +375,40 @@ class Command:
|
|
|
313
375
|
else:
|
|
314
376
|
util.safe_mkdir(self.args['work-dir'])
|
|
315
377
|
util.debug(f'create_work_dir: created {self.args["work-dir"]}')
|
|
316
|
-
|
|
317
|
-
|
|
378
|
+
|
|
379
|
+
if self.args['keep']:
|
|
380
|
+
open(keep_file, 'w', encoding='utf-8').close() # pylint: disable=consider-using-with
|
|
318
381
|
util.debug(f'create_work_dir: created {keep_file=}')
|
|
382
|
+
|
|
383
|
+
# Set the util.artifacts path with our work-dir:
|
|
384
|
+
util.artifacts.set_artifacts_json_dir(self.args['work-dir'])
|
|
385
|
+
|
|
319
386
|
return self.args['work-dir']
|
|
320
387
|
|
|
321
|
-
def
|
|
322
|
-
|
|
388
|
+
def artifacts_add(self, name: str, typ: str, description: str) -> None:
|
|
389
|
+
'''Adds a file to util.artifacts, derived classes may override'''
|
|
390
|
+
util.artifacts.add(name=name, typ=typ, description=description)
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def exec(self, work_dir: str, command_list: list, background: bool = False,
|
|
394
|
+
stop_on_error: bool = True, quiet: bool = False, tee_fpath: str = '',
|
|
395
|
+
shell: bool = False) -> (str, str, int):
|
|
396
|
+
'''Runs a command via subprocess and returns a tuple of (stderr, stdout, rc)
|
|
397
|
+
|
|
398
|
+
- work_dir: is passed to the subprocess call, so os.chdir(..) is not invoked.
|
|
399
|
+
- command_list: should be a list of individual strings w/out spaces.
|
|
400
|
+
- background: if False, does not print to stdout (but will print to any
|
|
401
|
+
tee_fpath or the global log
|
|
402
|
+
- shell: arg passed to subprocess, defaults to False.
|
|
403
|
+
'''
|
|
404
|
+
|
|
323
405
|
if not tee_fpath and getattr(command_list, 'tee_fpath', None):
|
|
324
406
|
tee_fpath = getattr(command_list, 'tee_fpath', '')
|
|
325
407
|
if not quiet:
|
|
326
408
|
util.info(f"exec: {' '.join(command_list)} (in {work_dir}, {tee_fpath=})")
|
|
327
|
-
original_cwd = util.getcwd()
|
|
328
|
-
os.chdir(work_dir)
|
|
329
409
|
|
|
330
|
-
stdout, stderr, return_code =
|
|
331
|
-
work_dir=
|
|
410
|
+
stdout, stderr, return_code = subprocess_run_background(
|
|
411
|
+
work_dir=work_dir,
|
|
332
412
|
command_list=command_list,
|
|
333
413
|
background=background,
|
|
334
414
|
fake=self.args.get('fake', False),
|
|
@@ -336,16 +416,23 @@ class Command:
|
|
|
336
416
|
shell=shell
|
|
337
417
|
)
|
|
338
418
|
|
|
339
|
-
os.chdir(original_cwd)
|
|
340
419
|
if return_code:
|
|
341
420
|
self.status += return_code
|
|
342
|
-
if stop_on_error:
|
|
343
|
-
|
|
421
|
+
if stop_on_error:
|
|
422
|
+
self.error(f"exec: returned with error (return code: {return_code})")
|
|
423
|
+
else:
|
|
424
|
+
util.debug(f"exec: returned with error (return code: {return_code})")
|
|
344
425
|
else:
|
|
345
426
|
util.debug(f"exec: returned without error (return code: {return_code})")
|
|
346
427
|
return stderr, stdout, return_code
|
|
347
428
|
|
|
348
|
-
def set_arg(
|
|
429
|
+
def set_arg( # pylint: disable=too-many-branches
|
|
430
|
+
self, key, value
|
|
431
|
+
) -> None:
|
|
432
|
+
'''Sets self.args[key] with value, and self.modified_args[key]=True
|
|
433
|
+
|
|
434
|
+
Does type checking, handles list type append
|
|
435
|
+
'''
|
|
349
436
|
|
|
350
437
|
# Do some minimal type handling, preserving the type(self.args[key])
|
|
351
438
|
if key not in self.args:
|
|
@@ -378,14 +465,14 @@ class Command:
|
|
|
378
465
|
else:
|
|
379
466
|
self.args[key] = True
|
|
380
467
|
else:
|
|
381
|
-
|
|
468
|
+
assert False, f'set_arg({key=}, {value=}) bool {type(cur_value)=} {type(value)=}'
|
|
382
469
|
|
|
383
470
|
elif isinstance(cur_value, int):
|
|
384
471
|
# if int, attempt to convert string or bool --> int
|
|
385
472
|
if isinstance(value, (bool, int, str)):
|
|
386
473
|
self.args[key] = int(value)
|
|
387
474
|
else:
|
|
388
|
-
|
|
475
|
+
assert False, f'set_arg({key=}, {value=}) int {type(cur_value)=} {type(value)=}'
|
|
389
476
|
|
|
390
477
|
|
|
391
478
|
else:
|
|
@@ -396,7 +483,9 @@ class Command:
|
|
|
396
483
|
util.debug(f'Set arg["{key}"]="{self.args[key]}"')
|
|
397
484
|
|
|
398
485
|
|
|
399
|
-
def get_argparser(
|
|
486
|
+
def get_argparser( # pylint: disable=too-many-branches
|
|
487
|
+
self, parser_arg_list=None
|
|
488
|
+
) -> argparse.ArgumentParser:
|
|
400
489
|
''' Returns an argparse.ArgumentParser() based on self.args (dict)
|
|
401
490
|
|
|
402
491
|
If parser_arg_list is not None, the ArgumentParser() is created using only the keys in
|
|
@@ -404,8 +493,8 @@ class Command:
|
|
|
404
493
|
'''
|
|
405
494
|
|
|
406
495
|
# Preference is --args-with-dashes, which then become parsed.args_with_dashes, b/c
|
|
407
|
-
# parsed.args-with-dashes is not legal python. Some of self.args.keys() still have - or _,
|
|
408
|
-
# this will handle both.
|
|
496
|
+
# parsed.args-with-dashes is not legal python. Some of self.args.keys() still have - or _,
|
|
497
|
+
# so this will handle both.
|
|
409
498
|
# Also, preference is for self.args.keys(), to be str with - dashes
|
|
410
499
|
parser = argparse.ArgumentParser(prog='eda', add_help=False, allow_abbrev=False)
|
|
411
500
|
bool_action_kwargs = util.get_argparse_bool_action_kwargs()
|
|
@@ -439,19 +528,19 @@ class Command:
|
|
|
439
528
|
help_kwargs = {'help': f'{type(value).__name__} default={value}'}
|
|
440
529
|
|
|
441
530
|
|
|
442
|
-
# It's important to set the default=None on these, except for list types where default
|
|
443
|
-
# If the parsed Namespace has values set to None or [], we do not update. This
|
|
444
|
-
# are processed that have args set, they cannot override the top
|
|
445
|
-
# nor be overriden by defaults.
|
|
446
|
-
if
|
|
447
|
-
# For bool, support --key and --no-key with
|
|
448
|
-
# Note, this means you cannot use --some-bool=True, or --some-bool=False, has to
|
|
449
|
-
# or --no-some-bool.
|
|
531
|
+
# It's important to set the default=None on these, except for list types where default
|
|
532
|
+
# is []. If the parsed Namespace has values set to None or [], we do not update. This
|
|
533
|
+
# means that as deps are processed that have args set, they cannot override the top
|
|
534
|
+
# level args that were already set, nor be overriden by defaults.
|
|
535
|
+
if isinstance(value, bool):
|
|
536
|
+
# For bool, support --key and --no-key with action=argparse.BooleanOptionalAction.
|
|
537
|
+
# Note, this means you cannot use --some-bool=True, or --some-bool=False, has to
|
|
538
|
+
# be --some-bool or --no-some-bool.
|
|
450
539
|
parser.add_argument(
|
|
451
540
|
*arguments, default=None, **bool_action_kwargs, **help_kwargs)
|
|
452
|
-
elif
|
|
541
|
+
elif isinstance(value, list):
|
|
453
542
|
parser.add_argument(*arguments, default=value, action='append', **help_kwargs)
|
|
454
|
-
elif
|
|
543
|
+
elif isinstance(value, (int, str)):
|
|
455
544
|
parser.add_argument(*arguments, default=value, type=type(value), **help_kwargs)
|
|
456
545
|
elif value is None:
|
|
457
546
|
parser.add_argument(*arguments, default=None, **help_kwargs)
|
|
@@ -462,10 +551,10 @@ class Command:
|
|
|
462
551
|
|
|
463
552
|
|
|
464
553
|
def run_argparser_on_list(
|
|
465
|
-
self, tokens: list, parser_arg_list=None, apply_parsed_args:bool=True
|
|
554
|
+
self, tokens: list, parser_arg_list=None, apply_parsed_args: bool = True
|
|
466
555
|
) -> (dict, list):
|
|
467
|
-
''' Creates an argparse.ArgumentParser() for all the keys in self.args, and attempts to
|
|
468
|
-
from the provided list. Parsed args are applied to self.args.
|
|
556
|
+
''' Creates an argparse.ArgumentParser() for all the keys in self.args, and attempts to
|
|
557
|
+
parse from the provided list. Parsed args are applied to self.args.
|
|
469
558
|
|
|
470
559
|
Returns a tuple of (parsed argparse.Namespace obj, list of unparsed args)
|
|
471
560
|
|
|
@@ -479,17 +568,16 @@ class Command:
|
|
|
479
568
|
parsed, unparsed = parser.parse_known_args(tokens + [''])
|
|
480
569
|
unparsed = list(filter(None, unparsed))
|
|
481
570
|
except argparse.ArgumentError:
|
|
482
|
-
self.error(f'problem {command_name=} attempting to parse_known_args for {tokens=}')
|
|
571
|
+
self.error(f'problem {self.command_name=} attempting to parse_known_args for {tokens=}')
|
|
483
572
|
|
|
484
573
|
parsed_as_dict = vars(parsed)
|
|
485
574
|
|
|
486
575
|
args_to_be_applied = {}
|
|
487
576
|
|
|
488
|
-
for key,value in parsed_as_dict.items():
|
|
489
|
-
# key
|
|
490
|
-
# etc.
|
|
577
|
+
for key, value in parsed_as_dict.items():
|
|
578
|
+
# key currently has _ instead of POSIX dashes, so we convert to dashes, for
|
|
579
|
+
# most self.args, like self.args['build-file'], etc.
|
|
491
580
|
if key not in self.args and '_' in key:
|
|
492
|
-
# try with dashes instead of _
|
|
493
581
|
key = key.replace('_', '-')
|
|
494
582
|
assert key in self.args, f'{key=} not in {self.args=}'
|
|
495
583
|
|
|
@@ -501,7 +589,8 @@ class Command:
|
|
|
501
589
|
return parsed, unparsed
|
|
502
590
|
|
|
503
591
|
|
|
504
|
-
def apply_args_from_dict(self, args_to_be_applied:dict) -> list:
|
|
592
|
+
def apply_args_from_dict(self, args_to_be_applied: dict) -> list:
|
|
593
|
+
'''Given a dict of key/value for self.args, apply them to self.args'''
|
|
505
594
|
util.debug('apply_args_from_dict() -- called by:',
|
|
506
595
|
f'{self.command_name=}, {self.__class__.__name__=},',
|
|
507
596
|
f'{args_to_be_applied=}')
|
|
@@ -523,7 +612,7 @@ class Command:
|
|
|
523
612
|
f"with different cur value (cur value={self.args.get(key, None)})")
|
|
524
613
|
continue
|
|
525
614
|
if self.args[key] != value:
|
|
526
|
-
util.debug(
|
|
615
|
+
util.debug("Command.run_argparser_on_list - setting set_arg b/c",
|
|
527
616
|
f" argparse -- {key=} {value=} (cur value={self.args[key]})")
|
|
528
617
|
self.set_arg(key, value) # Note this has special handling for lists already.
|
|
529
618
|
self.modified_args[key] = True
|
|
@@ -541,7 +630,7 @@ class Command:
|
|
|
541
630
|
_, unparsed = self.run_argparser_on_list(tokens)
|
|
542
631
|
if process_all and len(unparsed) > 0:
|
|
543
632
|
self.error(f"Didn't understand argument: '{unparsed=}' in",
|
|
544
|
-
f" {self.command_name=} context")
|
|
633
|
+
f" {self.command_name=} context, {pwd=}")
|
|
545
634
|
|
|
546
635
|
return unparsed
|
|
547
636
|
|
|
@@ -574,27 +663,33 @@ class Command:
|
|
|
574
663
|
|
|
575
664
|
Allows command handlers to make thier own customizations from --config-yaml=YAML.
|
|
576
665
|
'''
|
|
577
|
-
|
|
666
|
+
return
|
|
578
667
|
|
|
579
668
|
def update_tool_config(self) -> None:
|
|
580
669
|
'''Returns None. Hook for classes like CommandSim to make tool specific overrides.'''
|
|
581
|
-
|
|
670
|
+
return
|
|
582
671
|
|
|
583
672
|
def write_eda_config_and_args(self):
|
|
673
|
+
'''Attempts to write eda_output_config.yml to our work-dir'''
|
|
584
674
|
if not self.args.get('work-dir', None):
|
|
585
675
|
util.warning(f'Ouput work-dir not set, saving ouput eda_config to {os.getcwd()}')
|
|
586
|
-
|
|
587
|
-
|
|
676
|
+
eda_config.write_eda_config_and_args(
|
|
677
|
+
dirpath=self.args.get('work-dir', os.getcwd()), command_obj_ref=self
|
|
678
|
+
)
|
|
588
679
|
|
|
589
680
|
def is_export_enabled(self) -> bool:
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
return any(
|
|
681
|
+
'''Returns True if any self.args['export'] is set in any way (but not set to False
|
|
682
|
+
or empty list)'''
|
|
683
|
+
return any(arg.startswith('export') and v for arg,v in self.args.items())
|
|
593
684
|
|
|
594
685
|
def run(self) -> None:
|
|
686
|
+
'''Alias for do_it(self)'''
|
|
595
687
|
self.do_it()
|
|
596
688
|
|
|
597
689
|
def do_it(self) -> None:
|
|
690
|
+
'''Main entrypoint of running a command, usually called by process_tokens.
|
|
691
|
+
|
|
692
|
+
process_tokens(..) is the starting entrypoint from eda.py'''
|
|
598
693
|
self.write_eda_config_and_args()
|
|
599
694
|
self.error(f"No tool bound to command '{self.command_name}', you",
|
|
600
695
|
" probably need to setup tool, or use '--tool <name>'")
|
|
@@ -610,7 +705,9 @@ class Command:
|
|
|
610
705
|
self.set_tool_defines()
|
|
611
706
|
|
|
612
707
|
|
|
613
|
-
def help(
|
|
708
|
+
def help( # pylint: disable=dangerous-default-value,too-many-branches
|
|
709
|
+
self, tokens: list = []
|
|
710
|
+
) -> None:
|
|
614
711
|
'''Since we don't quite follow standard argparger help()/usage(), we'll format our own
|
|
615
712
|
|
|
616
713
|
if self.args_help has additional help information.
|
|
@@ -618,7 +715,7 @@ class Command:
|
|
|
618
715
|
|
|
619
716
|
# Indent long lines (>100) to indent=56 (this is where we leave off w/ {vstr:12} below.
|
|
620
717
|
def indent_me(text:str):
|
|
621
|
-
return
|
|
718
|
+
return indent_wrap_long_text(text, width=100, indent=56)
|
|
622
719
|
|
|
623
720
|
util.info('Help:')
|
|
624
721
|
# using bare 'print' here, since help was requested, avoids --color and --quiet
|
|
@@ -637,14 +734,14 @@ class Command:
|
|
|
637
734
|
lines.append(f"Generic help for command='{self.command_name}'"
|
|
638
735
|
f" (using '{self.__class__.__name__}')")
|
|
639
736
|
else:
|
|
640
|
-
lines.append(
|
|
737
|
+
lines.append("Generic help (from class Command):")
|
|
641
738
|
|
|
642
739
|
# Attempt to run argparser on args, but don't error if it fails.
|
|
643
740
|
unparsed = []
|
|
644
741
|
if tokens:
|
|
645
742
|
try:
|
|
646
743
|
_, unparsed = self.run_argparser_on_list(tokens=tokens)
|
|
647
|
-
except:
|
|
744
|
+
except Exception:
|
|
648
745
|
pass
|
|
649
746
|
|
|
650
747
|
for k in sorted(self.args.keys()):
|
|
@@ -653,13 +750,13 @@ class Command:
|
|
|
653
750
|
khelp = self.args_help.get(k, '')
|
|
654
751
|
if khelp:
|
|
655
752
|
khelp = f' - {khelp}'
|
|
656
|
-
if
|
|
753
|
+
if isinstance(v, bool):
|
|
657
754
|
lines.append(indent_me(f" --{k:20} : boolean : {vstr:12}{khelp}"))
|
|
658
|
-
elif
|
|
755
|
+
elif isinstance(v, int):
|
|
659
756
|
lines.append(indent_me(f" --{k:20} : integer : {vstr:12}{khelp}"))
|
|
660
|
-
elif
|
|
757
|
+
elif isinstance(v, list):
|
|
661
758
|
lines.append(indent_me(f" --{k:20} : list : {vstr:12}{khelp}"))
|
|
662
|
-
elif
|
|
759
|
+
elif isinstance(v, str):
|
|
663
760
|
vstr = "'" + v + "'"
|
|
664
761
|
lines.append(indent_me(f" --{k:20} : string : {vstr:12}{khelp}"))
|
|
665
762
|
else:
|
|
@@ -672,7 +769,10 @@ class Command:
|
|
|
672
769
|
print(f'Unparsed args: {unparsed}')
|
|
673
770
|
|
|
674
771
|
|
|
675
|
-
class CommandDesign(Command):
|
|
772
|
+
class CommandDesign(Command): # pylint: disable=too-many-instance-attributes
|
|
773
|
+
'''CommandDesign is the eda base class for command handlers that need to track files.
|
|
774
|
+
|
|
775
|
+
This is the base class for CommandSim, CommandSynth, and others.'''
|
|
676
776
|
|
|
677
777
|
# Used by for DEPS work_dir_add_srcs@ commands, by class methods:
|
|
678
778
|
# update_file_lists_for_work_dir(..), and resolve_target(..)
|
|
@@ -693,11 +793,16 @@ class CommandDesign(Command):
|
|
|
693
793
|
self.args_help.update({
|
|
694
794
|
'seed': 'design seed, default is 31-bit non-zero urandom',
|
|
695
795
|
'top': 'TOP level verilog/SV module or VHDL entity for this target',
|
|
696
|
-
'all-sv': (
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
796
|
+
'all-sv': (
|
|
797
|
+
'Maintain .sv and .v in single file list.'
|
|
798
|
+
' False: .sv flist separate from .v flist and separate compile(s)'
|
|
799
|
+
' True: .sv and .v files compiled together if possible'
|
|
800
|
+
),
|
|
801
|
+
'unprocessed-plusargs': (
|
|
802
|
+
'Args that began with +, but were not +define+ or +incdir+, +<name>, '
|
|
803
|
+
' or +<name>=<value>. These become tool dependent, for example "sim" commands will'
|
|
804
|
+
' treat as sim-plusargs'
|
|
805
|
+
),
|
|
701
806
|
})
|
|
702
807
|
self.defines = {}
|
|
703
808
|
self.incdirs = []
|
|
@@ -715,12 +820,18 @@ class CommandDesign(Command):
|
|
|
715
820
|
for (d,v) in self.config.get('defines', {}).items():
|
|
716
821
|
self.defines[d] = v
|
|
717
822
|
|
|
718
|
-
|
|
823
|
+
# cached_deps: key = abspath of DEPS markup file, value is a dict with
|
|
824
|
+
# keys 'data' and 'line_numbers'
|
|
825
|
+
self.cached_deps = {}
|
|
719
826
|
self.targets_dict = {} # key = targets that we've already processed in DEPS files
|
|
720
827
|
self.last_added_source_file_inferred_top = ''
|
|
721
828
|
|
|
722
|
-
|
|
723
|
-
|
|
829
|
+
|
|
830
|
+
def run_dep_commands(self) -> None:
|
|
831
|
+
'''Run shell/peakrdl style commands from DEPS files
|
|
832
|
+
|
|
833
|
+
These are deferred to maintain the deps ordering, and run in that order.
|
|
834
|
+
'''
|
|
724
835
|
self.run_dep_shell_commands()
|
|
725
836
|
# Update any work_dir_add_srcs@ in our self.files, self.files_v, etc, b/c
|
|
726
837
|
# self.args['work-dir'] now exists.
|
|
@@ -728,7 +839,10 @@ class CommandDesign(Command):
|
|
|
728
839
|
# Link any non-sources to our work-dir:
|
|
729
840
|
self.update_non_source_files_in_work_dir()
|
|
730
841
|
|
|
731
|
-
|
|
842
|
+
|
|
843
|
+
def run_dep_shell_commands(self) -> None:
|
|
844
|
+
'''Specifically runs shell command from DEPS files'''
|
|
845
|
+
|
|
732
846
|
# Runs from self.args['work-dir']
|
|
733
847
|
all_cmds_lists = []
|
|
734
848
|
|
|
@@ -747,7 +861,8 @@ class CommandDesign(Command):
|
|
|
747
861
|
log_fnames_count.update({target_node: lognum + 1})
|
|
748
862
|
all_cmds_lists += [
|
|
749
863
|
[], # blank line
|
|
750
|
-
# comment, where it came from, log to {node}__shell_{lognum}.log
|
|
864
|
+
# comment, where it came from, log to {node}__shell_{lognum}.log
|
|
865
|
+
# (or tee name from DEPS.yml)
|
|
751
866
|
[f'# command {i}: target: {d["target_path"]} : {target_node} --> {log}'],
|
|
752
867
|
]
|
|
753
868
|
if not run_from_work_dir:
|
|
@@ -761,10 +876,12 @@ class CommandDesign(Command):
|
|
|
761
876
|
|
|
762
877
|
d['exec_list'] = clist # update to tee_fpath is set.
|
|
763
878
|
|
|
764
|
-
util.write_shell_command_file(
|
|
765
|
-
|
|
879
|
+
util.write_shell_command_file(
|
|
880
|
+
dirpath=self.args['work-dir'], filename='pre_compile_dep_shell_commands.sh',
|
|
881
|
+
command_lists=all_cmds_lists
|
|
882
|
+
)
|
|
766
883
|
|
|
767
|
-
for i,d in enumerate(self.dep_shell_commands):
|
|
884
|
+
for i, d in enumerate(self.dep_shell_commands):
|
|
768
885
|
util.info(f'run_dep_shell_commands {i=}: {d=}')
|
|
769
886
|
clist = util.ShellCommandList(d['exec_list'])
|
|
770
887
|
tee_fpath=clist.tee_fpath
|
|
@@ -782,16 +899,23 @@ class CommandDesign(Command):
|
|
|
782
899
|
self.exec(run_from_dir, clist, tee_fpath=tee_fpath,
|
|
783
900
|
shell=self.config.get('deps_subprocess_shell', False))
|
|
784
901
|
|
|
785
|
-
|
|
786
|
-
|
|
902
|
+
|
|
903
|
+
def update_file_lists_for_work_dir(self) -> None:
|
|
904
|
+
'''Handles any source files that were creating by "shell" style commands in the
|
|
905
|
+
|
|
906
|
+
work-dir, these need to be added via self.add_file(..) in the correct order
|
|
907
|
+
of the dependencies. Basically, any files that were left with @EDA-WORK-DIR@ prefix
|
|
908
|
+
need to be patched and added.
|
|
909
|
+
'''
|
|
910
|
+
if not self.dep_work_dir_add_srcs:
|
|
787
911
|
return
|
|
788
912
|
|
|
789
|
-
# If we encounter any @EDA-WORK_DIR@some_file.v in self.files, self.files_v, etc,
|
|
790
|
-
# self.args['work-dir'] / some_file.v:
|
|
913
|
+
# If we encounter any @EDA-WORK_DIR@some_file.v in self.files, self.files_v, etc,
|
|
914
|
+
# then replace it with: self.args['work-dir'] / some_file.v:
|
|
791
915
|
_work_dir_add_srcs_path_string_len = len(self._work_dir_add_srcs_path_string)
|
|
792
916
|
work_dir_abspath = os.path.abspath(self.args['work-dir'])
|
|
793
|
-
for key in list(self.files.keys()): # list so it's not an iterator,
|
|
794
|
-
if
|
|
917
|
+
for key in list(self.files.keys()): # list so it's not an iterator, updates self.files.
|
|
918
|
+
if isinstance(key, str) and key.startswith(self._work_dir_add_srcs_path_string):
|
|
795
919
|
new_key = os.path.join(work_dir_abspath, key[_work_dir_add_srcs_path_string_len :])
|
|
796
920
|
self.files.pop(key)
|
|
797
921
|
self.files[new_key] = True
|
|
@@ -800,12 +924,23 @@ class CommandDesign(Command):
|
|
|
800
924
|
self.files_sdc]
|
|
801
925
|
for my_file_list in my_file_lists_list:
|
|
802
926
|
for i,value in enumerate(my_file_list):
|
|
803
|
-
if value and
|
|
804
|
-
|
|
927
|
+
if value and isinstance(value, str) and \
|
|
928
|
+
value.startswith(self._work_dir_add_srcs_path_string):
|
|
929
|
+
new_value = os.path.join(
|
|
930
|
+
work_dir_abspath, value[_work_dir_add_srcs_path_string_len :]
|
|
931
|
+
)
|
|
805
932
|
my_file_list[i] = new_value
|
|
806
933
|
util.debug(f"file lists: replaced {value} with {new_value}")
|
|
807
934
|
|
|
808
|
-
|
|
935
|
+
|
|
936
|
+
def update_non_source_files_in_work_dir(self) -> None:
|
|
937
|
+
'''For non-source files (that are tracked as 'reqs' in DEPS markup files) these
|
|
938
|
+
|
|
939
|
+
need to be copied or linked to the work-dir. For example, if some SV assumes it
|
|
940
|
+
can $readmemh('file_that_is_here.txt') but we're running out of work-dir. Linking
|
|
941
|
+
is the easy work-around vs trying to run-in-place of all SV files.
|
|
942
|
+
'''
|
|
943
|
+
|
|
809
944
|
for fname in self.files_non_source:
|
|
810
945
|
_, leaf_fname = os.path.split(fname)
|
|
811
946
|
destfile = os.path.join(self.args['work-dir'], leaf_fname)
|
|
@@ -815,16 +950,25 @@ class CommandDesign(Command):
|
|
|
815
950
|
util.info(f'{fname=} {self.files_caller_info=}')
|
|
816
951
|
self.error(f'Non-source file (reqs?) {relfname=} does not exist from {caller_info}')
|
|
817
952
|
elif not os.path.exists(destfile):
|
|
818
|
-
util.debug(f'updating non-source file to work-dir: Linked {fname=} to {destfile=},
|
|
953
|
+
util.debug(f'updating non-source file to work-dir: Linked {fname=} to {destfile=},',
|
|
954
|
+
f'from {caller_info}')
|
|
819
955
|
if sys.platform == "win32":
|
|
820
956
|
shutil.copyfile(fname, destfile) # On Windows, fall back to copying
|
|
821
957
|
else:
|
|
822
958
|
os.symlink(src=fname, dst=destfile)
|
|
823
959
|
|
|
824
|
-
|
|
960
|
+
@staticmethod
|
|
961
|
+
def get_top_name(name: str) -> str:
|
|
962
|
+
'''Attempt to get the 'top' module name from a file, such as path/to/mine.sv will
|
|
963
|
+
|
|
964
|
+
return "mine"'''
|
|
965
|
+
# TODO(drew): Use the helper method in util for this instead to peek in file contents?
|
|
825
966
|
return os.path.splitext(os.path.basename(name))[0]
|
|
826
967
|
|
|
827
|
-
|
|
968
|
+
|
|
969
|
+
def process_plusarg( # pylint: disable=too-many-branches
|
|
970
|
+
self, plusarg: str, pwd: str = os.getcwd()
|
|
971
|
+
) -> None:
|
|
828
972
|
'''Retuns str, parses a +define+, +incdir+, +key=value str; adds to internal.
|
|
829
973
|
|
|
830
974
|
Adds to self.defines, self.incdirs,
|
|
@@ -836,12 +980,12 @@ class CommandDesign(Command):
|
|
|
836
980
|
# args that come from shlex.quote(token), such as:
|
|
837
981
|
# token = '\'+define+OC_ROOT="/foo/bar/opencos"\''
|
|
838
982
|
# So we strip all outer ' or " on the plusarg:
|
|
839
|
-
plusarg =
|
|
983
|
+
plusarg = strip_outer_quotes(plusarg)
|
|
840
984
|
if not pwd:
|
|
841
985
|
pwd = ''
|
|
842
986
|
|
|
843
987
|
if plusarg.startswith('+define+'):
|
|
844
|
-
plusarg = plusarg
|
|
988
|
+
plusarg = plusarg[len('+define+'):]
|
|
845
989
|
m = re.match(r'^(\w+)$', plusarg)
|
|
846
990
|
if m:
|
|
847
991
|
k = m.group(1)
|
|
@@ -849,11 +993,12 @@ class CommandDesign(Command):
|
|
|
849
993
|
util.debug(f"Defined {k}")
|
|
850
994
|
return None
|
|
851
995
|
m = re.match(r'^(\w+)\=(\S+)$', plusarg)
|
|
852
|
-
if not m:
|
|
996
|
+
if not m:
|
|
997
|
+
m = re.match(r'^(\w+)\=(\"[^\"]*\")$', plusarg)
|
|
853
998
|
if m:
|
|
854
999
|
k = m.group(1)
|
|
855
1000
|
v = m.group(2)
|
|
856
|
-
if v and
|
|
1001
|
+
if v and isinstance(v, str):
|
|
857
1002
|
if v.startswith('%PWD%/'):
|
|
858
1003
|
v = v.replace('%PWD%', os.path.abspath(pwd))
|
|
859
1004
|
if v.startswith('%SEED%'):
|
|
@@ -865,7 +1010,7 @@ class CommandDesign(Command):
|
|
|
865
1010
|
return None
|
|
866
1011
|
|
|
867
1012
|
if plusarg.startswith('+incdir+'):
|
|
868
|
-
plusarg = plusarg
|
|
1013
|
+
plusarg = plusarg[len('+incdir+'):]
|
|
869
1014
|
m = re.match(r'^(\S+)$', plusarg)
|
|
870
1015
|
if m:
|
|
871
1016
|
incdir = m.group(1)
|
|
@@ -881,40 +1026,59 @@ class CommandDesign(Command):
|
|
|
881
1026
|
if not self.config.get('bare_plusarg_supported', False):
|
|
882
1027
|
self.error(f"bare plusarg(s) are not supported: {plusarg}'")
|
|
883
1028
|
return None
|
|
884
|
-
|
|
1029
|
+
if plusarg not in self.args['unprocessed-plusargs']:
|
|
885
1030
|
self.args['unprocessed-plusargs'].append(plusarg)
|
|
886
1031
|
# For anything added to unprocessed-plusarg, we have to return it, to let
|
|
887
1032
|
# derived classes have the option to handle it
|
|
888
1033
|
return plusarg
|
|
889
|
-
|
|
890
|
-
|
|
1034
|
+
|
|
1035
|
+
self.error(f"Didn't understand +plusarg: '{plusarg}'")
|
|
891
1036
|
return None
|
|
892
1037
|
|
|
893
1038
|
|
|
894
|
-
def append_shell_commands(self, cmds : list):
|
|
895
|
-
|
|
1039
|
+
def append_shell_commands(self, cmds : list) -> None:
|
|
1040
|
+
''' Given a cmds (list), where each item is a dict with:
|
|
1041
|
+
|
|
1042
|
+
{ 'target_node': str,
|
|
1043
|
+
'target_path': str,
|
|
1044
|
+
'exec_list': list
|
|
1045
|
+
}
|
|
1046
|
+
add to this class's dep_shell_commands for deferred processing
|
|
1047
|
+
'''
|
|
1048
|
+
|
|
896
1049
|
for entry in cmds:
|
|
897
|
-
if entry is None or
|
|
1050
|
+
if entry is None or not isinstance(entry, dict):
|
|
898
1051
|
continue
|
|
899
1052
|
if entry in self.dep_shell_commands:
|
|
900
|
-
# we've already run this exact command (target node, target path, exec list),
|
|
901
|
-
# again
|
|
1053
|
+
# we've already run this exact command (target node, target path, exec list),
|
|
1054
|
+
# don't run it again
|
|
902
1055
|
continue
|
|
903
1056
|
|
|
904
1057
|
assert 'exec_list' in entry, f'{entry=}'
|
|
905
1058
|
util.debug(f'adding - dep_shell_command: {entry=}')
|
|
906
1059
|
self.dep_shell_commands.append(entry)
|
|
907
1060
|
|
|
908
|
-
|
|
909
|
-
|
|
1061
|
+
|
|
1062
|
+
def append_work_dir_add_srcs(self, add_srcs: list, caller_info: str) -> None:
|
|
1063
|
+
'''Given add_srcs (list), where each item is a dict with:
|
|
1064
|
+
|
|
1065
|
+
{ 'target_node': str,
|
|
1066
|
+
'target_path': str,
|
|
1067
|
+
'file_list': list
|
|
1068
|
+
}
|
|
1069
|
+
|
|
1070
|
+
adds files to set self.dep_work_dir_add_src, and call add_file on it, basically
|
|
1071
|
+
resolving adding source files that were generated and residing in the work-dir.
|
|
1072
|
+
'''
|
|
910
1073
|
for entry in add_srcs:
|
|
911
|
-
if entry is None or
|
|
1074
|
+
if entry is None or not isinstance(entry, dict):
|
|
912
1075
|
continue
|
|
913
1076
|
|
|
914
1077
|
work_dir_files = entry['file_list']
|
|
915
1078
|
for filename in work_dir_files:
|
|
916
|
-
# Unfortunately, self.args['work-dir'] doesn't exist yet and hasn't been set,
|
|
917
|
-
# files as '@EDA-WORK_DIR@' + filename, and have to replace
|
|
1079
|
+
# Unfortunately, self.args['work-dir'] doesn't exist yet and hasn't been set,
|
|
1080
|
+
# so we'll add these files as '@EDA-WORK_DIR@' + filename, and have to replace
|
|
1081
|
+
# the EDA-WORK_DIR@ string later in our flow.
|
|
918
1082
|
filename_use = self._work_dir_add_srcs_path_string + filename
|
|
919
1083
|
dep_key_tuple = (
|
|
920
1084
|
entry['target_path'],
|
|
@@ -929,14 +1093,25 @@ class CommandDesign(Command):
|
|
|
929
1093
|
self.dep_work_dir_add_srcs.add(dep_key_tuple) # add to set()
|
|
930
1094
|
elif dep_key_tuple not in self.dep_work_dir_add_srcs:
|
|
931
1095
|
# we've already added the file so this dep was skipped for this one file.
|
|
932
|
-
util.warning(f'work_dir_add_srcs@ {dep_key_tuple=} but {filename_use=}'
|
|
933
|
-
|
|
1096
|
+
util.warning(f'work_dir_add_srcs@ {dep_key_tuple=} but {filename_use=}',
|
|
1097
|
+
'is already in self.files (duplicate dependency on generated',
|
|
1098
|
+
'file?)')
|
|
934
1099
|
|
|
935
1100
|
|
|
936
|
-
def resolve_target(
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
1101
|
+
def resolve_target(
|
|
1102
|
+
self, target: str, no_recursion: bool = False, caller_info: str = ''
|
|
1103
|
+
) -> bool:
|
|
1104
|
+
'''Returns True if target is found. Entry point for resolving a CLI path based target name.
|
|
1105
|
+
|
|
1106
|
+
Will recursively call resolve_target_core, looking for a filename existing
|
|
1107
|
+
or DEPS markup file in that path and a target (key) in the markup table.
|
|
1108
|
+
|
|
1109
|
+
This will populate all source file dependencies, commands, and other DEPS markup
|
|
1110
|
+
features as the target is resolved.
|
|
1111
|
+
'''
|
|
1112
|
+
util.debug(f"Entered resolve_target({target})")
|
|
1113
|
+
# self.target is a name we grab for the job (i.e. for naming work dir etc). we don't want
|
|
1114
|
+
# the path prefix.
|
|
940
1115
|
|
|
941
1116
|
self.target_path, self.target = os.path.split(target)
|
|
942
1117
|
|
|
@@ -950,7 +1125,7 @@ class CommandDesign(Command):
|
|
|
950
1125
|
# If the target is a file (we're at the root here processing CLI arg tokens)
|
|
951
1126
|
# and that file exists and has an extension, then there's no reason to go looking
|
|
952
1127
|
# in DEPS files, add the file and return True.
|
|
953
|
-
|
|
1128
|
+
_, file_ext = os.path.splitext(fpath)
|
|
954
1129
|
if forced_extension or file_ext:
|
|
955
1130
|
self.add_file(fpath, caller_info=caller_info,
|
|
956
1131
|
forced_extension=forced_extension)
|
|
@@ -958,23 +1133,25 @@ class CommandDesign(Command):
|
|
|
958
1133
|
|
|
959
1134
|
return self.resolve_target_core(target, no_recursion, caller_info)
|
|
960
1135
|
|
|
961
|
-
def resolve_target_core(
|
|
962
|
-
|
|
1136
|
+
def resolve_target_core( # pylint: disable=too-many-locals,too-many-branches
|
|
1137
|
+
self, target: str, no_recursion: bool, caller_info: str = ''
|
|
1138
|
+
) -> bool:
|
|
1139
|
+
'''Returns True if target is found. recursive point for resolving path or DEPS markup
|
|
1140
|
+
target names.'''
|
|
1141
|
+
|
|
1142
|
+
util.debug(f"Entered resolve_target_core({target=})")
|
|
963
1143
|
found_target = False
|
|
964
|
-
util.debug("Starting to resolve target '%s'" % (target))
|
|
965
1144
|
target_path, target_node = os.path.split(target)
|
|
966
1145
|
|
|
967
|
-
deps = None
|
|
968
|
-
data = None
|
|
1146
|
+
deps, data, deps_file = None, None, None
|
|
969
1147
|
found_deps_file = False
|
|
970
1148
|
|
|
971
1149
|
if self.config['deps_markup_supported']:
|
|
972
|
-
deps =
|
|
1150
|
+
deps = DepsFile(
|
|
973
1151
|
command_design_ref=self, target_path=target_path, cache=self.cached_deps
|
|
974
1152
|
)
|
|
975
1153
|
deps_file = deps.deps_file
|
|
976
1154
|
data = deps.data
|
|
977
|
-
data_only_lines = deps.line_numbers
|
|
978
1155
|
|
|
979
1156
|
# Continue if we have data, otherwise look for files other than DEPS.<yml|yaml>
|
|
980
1157
|
if data is not None:
|
|
@@ -987,7 +1164,7 @@ class CommandDesign(Command):
|
|
|
987
1164
|
|
|
988
1165
|
# For convenience, use an external class for this DEPS.yml table/dict
|
|
989
1166
|
# This could be re-used for any markup DEPS.json, DEPS.toml, DEPS.py, etc.
|
|
990
|
-
deps_processor =
|
|
1167
|
+
deps_processor = DepsProcessor(
|
|
991
1168
|
command_design_ref = self,
|
|
992
1169
|
deps_entry = entry,
|
|
993
1170
|
target = target,
|
|
@@ -1008,10 +1185,11 @@ class CommandDesign(Command):
|
|
|
1008
1185
|
# Recurse on the returned deps (ordered list), if they haven't already been traversed.
|
|
1009
1186
|
for x in deps_targets_to_resolve:
|
|
1010
1187
|
caller_info = deps.gen_caller_info(target_node)
|
|
1011
|
-
if x and
|
|
1188
|
+
if x and isinstance(x, tuple):
|
|
1012
1189
|
# if deps_processor.process_deps_entry() gave us a tuple, it's an
|
|
1013
1190
|
# unprocessed 'command' that we kept in order until now. Append it.
|
|
1014
|
-
assert len(x) == 2,
|
|
1191
|
+
assert len(x) == 2, \
|
|
1192
|
+
f'command tuple {x=} must be len 2, {target_node=} {deps_file=}'
|
|
1015
1193
|
shell_commands_list, work_dir_add_srcs_list = x
|
|
1016
1194
|
self.append_shell_commands( cmds=shell_commands_list )
|
|
1017
1195
|
self.append_work_dir_add_srcs( add_srcs=work_dir_add_srcs_list,
|
|
@@ -1025,32 +1203,40 @@ class CommandDesign(Command):
|
|
|
1025
1203
|
forced_extension=forced_extension)
|
|
1026
1204
|
else:
|
|
1027
1205
|
util.debug(f' ... Calling resolve_target_core({x=})')
|
|
1028
|
-
found_target |= self.resolve_target_core(
|
|
1206
|
+
found_target |= self.resolve_target_core(
|
|
1207
|
+
x, no_recursion, caller_info=caller_info
|
|
1208
|
+
)
|
|
1029
1209
|
|
|
1030
1210
|
|
|
1031
1211
|
# Done with DEPS.yml if it existed.
|
|
1032
1212
|
|
|
1033
1213
|
if not found_target:
|
|
1034
|
-
util.debug("Haven't been able to resolve
|
|
1214
|
+
util.debug(f"Haven't been able to resolve {target=} via DEPS")
|
|
1035
1215
|
known_file_extensions_for_source = []
|
|
1036
|
-
for x in
|
|
1037
|
-
known_file_extensions_for_source += self.config.get(
|
|
1216
|
+
for x in ('verilog', 'systemverilog', 'vhdl', 'cpp'):
|
|
1217
|
+
known_file_extensions_for_source += self.config.get(
|
|
1218
|
+
'file_extensions', {}).get(x, [])
|
|
1038
1219
|
for e in known_file_extensions_for_source:
|
|
1039
1220
|
try_file = target + e
|
|
1040
|
-
util.debug("Looking for
|
|
1221
|
+
util.debug(f"Looking for file {try_file}")
|
|
1041
1222
|
if os.path.exists(try_file):
|
|
1042
|
-
self.add_file(try_file, caller_info=
|
|
1223
|
+
self.add_file(try_file, caller_info=f'n/a::{target}::n/a')
|
|
1043
1224
|
found_target = True
|
|
1044
1225
|
break # move on to the next target
|
|
1045
1226
|
if not found_target: # if STILL not found_this_target...
|
|
1046
|
-
self.error("Unable to resolve target
|
|
1227
|
+
self.error(f"Unable to resolve {target=}")
|
|
1047
1228
|
|
|
1048
1229
|
# if we've found any target since being called, it means we found the one we were called for
|
|
1049
1230
|
return found_target
|
|
1050
1231
|
|
|
1051
|
-
def add_file(
|
|
1052
|
-
|
|
1053
|
-
|
|
1232
|
+
def add_file(
|
|
1233
|
+
self, filename: str, use_abspath: bool = True, add_to_non_sources: bool = False,
|
|
1234
|
+
caller_info: str = '', forced_extension: str = ''
|
|
1235
|
+
) -> str:
|
|
1236
|
+
'''Given a filename, add it to one of self.files_sv or similar lists
|
|
1237
|
+
|
|
1238
|
+
based on file extension or prefix directive.'''
|
|
1239
|
+
_, file_ext = os.path.splitext(filename)
|
|
1054
1240
|
if use_abspath:
|
|
1055
1241
|
file_abspath = os.path.abspath(filename)
|
|
1056
1242
|
else:
|
|
@@ -1058,8 +1244,8 @@ class CommandDesign(Command):
|
|
|
1058
1244
|
|
|
1059
1245
|
|
|
1060
1246
|
if file_abspath in self.files:
|
|
1061
|
-
util.debug("Not adding file
|
|
1062
|
-
return
|
|
1247
|
+
util.debug(f"Not adding file {file_abspath}, already have it")
|
|
1248
|
+
return ''
|
|
1063
1249
|
|
|
1064
1250
|
known_file_ext_dict = self.config.get('file_extensions', {})
|
|
1065
1251
|
v_file_ext_list = known_file_ext_dict.get('verilog', [])
|
|
@@ -1082,34 +1268,43 @@ class CommandDesign(Command):
|
|
|
1082
1268
|
|
|
1083
1269
|
if add_to_non_sources:
|
|
1084
1270
|
self.files_non_source.append(file_abspath)
|
|
1085
|
-
util.debug("Added non-source file file
|
|
1271
|
+
util.debug(f"Added non-source file file {filename} as {file_abspath}")
|
|
1086
1272
|
elif file_ext in v_file_ext_list and not self.args['all-sv']:
|
|
1087
1273
|
self.files_v.append(file_abspath)
|
|
1088
|
-
util.debug("Added Verilog file
|
|
1089
|
-
elif file_ext in sv_file_ext_list or
|
|
1274
|
+
util.debug(f"Added Verilog file {filename} as {file_abspath}")
|
|
1275
|
+
elif file_ext in sv_file_ext_list or \
|
|
1276
|
+
((file_ext in v_file_ext_list) and self.args['all-sv']):
|
|
1090
1277
|
self.files_sv.append(file_abspath)
|
|
1091
|
-
util.debug("Added SystemVerilog file
|
|
1278
|
+
util.debug(f"Added SystemVerilog file {filename} as {file_abspath}")
|
|
1092
1279
|
elif file_ext in vhdl_file_ext_list:
|
|
1093
1280
|
self.files_vhd.append(file_abspath)
|
|
1094
|
-
util.debug("Added VHDL file
|
|
1281
|
+
util.debug(f"Added VHDL file {filename} as {file_abspath}")
|
|
1095
1282
|
elif file_ext in cpp_file_ext_list:
|
|
1096
1283
|
self.files_cpp.append(file_abspath)
|
|
1097
|
-
util.debug("Added C++ file
|
|
1284
|
+
util.debug(f"Added C++ file {filename} as {file_abspath}")
|
|
1098
1285
|
elif file_ext in sdc_file_ext_list:
|
|
1099
1286
|
self.files_sdc.append(file_abspath)
|
|
1100
|
-
util.debug("Added SDC file
|
|
1287
|
+
util.debug(f"Added SDC file {filename} as {file_abspath}")
|
|
1101
1288
|
else:
|
|
1102
1289
|
# unknown file extension. In these cases we link the file to the working directory
|
|
1103
|
-
# so it is available (for example, a .mem file that is expected to exist with relative
|
|
1290
|
+
# so it is available (for example, a .mem file that is expected to exist with relative
|
|
1291
|
+
# path)
|
|
1104
1292
|
self.files_non_source.append(file_abspath)
|
|
1105
|
-
util.debug("Added non-source file
|
|
1293
|
+
util.debug(f"Added non-source file {filename} as {file_abspath}")
|
|
1106
1294
|
|
|
1107
1295
|
self.files[file_abspath] = True
|
|
1108
1296
|
self.files_caller_info[file_abspath] = caller_info
|
|
1109
1297
|
return file_abspath
|
|
1110
1298
|
|
|
1111
|
-
def process_tokens(
|
|
1112
|
-
|
|
1299
|
+
def process_tokens( # pylint: disable=too-many-locals,too-many-branches,too-many-statements
|
|
1300
|
+
self, tokens: list, process_all: bool = True,
|
|
1301
|
+
pwd: str = os.getcwd()
|
|
1302
|
+
) -> list:
|
|
1303
|
+
'''Returns a list of unparsed args (self.args did not have these keys present)
|
|
1304
|
+
|
|
1305
|
+
Entrypoint from eda.py calling a command handler obj.process_tokens(args).
|
|
1306
|
+
Dervied classes are expected to call this parent process_tokens to parse and handle
|
|
1307
|
+
their self.args dict'''
|
|
1113
1308
|
|
|
1114
1309
|
util.debug(f'CommandDesign - process_tokens start - {tokens=}')
|
|
1115
1310
|
|
|
@@ -1123,19 +1318,21 @@ class CommandDesign(Command):
|
|
|
1123
1318
|
# walk the list, remove all items after we're done.
|
|
1124
1319
|
remove_list = []
|
|
1125
1320
|
for token in unparsed:
|
|
1126
|
-
# Since this is a raw argparser, we may have args that come from shlex.quote(token),
|
|
1127
|
-
#
|
|
1128
|
-
#
|
|
1321
|
+
# Since this is a raw argparser, we may have args that come from shlex.quote(token),
|
|
1322
|
+
# such as:
|
|
1323
|
+
# token = '\'+define+OC_ROOT="/foo/bar/opencos"\''
|
|
1324
|
+
# So we have to check for strings that have been escaped for shell with extra single
|
|
1325
|
+
# quotes.
|
|
1129
1326
|
m = re.match(r"^\'?\+\w+", token)
|
|
1130
1327
|
if m:
|
|
1131
1328
|
# Copy and strip all outer ' or " on the plusarg:
|
|
1132
|
-
plusarg =
|
|
1329
|
+
plusarg = strip_outer_quotes(token)
|
|
1133
1330
|
self.process_plusarg(plusarg, pwd=pwd)
|
|
1134
1331
|
remove_list.append(token)
|
|
1135
1332
|
for x in remove_list:
|
|
1136
1333
|
unparsed.remove(x)
|
|
1137
1334
|
|
|
1138
|
-
if
|
|
1335
|
+
if not unparsed and self.error_on_no_files_or_targets:
|
|
1139
1336
|
# derived classes can set error_on_no_files_or_targets=True
|
|
1140
1337
|
# For example: CommandSim will error (requires files/targets),
|
|
1141
1338
|
# CommandWaves does not (files/targets not required)
|
|
@@ -1143,11 +1340,11 @@ class CommandDesign(Command):
|
|
|
1143
1340
|
# check the DEPS markup file for a single target, and if so run it
|
|
1144
1341
|
# on the one and only target.
|
|
1145
1342
|
if self.config['deps_markup_supported']:
|
|
1146
|
-
deps =
|
|
1343
|
+
deps = DepsFile(
|
|
1147
1344
|
command_design_ref=self, target_path=os.getcwd(), cache=self.cached_deps
|
|
1148
1345
|
)
|
|
1149
1346
|
if deps.deps_file and deps.data:
|
|
1150
|
-
all_targets =
|
|
1347
|
+
all_targets = deps_data_get_all_targets(deps.data)
|
|
1151
1348
|
if all_targets:
|
|
1152
1349
|
target = all_targets[-1]
|
|
1153
1350
|
unparsed.append(target)
|
|
@@ -1169,7 +1366,7 @@ class CommandDesign(Command):
|
|
|
1169
1366
|
file_exists, fpath, forced_extension = files.get_source_file(token)
|
|
1170
1367
|
if file_exists:
|
|
1171
1368
|
file_abspath = os.path.abspath(fpath)
|
|
1172
|
-
|
|
1369
|
+
_, file_ext = os.path.splitext(file_abspath)
|
|
1173
1370
|
if not forced_extension and not file_ext:
|
|
1174
1371
|
# This probably isn't a file we want to use
|
|
1175
1372
|
util.warning(f'looking for deps {token=}, found {file_abspath=}' \
|
|
@@ -1189,11 +1386,12 @@ class CommandDesign(Command):
|
|
|
1189
1386
|
remove_list.append(token)
|
|
1190
1387
|
continue # done with token, consume it, we added the file.
|
|
1191
1388
|
|
|
1192
|
-
# we appear to be dealing with a target name which needs to be resolved (usually
|
|
1389
|
+
# we appear to be dealing with a target name which needs to be resolved (usually
|
|
1390
|
+
# recursively)
|
|
1193
1391
|
if token.startswith(os.sep):
|
|
1194
1392
|
target_name = token # if it's absolute path, don't prepend anything
|
|
1195
1393
|
else:
|
|
1196
|
-
target_name = os.path.join(".", token) # prepend ./
|
|
1394
|
+
target_name = os.path.join(".", token) # prepend ./ to make it like: <path>/<file>
|
|
1197
1395
|
|
|
1198
1396
|
util.debug(f'Calling self.resolve_target on {target_name=} ({token=})')
|
|
1199
1397
|
if self.resolve_target(target_name, caller_info=caller_info):
|
|
@@ -1242,49 +1440,81 @@ class CommandDesign(Command):
|
|
|
1242
1440
|
self.target = self.args['top']
|
|
1243
1441
|
|
|
1244
1442
|
if self.error_on_missing_top and not self.args.get('top', ''):
|
|
1245
|
-
self.error(
|
|
1443
|
+
self.error("Did not get a --top or DEPS top, required to run command",
|
|
1246
1444
|
f"'{self.command_name}' for tool={self.args.get('tool', None)}")
|
|
1247
1445
|
|
|
1248
1446
|
return unparsed
|
|
1249
1447
|
|
|
1250
1448
|
|
|
1251
|
-
def get_command_line_args(
|
|
1449
|
+
def get_command_line_args( # pylint: disable=dangerous-default-value
|
|
1450
|
+
self, remove_args: list = [], remove_args_startswith: list = []
|
|
1451
|
+
) -> list:
|
|
1252
1452
|
'''Returns a list of all the args if you wanted to re-run this command
|
|
1253
1453
|
(excludes eda, command, target).'''
|
|
1254
1454
|
|
|
1255
1455
|
# This will not set bool's that are False, does not add --no-<somearg>
|
|
1256
|
-
# nor --<somearg>=False
|
|
1257
|
-
# This will not set str's that are empty
|
|
1258
|
-
# this will not set ints that are 0
|
|
1456
|
+
# nor --<somearg>=False (it's unclear which are 'store_true' v/ bool-action-kwargs)
|
|
1457
|
+
# This will not set str's that are empty
|
|
1458
|
+
# this will not set ints that are 0 (unless modified)
|
|
1459
|
+
# TODO(drew): we may want to revisit this if we tracked default arg values,
|
|
1460
|
+
# or perhaps if they have been "modified".
|
|
1259
1461
|
ret = []
|
|
1260
1462
|
for k,v in self.args.items():
|
|
1261
1463
|
|
|
1262
1464
|
# Some args cannot be extracted and work, so omit these:
|
|
1263
1465
|
if k in ['top-path'] + remove_args:
|
|
1264
1466
|
continue
|
|
1265
|
-
if any(
|
|
1467
|
+
if any(k.startswith(x) for x in remove_args_startswith):
|
|
1266
1468
|
continue
|
|
1267
1469
|
|
|
1268
|
-
|
|
1470
|
+
is_modified = self.modified_args.get(k, False)
|
|
1471
|
+
|
|
1472
|
+
if isinstance(v, bool) and v:
|
|
1269
1473
|
ret.append(f'--{k}')
|
|
1270
|
-
elif
|
|
1474
|
+
elif isinstance(v, int) and (bool(v) or is_modified):
|
|
1271
1475
|
ret.append(f'--{k}={v}')
|
|
1272
|
-
elif
|
|
1476
|
+
elif isinstance(v, str) and v:
|
|
1273
1477
|
ret.append(f'--{k}={v}')
|
|
1274
|
-
elif
|
|
1478
|
+
elif isinstance(v, list):
|
|
1275
1479
|
for item in v:
|
|
1276
|
-
if item or
|
|
1480
|
+
if item or isinstance(item, (bool, str)):
|
|
1277
1481
|
# don't print bool/str that are blank.
|
|
1278
1482
|
ret.append(f'--{k}={item}') # lists append
|
|
1279
1483
|
|
|
1280
1484
|
return ret
|
|
1281
1485
|
|
|
1282
1486
|
|
|
1283
|
-
|
|
1284
|
-
|
|
1487
|
+
#_THREADS_START = 0
|
|
1488
|
+
#_THREADS_DONE = 0
|
|
1489
|
+
|
|
1490
|
+
class ThreadStats:
|
|
1491
|
+
'''To avoid globals for two ints, keep a class holder so CommandParallel and
|
|
1492
|
+
CommandParallelWorker can share values'''
|
|
1493
|
+
|
|
1494
|
+
done = 0
|
|
1495
|
+
started = 0
|
|
1496
|
+
|
|
1497
|
+
def all_done(self) -> bool:
|
|
1498
|
+
'''Returns True if all started jobs are done'''
|
|
1499
|
+
return self.started == self.done
|
|
1500
|
+
|
|
1501
|
+
def get_remaining(self) -> int:
|
|
1502
|
+
'''Returns remaining number of jobs'''
|
|
1503
|
+
return max(self.started - self.done, 0)
|
|
1504
|
+
|
|
1505
|
+
def __repr__(self) -> str:
|
|
1506
|
+
'''Pretty print DONE/STARTED'''
|
|
1507
|
+
return f'{self.done}/{self.started}'
|
|
1508
|
+
|
|
1285
1509
|
|
|
1286
1510
|
class CommandParallelWorker(threading.Thread):
|
|
1287
|
-
|
|
1511
|
+
'''Helper class for a single threaded job, often run by a CommandParallel command
|
|
1512
|
+
handler class'''
|
|
1513
|
+
|
|
1514
|
+
def __init__(
|
|
1515
|
+
self, n, work_queue: queue.Queue, done_queue: queue.Queue,
|
|
1516
|
+
threads_stats: ThreadStats
|
|
1517
|
+
):
|
|
1288
1518
|
threading.Thread.__init__(self)
|
|
1289
1519
|
self.n = n
|
|
1290
1520
|
self.work_queue = work_queue
|
|
@@ -1294,54 +1524,46 @@ class CommandParallelWorker(threading.Thread):
|
|
|
1294
1524
|
self.proc = None
|
|
1295
1525
|
self.pid = None
|
|
1296
1526
|
self.last_timer_debug = 0
|
|
1527
|
+
self.threads_stats = threads_stats # ref to shared object
|
|
1297
1528
|
util.debug(f"WORKER_{n}: START")
|
|
1298
1529
|
|
|
1299
1530
|
def run(self):
|
|
1300
|
-
|
|
1301
|
-
|
|
1531
|
+
'''Runs this single job via threading. This is typically created and called by
|
|
1532
|
+
|
|
1533
|
+
a CommandParallel class handler
|
|
1534
|
+
'''
|
|
1535
|
+
|
|
1302
1536
|
while True:
|
|
1303
1537
|
# Get the work from the queue and expand the tuple
|
|
1304
|
-
i, command_list, job_name,
|
|
1538
|
+
i, command_list, job_name, _ = self.work_queue.get()
|
|
1305
1539
|
self.job_name = job_name
|
|
1306
1540
|
try:
|
|
1307
1541
|
util.debug(f"WORKER_{self.n}: Running job {i}: {job_name}")
|
|
1308
|
-
PIPE=subprocess.PIPE
|
|
1309
|
-
STDOUT=subprocess.STDOUT
|
|
1310
1542
|
util.debug(f"WORKER_{self.n}: Calling Popen")
|
|
1311
|
-
proc = subprocess.Popen(
|
|
1543
|
+
proc = subprocess.Popen( # pylint: disable=consider-using-with
|
|
1544
|
+
command_list, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
|
|
1545
|
+
)
|
|
1312
1546
|
self.proc = proc
|
|
1313
1547
|
util.debug(f"WORKER_{self.n}: Opened process, PID={proc.pid}")
|
|
1314
1548
|
self.pid = proc.pid
|
|
1315
|
-
|
|
1316
|
-
while proc.returncode
|
|
1549
|
+
self.threads_stats.started += 1
|
|
1550
|
+
while proc.returncode is None:
|
|
1317
1551
|
try:
|
|
1318
1552
|
if (time.time() - self.last_timer_debug) > 10:
|
|
1319
1553
|
util.debug(f"WORKER_{self.n}: Calling proc.communicate")
|
|
1320
1554
|
stdout, stderr = proc.communicate(timeout=0.5)
|
|
1321
|
-
util.debug(f"WORKER_{self.n}: got: \n*** stdout:\n{stdout}\n***
|
|
1555
|
+
util.debug(f"WORKER_{self.n}: got: \n*** stdout:\n{stdout}\n***",
|
|
1556
|
+
f"stderr:{stderr}")
|
|
1322
1557
|
except subprocess.TimeoutExpired:
|
|
1323
1558
|
if (time.time() - self.last_timer_debug) > 10:
|
|
1324
|
-
util.debug(f"WORKER_{self.n}: Timer expired, stop_request=
|
|
1559
|
+
util.debug(f"WORKER_{self.n}: Timer expired, stop_request=",
|
|
1560
|
+
f"{self.stop_request}")
|
|
1325
1561
|
self.last_timer_debug = time.time()
|
|
1326
|
-
pass
|
|
1327
1562
|
if self.stop_request:
|
|
1328
1563
|
util.debug(f"WORKER_{self.n}: got stop request, issuing SIGINT")
|
|
1329
1564
|
proc.send_signal(signal.SIGINT)
|
|
1330
1565
|
util.debug(f"WORKER_{self.n}: got stop request, calling proc.wait")
|
|
1331
1566
|
proc.wait()
|
|
1332
|
-
if False and self.stop_request:
|
|
1333
|
-
util.debug(f"WORKER_{self.n}: got stop request, issuing proc.terminate")
|
|
1334
|
-
proc.terminate()
|
|
1335
|
-
util.debug(f"WORKER_{self.n}: proc poll returns is now {proc.poll()}")
|
|
1336
|
-
try:
|
|
1337
|
-
util.debug(f"WORKER_{self.n}: Calling proc.communicate")
|
|
1338
|
-
stdout, stderr = proc.communicate(timeout=0.2) # for completeness, in case we ever pipe/search stdout/stderr
|
|
1339
|
-
util.debug(f"WORKER_{self.n}: got: \n*** stdout:\n{stdout}\n*** stderr:{stderr}")
|
|
1340
|
-
except subprocess.TimeoutExpired:
|
|
1341
|
-
util.debug(f"WORKER_{self.n}: timeout waiting for comminicate after terminate")
|
|
1342
|
-
except:
|
|
1343
|
-
pass
|
|
1344
|
-
util.debug(f"WORKER_{self.n}: proc poll returns is now {proc.poll()}")
|
|
1345
1567
|
|
|
1346
1568
|
util.debug(f"WORKER_{self.n}: -- out of while loop")
|
|
1347
1569
|
self.pid = None
|
|
@@ -1350,99 +1572,113 @@ class CommandParallelWorker(threading.Thread):
|
|
|
1350
1572
|
util.debug(f"WORKER_{self.n}: proc poll returns is now {proc.poll()}")
|
|
1351
1573
|
try:
|
|
1352
1574
|
util.debug(f"WORKER_{self.n}: Calling proc.communicate one last time")
|
|
1353
|
-
stdout, stderr = proc.communicate(timeout=0.1)
|
|
1354
|
-
util.debug(f"WORKER_{self.n}: got: \n*** stdout:\n{stdout}\n***
|
|
1575
|
+
stdout, stderr = proc.communicate(timeout=0.1)
|
|
1576
|
+
util.debug(f"WORKER_{self.n}: got: \n*** stdout:\n{stdout}\n***",
|
|
1577
|
+
f"stderr:{stderr}")
|
|
1355
1578
|
except subprocess.TimeoutExpired:
|
|
1356
1579
|
util.debug(f"WORKER_{self.n}: timeout waiting for communicate after loop?")
|
|
1357
|
-
except:
|
|
1358
|
-
|
|
1580
|
+
except Exception as e:
|
|
1581
|
+
util.debug(f"WORKER_{self.n}: timeout with exception {e=}")
|
|
1582
|
+
|
|
1359
1583
|
return_code = proc.poll()
|
|
1360
|
-
util.debug(f"WORKER_{self.n}: Finished job {i}: {job_name} with return code
|
|
1584
|
+
util.debug(f"WORKER_{self.n}: Finished job {i}: {job_name} with return code",
|
|
1585
|
+
f"{return_code}")
|
|
1361
1586
|
self.done_queue.put((i, job_name, return_code))
|
|
1362
1587
|
finally:
|
|
1363
1588
|
util.debug(f"WORKER_{self.n}: -- in finally block")
|
|
1364
1589
|
self.work_queue.task_done()
|
|
1365
|
-
|
|
1590
|
+
self.threads_stats.done += 1
|
|
1366
1591
|
|
|
1367
1592
|
|
|
1368
1593
|
class CommandParallel(Command):
|
|
1594
|
+
'''Base class command handler for running multiple eda jobs, such as for child classes
|
|
1595
|
+
|
|
1596
|
+
in commands.multi or commands.sweep
|
|
1597
|
+
'''
|
|
1369
1598
|
def __init__(self, config: dict):
|
|
1370
1599
|
Command.__init__(self, config=config)
|
|
1371
1600
|
self.jobs = []
|
|
1372
1601
|
self.jobs_status = []
|
|
1373
1602
|
self.args['parallel'] = 1
|
|
1374
1603
|
self.worker_threads = []
|
|
1604
|
+
self.threads_stats = ThreadStats()
|
|
1375
1605
|
|
|
1376
|
-
def __del__(self):
|
|
1377
|
-
util.debug(f"In Command.__del__, threads done/started: {
|
|
1378
|
-
if
|
|
1606
|
+
def __del__(self): # pylint: disable=too-many-branches
|
|
1607
|
+
util.debug(f"In Command.__del__, threads done/started: {self.threads_stats}")
|
|
1608
|
+
if self.threads_stats.all_done():
|
|
1379
1609
|
return
|
|
1380
|
-
util.warning(f"Need to shut down {
|
|
1610
|
+
util.warning(f"Need to shut down {self.threads_stats.get_remaining()} worker threads...")
|
|
1381
1611
|
for w in self.worker_threads:
|
|
1382
1612
|
if w.proc:
|
|
1383
1613
|
util.warning(f"Requesting stop of PID {w.pid}: {w.job_name}")
|
|
1384
1614
|
w.stop_request = True
|
|
1385
|
-
for
|
|
1386
|
-
util.debug(f"Threads done/started: {
|
|
1387
|
-
if
|
|
1388
|
-
util.info(
|
|
1615
|
+
for _ in range(10):
|
|
1616
|
+
util.debug(f"Threads done/started: {self.threads_stats}")
|
|
1617
|
+
if self.threads_stats.started == self.threads_stats.done:
|
|
1618
|
+
util.info("All threads done")
|
|
1389
1619
|
return
|
|
1390
1620
|
time.sleep(1)
|
|
1391
|
-
subprocess.Popen(['stty', 'sane']).wait()
|
|
1392
|
-
util.debug(
|
|
1621
|
+
subprocess.Popen(['stty', 'sane']).wait() # pylint: disable=consider-using-with
|
|
1622
|
+
util.debug("Scanning workers again")
|
|
1393
1623
|
for w in self.worker_threads:
|
|
1394
1624
|
if w.proc:
|
|
1395
1625
|
util.info(f"need to SIGINT WORKER_{w.n}, may need manual cleanup, check 'ps'")
|
|
1396
1626
|
if w.pid:
|
|
1397
1627
|
os.kill(w.pid, signal.SIGINT)
|
|
1398
|
-
for
|
|
1399
|
-
util.debug(f"Threads done/started: {
|
|
1400
|
-
if
|
|
1401
|
-
util.info(
|
|
1628
|
+
for _ in range(5):
|
|
1629
|
+
util.debug(f"Threads done/started: {self.threads_stats}")
|
|
1630
|
+
if self.threads_stats.all_done():
|
|
1631
|
+
util.info("All threads done")
|
|
1402
1632
|
return
|
|
1403
1633
|
time.sleep(1)
|
|
1404
|
-
subprocess.Popen(['stty', 'sane']).wait()
|
|
1405
|
-
util.debug(
|
|
1634
|
+
subprocess.Popen(['stty', 'sane']).wait() # pylint: disable=consider-using-with
|
|
1635
|
+
util.debug("Scanning workers again")
|
|
1406
1636
|
for w in self.worker_threads:
|
|
1407
1637
|
if w.proc:
|
|
1408
1638
|
util.info(f"need to TERM WORKER_{w.n}, probably needs manual cleanup, check 'ps'")
|
|
1409
1639
|
if w.pid:
|
|
1410
1640
|
os.kill(w.pid, signal.SIGTERM)
|
|
1411
|
-
for
|
|
1412
|
-
util.debug(f"Threads done/started: {
|
|
1413
|
-
if
|
|
1414
|
-
util.info(
|
|
1641
|
+
for _ in range(5):
|
|
1642
|
+
util.debug(f"Threads done/started: {self.threads_stats}")
|
|
1643
|
+
if self.threads_stats.all_done():
|
|
1644
|
+
util.info("All threads done")
|
|
1415
1645
|
return
|
|
1416
1646
|
time.sleep(1)
|
|
1417
|
-
subprocess.Popen(['stty', 'sane']).wait()
|
|
1418
|
-
util.debug(
|
|
1647
|
+
subprocess.Popen(['stty', 'sane']).wait() # pylint: disable=consider-using-with
|
|
1648
|
+
util.debug("Scanning workers again")
|
|
1419
1649
|
for w in self.worker_threads:
|
|
1420
1650
|
if w.proc:
|
|
1421
1651
|
util.info(f"need to KILL WORKER_{w.n}, probably needs manual cleanup, check 'ps'")
|
|
1422
1652
|
if w.pid:
|
|
1423
1653
|
os.kill(w.pid, signal.SIGKILL)
|
|
1424
1654
|
util.stop_log()
|
|
1425
|
-
subprocess.Popen(['stty', 'sane']).wait()
|
|
1655
|
+
subprocess.Popen(['stty', 'sane']).wait() # pylint: disable=consider-using-with
|
|
1656
|
+
|
|
1657
|
+
def run_jobs( # pylint: disable=too-many-locals,too-many-branches,too-many-statements
|
|
1658
|
+
self, command: str
|
|
1659
|
+
) -> None:
|
|
1660
|
+
'''Runs all "jobs" in self.jobs list, either serially or in parallel'''
|
|
1426
1661
|
|
|
1427
|
-
|
|
1428
|
-
#
|
|
1429
|
-
#
|
|
1430
|
-
#
|
|
1431
|
-
# UI output functional and awesome sauce
|
|
1662
|
+
# this is where we actually run the jobs. it's a messy piece of code and prob could use
|
|
1663
|
+
# refactoring but the goal was to share as much as possible (job start, end, pass/fail
|
|
1664
|
+
# judgement, etc) while supporting various mode combinations (parallel mode, verbose mode,
|
|
1665
|
+
# fancy mode, etc) and keeping the UI output functional and awesome sauce
|
|
1432
1666
|
|
|
1433
1667
|
# walk targets to find the longest name, for display reasons
|
|
1434
1668
|
longest_job_name = 0
|
|
1435
1669
|
total_jobs = len(self.jobs)
|
|
1436
1670
|
self.jobs_status = [None] * total_jobs
|
|
1437
1671
|
for i in range(total_jobs):
|
|
1438
|
-
|
|
1439
|
-
if l>longest_job_name: longest_job_name = l
|
|
1672
|
+
longest_job_name = max(longest_job_name, len(self.jobs[i]['name']))
|
|
1440
1673
|
|
|
1441
1674
|
run_parallel = self.args['parallel'] > 1
|
|
1442
1675
|
|
|
1443
1676
|
# figure out the width to print various numbers
|
|
1444
1677
|
jobs_digits = len(f"{total_jobs}")
|
|
1445
|
-
|
|
1678
|
+
|
|
1679
|
+
job_done_return_code = None
|
|
1680
|
+
job_done_run_time = 0
|
|
1681
|
+
job_done_name = ''
|
|
1446
1682
|
|
|
1447
1683
|
# run the jobs!
|
|
1448
1684
|
running_jobs = {}
|
|
@@ -1453,9 +1689,10 @@ class CommandParallel(Command):
|
|
|
1453
1689
|
jobs_launched = 0
|
|
1454
1690
|
num_parallel = min(len(self.jobs), self.args['parallel'])
|
|
1455
1691
|
# 16 should really be the size of window or ?
|
|
1456
|
-
|
|
1457
|
-
# we will enter fancy mode if we are parallel and we can leave 6 lines of regular scrolling
|
|
1458
|
-
|
|
1692
|
+
_, lines = shutil.get_terminal_size()
|
|
1693
|
+
# we will enter fancy mode if we are parallel and we can leave 6 lines of regular scrolling
|
|
1694
|
+
# output
|
|
1695
|
+
fancy_mode = all([util.args['fancy'], num_parallel > 1, num_parallel <= (lines-6)])
|
|
1459
1696
|
multi_cwd = util.getcwd() + os.sep
|
|
1460
1697
|
|
|
1461
1698
|
self.patch_jobs_for_duplicate_target_names()
|
|
@@ -1466,8 +1703,12 @@ class CommandParallel(Command):
|
|
|
1466
1703
|
work_queue = queue.Queue()
|
|
1467
1704
|
done_queue = queue.Queue()
|
|
1468
1705
|
for x in range(num_parallel):
|
|
1469
|
-
worker = CommandParallelWorker(
|
|
1470
|
-
|
|
1706
|
+
worker = CommandParallelWorker(
|
|
1707
|
+
n=x, work_queue=work_queue, done_queue=done_queue,
|
|
1708
|
+
threads_stats=self.threads_stats
|
|
1709
|
+
)
|
|
1710
|
+
# Setting daemon to True will let the main thread exit even though the workers are
|
|
1711
|
+
# blocking
|
|
1471
1712
|
worker.daemon = True
|
|
1472
1713
|
worker.start()
|
|
1473
1714
|
self.worker_threads.append(worker)
|
|
@@ -1478,22 +1719,30 @@ class CommandParallel(Command):
|
|
|
1478
1719
|
for x in range(num_parallel):
|
|
1479
1720
|
util.fancy_print(f"Starting worker {x}", x)
|
|
1480
1721
|
|
|
1481
|
-
while
|
|
1722
|
+
while self.jobs or running_jobs:
|
|
1482
1723
|
job_done = False
|
|
1483
1724
|
job_done_quiet = False
|
|
1484
1725
|
anything_done = False
|
|
1485
1726
|
|
|
1486
1727
|
def sprint_job_line(job_number=0, job_name="", final=False, hide_stats=False):
|
|
1487
|
-
return (
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1728
|
+
return (
|
|
1729
|
+
"INFO: [EDA] " +
|
|
1730
|
+
string_or_space(
|
|
1731
|
+
f"[job {job_number:0{jobs_digits}d}/{total_jobs:0{jobs_digits}d} ",
|
|
1732
|
+
final) +
|
|
1733
|
+
string_or_space("| pass ", hide_stats or final) +
|
|
1734
|
+
string_or_space(
|
|
1735
|
+
f"{len(passed_jobs):0{jobs_digits}d}/{jobs_complete:0{jobs_digits}d} ",
|
|
1736
|
+
hide_stats) +
|
|
1737
|
+
string_or_space(f"@ {(100 * (jobs_complete)) / total_jobs : 5.1f}%",
|
|
1738
|
+
hide_stats or final) +
|
|
1739
|
+
string_or_space("] ", final) +
|
|
1740
|
+
f"{command} {(job_name+' ').ljust(longest_job_name+3,'.')}"
|
|
1741
|
+
)
|
|
1742
|
+
|
|
1743
|
+
# for any kind of run (parallel or not, fancy or not, verbose or not) ...
|
|
1744
|
+
# can we launch a job?
|
|
1745
|
+
if self.jobs and (len(running_jobs) < num_parallel):
|
|
1497
1746
|
# we are launching a job
|
|
1498
1747
|
jobs_launched += 1
|
|
1499
1748
|
anything_done = True
|
|
@@ -1501,15 +1750,17 @@ class CommandParallel(Command):
|
|
|
1501
1750
|
# TODO(drew): it might be nice to pass more items on 'job' dict, like
|
|
1502
1751
|
# logfile or job-name, so CommandSweep or CommandMulti don't have to set
|
|
1503
1752
|
# via args. on their command_list.
|
|
1504
|
-
if job['name'].startswith(multi_cwd):
|
|
1505
|
-
|
|
1753
|
+
if job['name'].startswith(multi_cwd):
|
|
1754
|
+
job['name'] = job['name'][len(multi_cwd):]
|
|
1755
|
+
# in all but fancy mode, we will print this text at the launch of a job. It may
|
|
1756
|
+
# get a newline below
|
|
1506
1757
|
job_text = sprint_job_line(jobs_launched, job['name'], hide_stats=run_parallel)
|
|
1507
1758
|
command_list = job['command_list']
|
|
1508
1759
|
cwd = util.getcwd()
|
|
1509
1760
|
|
|
1510
1761
|
if run_parallel:
|
|
1511
1762
|
# multithreaded job launch: add to queue
|
|
1512
|
-
worker = workers.pop(0)
|
|
1763
|
+
worker = workers.pop(0)
|
|
1513
1764
|
running_jobs[str(jobs_launched)] = { 'name' : job['name'],
|
|
1514
1765
|
'number' : jobs_launched,
|
|
1515
1766
|
'worker' : worker,
|
|
@@ -1519,15 +1770,20 @@ class CommandParallel(Command):
|
|
|
1519
1770
|
suffix = "<START>"
|
|
1520
1771
|
if fancy_mode:
|
|
1521
1772
|
util.fancy_print(job_text+suffix, worker)
|
|
1773
|
+
elif failed_jobs:
|
|
1774
|
+
# if we aren't in fancy mode, we will print a START line, periodic RUNNING
|
|
1775
|
+
# lines, and PASS/FAIL line per-job
|
|
1776
|
+
util.print_orange(job_text + Colors.yellow + suffix)
|
|
1522
1777
|
else:
|
|
1523
|
-
|
|
1524
|
-
if len(failed_jobs): util.print_orange(job_text + util.string_yellow + suffix)
|
|
1525
|
-
else: util.print_yellow(job_text + util.string_yellow + suffix)
|
|
1778
|
+
util.print_yellow(job_text + Colors.yellow + suffix)
|
|
1526
1779
|
else:
|
|
1527
|
-
# single-threaded job launch, we are going to print out job info as we start
|
|
1528
|
-
# since non-verbose silences the job and prints only
|
|
1529
|
-
|
|
1530
|
-
|
|
1780
|
+
# single-threaded job launch, we are going to print out job info as we start
|
|
1781
|
+
# each job... no newline. since non-verbose silences the job and prints only
|
|
1782
|
+
# <PASS>/<FAIL> after the trailing "..." we leave here
|
|
1783
|
+
if failed_jobs:
|
|
1784
|
+
util.print_orange(job_text, end="")
|
|
1785
|
+
else:
|
|
1786
|
+
util.print_yellow(job_text, end="")
|
|
1531
1787
|
job_done_number = jobs_launched
|
|
1532
1788
|
job_done_name = job['name']
|
|
1533
1789
|
job_start_time = time.time()
|
|
@@ -1539,13 +1795,16 @@ class CommandParallel(Command):
|
|
|
1539
1795
|
_, _, job_done_return_code = self.exec(
|
|
1540
1796
|
cwd, command_list, background=False, stop_on_error=False, quiet=False
|
|
1541
1797
|
)
|
|
1542
|
-
# reprint the job text previously printed before running job(and given
|
|
1798
|
+
# reprint the job text previously printed before running job (and given
|
|
1799
|
+
# "\n" after the trailing "...")
|
|
1543
1800
|
else:
|
|
1544
1801
|
# run job, swallowing output (hope you have a logfile)
|
|
1545
1802
|
_, _, job_done_return_code = self.exec(
|
|
1546
1803
|
cwd, command_list, background=True, stop_on_error=False, quiet=True
|
|
1547
1804
|
)
|
|
1548
|
-
|
|
1805
|
+
# in this case, we have the job start text (trailing "...", no newline)
|
|
1806
|
+
# printed
|
|
1807
|
+
job_done_quiet = True
|
|
1549
1808
|
job_done = True
|
|
1550
1809
|
job_done_run_time = time.time() - job_start_time
|
|
1551
1810
|
# Since we consumed the job, use the job['index'] to track the per-job status:
|
|
@@ -1553,15 +1812,16 @@ class CommandParallel(Command):
|
|
|
1553
1812
|
if run_parallel:
|
|
1554
1813
|
# parallel run, check for completed job
|
|
1555
1814
|
if done_queue.qsize():
|
|
1556
|
-
# we're collecting a finished job from a worker thread. note we will only
|
|
1557
|
-
# loop, so as to share job completion code
|
|
1815
|
+
# we're collecting a finished job from a worker thread. note we will only
|
|
1816
|
+
# reap one job per iter of the big loop, so as to share job completion code
|
|
1817
|
+
# at the bottom
|
|
1558
1818
|
anything_done = True
|
|
1559
1819
|
job_done = True
|
|
1560
1820
|
job_done_number, job_done_name, job_done_return_code = done_queue.get()
|
|
1561
1821
|
t = running_jobs[str(job_done_number)]
|
|
1562
1822
|
# in fancy mode, we need to clear the worker line related to this job.
|
|
1563
1823
|
if fancy_mode:
|
|
1564
|
-
util.fancy_print(
|
|
1824
|
+
util.fancy_print("INFO: [EDA] Parallel: Worker Idle ...", t['worker'])
|
|
1565
1825
|
job_done_run_time = time.time() - t['start_time']
|
|
1566
1826
|
util.debug(f"removing job #{job_done_number} from running jobs")
|
|
1567
1827
|
del running_jobs[str(job_done_number)]
|
|
@@ -1573,33 +1833,40 @@ class CommandParallel(Command):
|
|
|
1573
1833
|
if (fancy_mode or (time.time() - t['update_time']) > 30):
|
|
1574
1834
|
t['update_time'] = time.time()
|
|
1575
1835
|
job_text = sprint_job_line(t['number'], t['name'], hide_stats=True)
|
|
1576
|
-
suffix = f"<RUNNING: {
|
|
1836
|
+
suffix = f"<RUNNING: {sprint_time(time.time() - t['start_time'])}>"
|
|
1577
1837
|
if fancy_mode:
|
|
1578
1838
|
util.fancy_print(f"{job_text}{suffix}", t['worker'])
|
|
1839
|
+
elif failed_jobs:
|
|
1840
|
+
util.print_orange(job_text + Colors.yellow + suffix)
|
|
1579
1841
|
else:
|
|
1580
|
-
|
|
1581
|
-
else: util.print_yellow(job_text+util.string_yellow+suffix)
|
|
1842
|
+
util.print_yellow(job_text + Colors.yellow + suffix)
|
|
1582
1843
|
|
|
1583
1844
|
# shared job completion code
|
|
1584
|
-
# single or multi-threaded, we can arrive here to harvest <= 1 jobs, and need
|
|
1585
|
-
# we expect the start of a status line to have been
|
|
1845
|
+
# single or multi-threaded, we can arrive here to harvest <= 1 jobs, and need
|
|
1846
|
+
# {job, return_code} valid, and we expect the start of a status line to have been
|
|
1847
|
+
# printed, ready for pass/fail
|
|
1586
1848
|
if job_done:
|
|
1587
1849
|
jobs_complete += 1
|
|
1588
1850
|
if job_done_return_code is None or job_done_return_code:
|
|
1589
|
-
# embed the color code, to change color of pass/fail during the
|
|
1851
|
+
# embed the color code, to change color of pass/fail during the
|
|
1852
|
+
# util.print_orange/yellow below
|
|
1590
1853
|
if job_done_return_code == 124:
|
|
1591
|
-
# bash uses 124 for bash timeout errors, if that was preprended to the
|
|
1592
|
-
|
|
1854
|
+
# bash uses 124 for bash timeout errors, if that was preprended to the
|
|
1855
|
+
# command list.
|
|
1856
|
+
suffix = f"{Colors.red}<TOUT: {sprint_time(job_done_run_time)}>"
|
|
1593
1857
|
else:
|
|
1594
|
-
suffix = f"{
|
|
1858
|
+
suffix = f"{Colors.red}<FAIL: {sprint_time(job_done_run_time)}>"
|
|
1595
1859
|
failed_jobs.append(job_done_name)
|
|
1596
1860
|
else:
|
|
1597
|
-
suffix = f"{
|
|
1861
|
+
suffix = f"{Colors.green}<PASS: {sprint_time(job_done_run_time)}>"
|
|
1598
1862
|
passed_jobs.append(job_done_name)
|
|
1599
1863
|
# we want to print in one shot, because in fancy modes that's all that we're allowed
|
|
1600
|
-
job_done_text = "" if job_done_quiet else sprint_job_line(job_done_number,
|
|
1601
|
-
|
|
1602
|
-
|
|
1864
|
+
job_done_text = "" if job_done_quiet else sprint_job_line(job_done_number,
|
|
1865
|
+
job_done_name)
|
|
1866
|
+
if failed_jobs:
|
|
1867
|
+
util.print_orange(f"{job_done_text}{suffix}")
|
|
1868
|
+
else:
|
|
1869
|
+
util.print_yellow(f"{job_done_text}{suffix}")
|
|
1603
1870
|
self.jobs_status[job_done_number-1] = job_done_return_code
|
|
1604
1871
|
|
|
1605
1872
|
if not anything_done:
|
|
@@ -1607,23 +1874,26 @@ class CommandParallel(Command):
|
|
|
1607
1874
|
|
|
1608
1875
|
if total_jobs:
|
|
1609
1876
|
emoji = "< :) >" if (len(passed_jobs) == total_jobs) else "< :( >"
|
|
1610
|
-
util.info(sprint_job_line(final=True,job_name="jobs passed")+emoji, start="")
|
|
1877
|
+
util.info(sprint_job_line(final=True, job_name="jobs passed") + emoji, start="")
|
|
1611
1878
|
else:
|
|
1612
|
-
util.info(
|
|
1879
|
+
util.info("Parallel: <No jobs found>")
|
|
1613
1880
|
# Make sure all jobs have a set status:
|
|
1614
|
-
for i,rc in enumerate(self.jobs_status):
|
|
1615
|
-
if rc is None or
|
|
1881
|
+
for i, rc in enumerate(self.jobs_status):
|
|
1882
|
+
if rc is None or not isinstance(rc, int):
|
|
1616
1883
|
self.error(f'job {i=} {rc=} did not return a proper return code')
|
|
1617
|
-
jobs_status[i] =
|
|
1884
|
+
self.jobs_status[i] = 2
|
|
1618
1885
|
|
|
1619
1886
|
# if self.status > 0, then keep it non-zero, else set it if we still have running jobs.
|
|
1620
1887
|
if self.status == 0:
|
|
1621
|
-
|
|
1888
|
+
if self.jobs_status:
|
|
1889
|
+
self.status = max(self.jobs_status)
|
|
1890
|
+
# else keep at 0, empty list.
|
|
1622
1891
|
util.fancy_stop()
|
|
1623
1892
|
|
|
1624
1893
|
@staticmethod
|
|
1625
1894
|
def get_name_from_target(target: str) -> str:
|
|
1626
|
-
|
|
1895
|
+
'''Given a target path, strip leftmost path separators to get a shorter string name'''
|
|
1896
|
+
return target.replace('../', '').lstrip('./').lstrip(os.path.sep)
|
|
1627
1897
|
|
|
1628
1898
|
|
|
1629
1899
|
def update_args_list(self, args: list, tool: str) -> None:
|
|
@@ -1667,7 +1937,7 @@ class CommandParallel(Command):
|
|
|
1667
1937
|
tokens=tokens.copy(),
|
|
1668
1938
|
apply_parsed_args=False,
|
|
1669
1939
|
)
|
|
1670
|
-
util.debug(f'{self.command_name}: {single_cmd_unparsed=}')
|
|
1940
|
+
util.debug(f'{self.command_name}: {single_cmd_parsed=}, {single_cmd_unparsed=}')
|
|
1671
1941
|
|
|
1672
1942
|
# There should not be any single_cmd_unparsed args starting with '-'
|
|
1673
1943
|
bad_remaining_args = [x for x in single_cmd_unparsed if x.startswith('-')]
|
|
@@ -1694,7 +1964,7 @@ class CommandParallel(Command):
|
|
|
1694
1964
|
if item.startswith(f'--{arg_name}='):
|
|
1695
1965
|
_, name = item.split(f'--{arg_name}=')
|
|
1696
1966
|
return name
|
|
1697
|
-
|
|
1967
|
+
if item == f'--{arg_name}':
|
|
1698
1968
|
return job_dict['command_list'][i + 1]
|
|
1699
1969
|
return ''
|
|
1700
1970
|
|
|
@@ -1704,7 +1974,7 @@ class CommandParallel(Command):
|
|
|
1704
1974
|
if item.startswith(f'--{arg_name}='):
|
|
1705
1975
|
job_dict['command_list'][i] = f'--{arg_name}=' + new_value
|
|
1706
1976
|
return True
|
|
1707
|
-
|
|
1977
|
+
if item == f'--{arg_name}':
|
|
1708
1978
|
job_dict['command_list'][i + 1] = new_value
|
|
1709
1979
|
return True
|
|
1710
1980
|
return False
|
|
@@ -1729,7 +1999,6 @@ class CommandParallel(Command):
|
|
|
1729
1999
|
tpath, _ = os.path.split(job_dict['target'])
|
|
1730
2000
|
|
|
1731
2001
|
# prepend path information to job-name:
|
|
1732
|
-
patched_sub_work_dir = False
|
|
1733
2002
|
patched_target_path = os.path.relpath(tpath).replace(os.sep, '_')
|
|
1734
2003
|
new_job_name = f'{patched_target_path}.{key}'
|
|
1735
2004
|
replace_job_arg(job_dict, arg_name='job-name', new_value=new_job_name)
|