openubmc-bingo 0.5.277__py3-none-any.whl → 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of openubmc-bingo might be problematic. Click here for more details.
- bmcgo/__init__.py +1 -1
- bmcgo/bmcgo_config.py +22 -10
- bmcgo/cli/cli.py +86 -39
- bmcgo/cli/config.conan2.yaml +9 -0
- bmcgo/codegen/lua/codegen.py +1 -1
- bmcgo/codegen/lua/script/gen_intf_rpc_json.py +15 -14
- bmcgo/component/analysis/analysis.py +8 -8
- bmcgo/component/analysis/intf_validation.py +23 -12
- bmcgo/component/build.py +76 -14
- bmcgo/component/component_dt_version_parse.py +3 -2
- bmcgo/component/component_helper.py +43 -15
- bmcgo/component/coverage/incremental_cov.py +2 -2
- bmcgo/component/deploy.py +68 -14
- bmcgo/component/gen.py +1 -1
- bmcgo/component/package_info.py +128 -38
- bmcgo/component/template_v2/conanbase.py.mako +352 -0
- bmcgo/component/template_v2/conanfile.deploy.py.mako +26 -0
- bmcgo/component/test.py +53 -20
- bmcgo/frame.py +7 -3
- bmcgo/functional/analysis.py +3 -2
- bmcgo/functional/check.py +10 -6
- bmcgo/functional/conan_index_build.py +79 -20
- bmcgo/functional/csr_build.py +1 -1
- bmcgo/functional/diff.py +1 -1
- bmcgo/functional/fetch.py +1 -1
- bmcgo/functional/full_component.py +32 -24
- bmcgo/functional/git_history.py +220 -0
- bmcgo/functional/maintain.py +55 -12
- bmcgo/functional/new.py +1 -1
- bmcgo/functional/schema_valid.py +2 -2
- bmcgo/logger.py +2 -3
- bmcgo/misc.py +130 -9
- bmcgo/tasks/conan/__init__.py +10 -0
- bmcgo/tasks/conan/conanfile.py +45 -0
- bmcgo/tasks/task.py +27 -4
- bmcgo/tasks/task_build_conan.py +399 -52
- bmcgo/tasks/task_buildgppbin.py +8 -2
- bmcgo/tasks/task_download_buildtools.py +76 -0
- bmcgo/tasks/task_download_dependency.py +1 -0
- bmcgo/tasks/task_hpm_envir_prepare.py +1 -1
- bmcgo/utils/build_conans.py +231 -0
- bmcgo/utils/component_post.py +6 -4
- bmcgo/utils/component_version_check.py +10 -5
- bmcgo/utils/config.py +76 -40
- bmcgo/utils/fetch_component_code.py +44 -25
- bmcgo/utils/tools.py +239 -117
- bmcgo/worker.py +2 -2
- {openubmc_bingo-0.5.277.dist-info → openubmc_bingo-0.6.0.dist-info}/METADATA +4 -2
- {openubmc_bingo-0.5.277.dist-info → openubmc_bingo-0.6.0.dist-info}/RECORD +52 -45
- {openubmc_bingo-0.5.277.dist-info → openubmc_bingo-0.6.0.dist-info}/WHEEL +0 -0
- {openubmc_bingo-0.5.277.dist-info → openubmc_bingo-0.6.0.dist-info}/entry_points.txt +0 -0
- {openubmc_bingo-0.5.277.dist-info → openubmc_bingo-0.6.0.dist-info}/top_level.txt +0 -0
bmcgo/tasks/task_build_conan.py
CHANGED
|
@@ -27,10 +27,10 @@ import random
|
|
|
27
27
|
import pathlib
|
|
28
28
|
from multiprocessing import Process, Queue
|
|
29
29
|
from copy import deepcopy
|
|
30
|
+
from collections import Counter
|
|
30
31
|
|
|
31
32
|
import yaml
|
|
32
33
|
from colorama import Fore, Back, Style
|
|
33
|
-
from conans.model.profile import Profile
|
|
34
34
|
|
|
35
35
|
from bmcgo.tasks.task import Task
|
|
36
36
|
from bmcgo.utils.config import Config
|
|
@@ -40,6 +40,9 @@ from bmcgo import errors, misc
|
|
|
40
40
|
from bmcgo.utils.tools import Tools
|
|
41
41
|
from bmcgo.errors import BmcGoException
|
|
42
42
|
from bmcgo.component.component_helper import ComponentHelper
|
|
43
|
+
from bmcgo.utils.build_conans import BuildConans
|
|
44
|
+
from bmcgo.component.deploy import GraphNode
|
|
45
|
+
from bmcgo.tasks.misc import MODULE_SYMVERS, SDK_PATH
|
|
43
46
|
|
|
44
47
|
IBMC_LOCK_FILE = "openubmc.lock"
|
|
45
48
|
SDK_ROOT = "/opt/hi1711sdk"
|
|
@@ -56,14 +59,30 @@ PERSIST_TYPES = {
|
|
|
56
59
|
PRIMARY_KEYS = "PrimaryKeys"
|
|
57
60
|
tools = Tools()
|
|
58
61
|
|
|
62
|
+
PERSIST_TYPES = {
|
|
63
|
+
"TemporaryPer": "TemporaryPer",
|
|
64
|
+
"TemporaryPerRetain": "TemporaryPer",
|
|
65
|
+
"ResetPer": "ResetPer",
|
|
66
|
+
"ResetPerRetain": "ResetPer",
|
|
67
|
+
"PoweroffPer": "PoweroffPer",
|
|
68
|
+
"PoweroffPerRetain": "PoweroffPer",
|
|
69
|
+
"PermanentPer": "PermanentPer"
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
PRIMARY_KEYS = "PrimaryKeys"
|
|
73
|
+
|
|
74
|
+
cwd_dir = os.path.dirname(os.path.realpath(__file__))
|
|
75
|
+
|
|
59
76
|
|
|
60
77
|
class CopyComponent(Process):
|
|
61
|
-
def __init__(self, work: Task, comp: str,
|
|
78
|
+
def __init__(self, work: Task, comp: str, package_folder: str, profile):
|
|
62
79
|
super().__init__()
|
|
63
80
|
self.work = work
|
|
64
81
|
self.config = self.work.config
|
|
65
82
|
self.comp = comp
|
|
83
|
+
self.package_folder = package_folder
|
|
66
84
|
self.profile = profile
|
|
85
|
+
self.graphfile = None
|
|
67
86
|
|
|
68
87
|
def link_recursive_deal(self, file_name, ownership):
|
|
69
88
|
self.work.run_command(f"chown -h {ownership} {file_name}", sudo=True)
|
|
@@ -74,16 +93,16 @@ class CopyComponent(Process):
|
|
|
74
93
|
self.work.work_name = self.comp
|
|
75
94
|
# 复制组件文件到rootfs中
|
|
76
95
|
# rc.sysinit脚本只允许使用rootfs_user仓库的
|
|
77
|
-
rc_sysinit_path = os.path.join(self.
|
|
96
|
+
rc_sysinit_path = os.path.join(self.package_folder, "etc/rc.d/rc.sysinit")
|
|
78
97
|
if self.comp != "rootfs_user" and os.path.isfile(rc_sysinit_path):
|
|
79
98
|
shutil.rmtree(rc_sysinit_path, ignore_errors=True)
|
|
80
99
|
# 由于复制时有很多同名路径,cp命令有概率失败,10次复制尝试
|
|
81
100
|
self._copy_files()
|
|
82
101
|
|
|
83
102
|
# 执行权限控制逻辑
|
|
84
|
-
per_cfg = os.path.join(self.
|
|
103
|
+
per_cfg = os.path.join(self.package_folder, "permissions.ini")
|
|
85
104
|
if not os.path.isfile(per_cfg):
|
|
86
|
-
self.work.warning("权限文件 {} 不存在, 所以无法设置组件 {} 的权限".format(per_cfg, self.
|
|
105
|
+
self.work.warning("权限文件 {} 不存在, 所以无法设置组件 {} 的权限".format(per_cfg, self.package_folder))
|
|
87
106
|
return
|
|
88
107
|
self.work.info("依据配置文件: {}, 修改权限".format(per_cfg))
|
|
89
108
|
# NOTE 这里的配置时间, 偶发复制未完成, 就开始修改权限
|
|
@@ -130,7 +149,7 @@ class CopyComponent(Process):
|
|
|
130
149
|
|
|
131
150
|
def _copy_files(self):
|
|
132
151
|
# 由于复制时有很多同名路径,cp命令有概率失败,10次复制尝试
|
|
133
|
-
cmd = "sudo cp -dfr {}/. {}".format(self.
|
|
152
|
+
cmd = "sudo cp -dfr {}/. {}".format(self.package_folder, self.config.rootfs_path)
|
|
134
153
|
for _ in range(0, 10):
|
|
135
154
|
try:
|
|
136
155
|
# 尝试复制到目标目录,成功则break退出循环
|
|
@@ -165,22 +184,33 @@ class ConanLockParse:
|
|
|
165
184
|
self._process_list = []
|
|
166
185
|
|
|
167
186
|
def conan_install(self, cmd, com):
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
187
|
+
com_name = com[:com.index("/")]
|
|
188
|
+
com_install_log = os.path.join(self.work.config.temp_path, f"log/conan_install/{com_name}.log")
|
|
189
|
+
with os.fdopen(os.open(com_install_log, os.O_RDWR | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), "a+") as f:
|
|
190
|
+
self.work.info(f"命令 {cmd} " + Fore.GREEN + "开始" + Style.RESET_ALL)
|
|
191
|
+
real_cmd = shlex.split(cmd)
|
|
192
|
+
pipe = subprocess.Popen(real_cmd, stdout=f, stderr=f)
|
|
193
|
+
start_time = time.time()
|
|
194
|
+
while True:
|
|
195
|
+
if pipe.poll() is not None:
|
|
196
|
+
break
|
|
197
|
+
# 1800秒超时
|
|
198
|
+
if time.time() - start_time > 1800:
|
|
199
|
+
pipe.kill()
|
|
200
|
+
self.work.error(f"================== {com} 构建超时 ==================")
|
|
201
|
+
self._queue.put(self.error_str)
|
|
202
|
+
return
|
|
203
|
+
time.sleep(1)
|
|
204
|
+
if pipe.returncode != 0:
|
|
205
|
+
self.work.error(f"================== {com} 构建失败日志起始位置 ==================")
|
|
206
|
+
f.seek(0)
|
|
207
|
+
conan_log = f.read()
|
|
208
|
+
self.work.info(conan_log)
|
|
209
|
+
self.work.error(f"================== {com} 构建失败日志结束位置 ==================")
|
|
210
|
+
self._queue.put(self.error_str)
|
|
211
|
+
return
|
|
212
|
+
self._queue.put(com)
|
|
213
|
+
self.work.info(f"命令 {cmd} " + Fore.GREEN + "执行完成" + Style.RESET_ALL)
|
|
184
214
|
|
|
185
215
|
def wait_task_finished(self):
|
|
186
216
|
# 所有入度为0任务拉起后等待完成消息
|
|
@@ -210,6 +240,9 @@ class ConanLockParse:
|
|
|
210
240
|
self._degree = {x: len(self._bundle[x].get('requires')) if self._bundle[x].get('requires') is not None else 0
|
|
211
241
|
for x in self._bundle.keys()}
|
|
212
242
|
|
|
243
|
+
conan_log_path = os.path.join(self.work.config.temp_path, "log/conan_install")
|
|
244
|
+
os.makedirs(conan_log_path, exist_ok=True)
|
|
245
|
+
|
|
213
246
|
self.work.success(">>>>>>>>>>>>>>>>> 开始构建所有组件 <<<<<<<<<<<<<<<<")
|
|
214
247
|
self.work.info("这些进程将要花费几分钟(2-3分钟), 请耐心等待")
|
|
215
248
|
while self._degree:
|
|
@@ -226,7 +259,6 @@ class ConanLockParse:
|
|
|
226
259
|
t = Process(target=self.conan_install, args=(cmd_tmp, com))
|
|
227
260
|
t.start()
|
|
228
261
|
self._process_list.append(t)
|
|
229
|
-
self.work.info(f"命令 {cmd_tmp} " + Fore.GREEN + "开始" + Style.RESET_ALL)
|
|
230
262
|
# 入度为0的任务构建拉起后弹出
|
|
231
263
|
self._degree.pop(com)
|
|
232
264
|
else:
|
|
@@ -243,7 +275,11 @@ class TaskClass(Task):
|
|
|
243
275
|
|
|
244
276
|
def __init__(self, config: Config, work_name=""):
|
|
245
277
|
super(TaskClass, self).__init__(config, work_name)
|
|
278
|
+
self.skip_install_comp = False
|
|
279
|
+
self.only_manifest_yml = False
|
|
246
280
|
self.update_path()
|
|
281
|
+
# 记录每个组件的GraphNode信息
|
|
282
|
+
self.package_info = os.path.join(self.config.rootfs_path, "package_info")
|
|
247
283
|
|
|
248
284
|
@property
|
|
249
285
|
def subsys_dir(self):
|
|
@@ -261,6 +297,13 @@ class TaskClass(Task):
|
|
|
261
297
|
return PERSIST_TYPES[item]
|
|
262
298
|
return None
|
|
263
299
|
|
|
300
|
+
@staticmethod
|
|
301
|
+
def match_persist_type(usage):
|
|
302
|
+
for item in usage:
|
|
303
|
+
if item in PERSIST_TYPES:
|
|
304
|
+
return PERSIST_TYPES[item]
|
|
305
|
+
return None
|
|
306
|
+
|
|
264
307
|
def find_conan_package_and_write(self, search, comps, file_handler, stage):
|
|
265
308
|
if search.find("/") < 0:
|
|
266
309
|
search += "/"
|
|
@@ -273,18 +316,19 @@ class TaskClass(Task):
|
|
|
273
316
|
if stage != misc.StageEnum.STAGE_DEV.value and comp.endswith(f"/{misc.StageEnum.STAGE_DEV.value}"):
|
|
274
317
|
err_msg = f"组件包 {comp} user/channel 配置错误, 必须以 \"{stage}\" 作为结尾, 终止构建"
|
|
275
318
|
raise errors.BmcGoException(err_msg)
|
|
319
|
+
if misc.conan_v2():
|
|
320
|
+
comp = comp.lower()
|
|
276
321
|
self.add_new_dependencies(comp, file_handler)
|
|
277
322
|
return
|
|
278
323
|
raise errors.ConfigException(f"未知组件: {search}, 请检查配置 !!!")
|
|
279
324
|
|
|
280
325
|
def update_path(self):
|
|
281
326
|
# conan source folder
|
|
282
|
-
self.conan_source = os.path.join(self.config.
|
|
283
|
-
f"conan_source_{self.config.build_type}_{self.config.stage}/openubmc")
|
|
327
|
+
self.conan_source = os.path.join(self.config.build_path, misc.community_name())
|
|
284
328
|
# conan install folder
|
|
285
329
|
self.conan_install = os.path.join(self.config.build_path, "conan_install")
|
|
286
330
|
# openubmc install folder
|
|
287
|
-
self.openubmc_ins_dir = os.path.join(self.conan_install,
|
|
331
|
+
self.openubmc_ins_dir = os.path.join(self.conan_install, misc.community_name())
|
|
288
332
|
# openubmc top rootfs folder
|
|
289
333
|
self.top_rootfs_dir = os.path.join(self.conan_install, "rootfs")
|
|
290
334
|
self.board_option = ""
|
|
@@ -320,6 +364,18 @@ class TaskClass(Task):
|
|
|
320
364
|
else:
|
|
321
365
|
self.skip_package = False
|
|
322
366
|
|
|
367
|
+
def set_skip_install_comp(self, value):
|
|
368
|
+
if value:
|
|
369
|
+
self.skip_install_comp = True
|
|
370
|
+
else:
|
|
371
|
+
self.skip_install_comp = False
|
|
372
|
+
|
|
373
|
+
def set_only_manifest_yml(self, value):
|
|
374
|
+
if value:
|
|
375
|
+
self.only_manifest_yml = True
|
|
376
|
+
else:
|
|
377
|
+
self.only_manifest_yml = False
|
|
378
|
+
|
|
323
379
|
def package_dependencies_parse(self, default_component: list):
|
|
324
380
|
"""解析manufacture或者tosupporte配置,不同的包设置不同组件的不同编译选项
|
|
325
381
|
参数:
|
|
@@ -346,6 +402,58 @@ class TaskClass(Task):
|
|
|
346
402
|
self.success(f"获取到依赖: {conan}")
|
|
347
403
|
self.depdencies.append(conan)
|
|
348
404
|
|
|
405
|
+
def merge_manifest_v2(self):
|
|
406
|
+
comps = []
|
|
407
|
+
for f in os.listdir(self.subsys_dir):
|
|
408
|
+
with open(os.path.join(self.subsys_dir, f)) as fp:
|
|
409
|
+
yml = yaml.safe_load(fp)
|
|
410
|
+
deps = yml.get('dependencies')
|
|
411
|
+
for dep in deps:
|
|
412
|
+
conan = dep.get(misc.CONAN)
|
|
413
|
+
if conan:
|
|
414
|
+
comps.append(conan)
|
|
415
|
+
self.debug("依赖列表: {}".format(comps))
|
|
416
|
+
# 重建新的依赖关系
|
|
417
|
+
# 从单板目录manifest.yml,与subsys/<stage>目录下的组件合并
|
|
418
|
+
new_fd = os.fdopen(os.open("manifest.yml", os.O_WRONLY | os.O_CREAT,
|
|
419
|
+
stat.S_IWUSR | stat.S_IRUSR), 'w')
|
|
420
|
+
new_fd.write("base:\n")
|
|
421
|
+
ver = self.get_manufacture_config("base/version")
|
|
422
|
+
|
|
423
|
+
new_fd.write(f" version: \"{ver}\"\n")
|
|
424
|
+
self.info(f"包版本号: {ver}")
|
|
425
|
+
|
|
426
|
+
new_fd.write("dependencies:\n")
|
|
427
|
+
deps = self._get_dependencies_pkg()
|
|
428
|
+
|
|
429
|
+
# 如果是打包,依赖信息调整
|
|
430
|
+
deps = self.package_dependencies_parse(deps)
|
|
431
|
+
# 由于manifest.yml当中有对于此的新的配置,此处将配置读出,并重新分配
|
|
432
|
+
skynet_with_enable_luajit = False
|
|
433
|
+
for dep in deps:
|
|
434
|
+
conan = dep.get(misc.CONAN)
|
|
435
|
+
if not conan:
|
|
436
|
+
continue
|
|
437
|
+
if conan.find("@") > 0:
|
|
438
|
+
self.add_new_dependencies(conan.lower(), new_fd)
|
|
439
|
+
else:
|
|
440
|
+
self.find_conan_package_and_write(conan, comps, new_fd, self.config.stage)
|
|
441
|
+
options = dep.get("options", {})
|
|
442
|
+
name = conan.split("/")[0]
|
|
443
|
+
for key, val in options.items():
|
|
444
|
+
# skynet特殊处理:manifest.yml指定enable_luajit特性时需要覆盖用户输入
|
|
445
|
+
if name == "skynet" and key == "enable_luajit":
|
|
446
|
+
self.warning(f"根据manifest.yml配置,当前产品的enable_luajit配置为{val},忽略命令行指定的-jit参数")
|
|
447
|
+
skynet_with_enable_luajit = True
|
|
448
|
+
self.config.set_enable_luajit(val)
|
|
449
|
+
self.board_option = self.board_option + " -o */*:{}={}".format(key, val)
|
|
450
|
+
else:
|
|
451
|
+
self.board_option = self.board_option + " -o {}/*:{}={}".format(name, key, val)
|
|
452
|
+
# 当使能Luajit又未向skynet传递enable_luajit配置项时需要添加使能参数
|
|
453
|
+
if not skynet_with_enable_luajit and self.config.enable_luajit:
|
|
454
|
+
self.board_option += f" -o */*:enable_luajit={self.config.enable_luajit}"
|
|
455
|
+
new_fd.close()
|
|
456
|
+
|
|
349
457
|
def merge_manifest(self):
|
|
350
458
|
comps = []
|
|
351
459
|
for f in os.listdir(self.subsys_dir):
|
|
@@ -364,7 +472,7 @@ class TaskClass(Task):
|
|
|
364
472
|
new_fd.write("base:\n")
|
|
365
473
|
ver = self.get_manufacture_config("base/version")
|
|
366
474
|
|
|
367
|
-
new_fd.write(f" version: \"{ver}@{
|
|
475
|
+
new_fd.write(f" version: \"{ver}@{misc.conan_user()}/{misc.StageEnum.STAGE_RC.value}\"\n")
|
|
368
476
|
self.info(f"包版本号: {ver}")
|
|
369
477
|
|
|
370
478
|
new_fd.write("dependencies:\n")
|
|
@@ -384,7 +492,7 @@ class TaskClass(Task):
|
|
|
384
492
|
stage = self.config.stage
|
|
385
493
|
if stage != misc.StageEnum.STAGE_STABLE.value:
|
|
386
494
|
stage = misc.StageEnum.STAGE_RC.value
|
|
387
|
-
conan += f"@{
|
|
495
|
+
conan += f"@{misc.conan_user()}/{stage}"
|
|
388
496
|
self.add_new_dependencies(conan, new_fd)
|
|
389
497
|
else:
|
|
390
498
|
self.find_conan_package_and_write(conan, comps, new_fd, self.config.stage)
|
|
@@ -400,17 +508,26 @@ class TaskClass(Task):
|
|
|
400
508
|
# 当使能Luajit又未向skynet传递enable_luajit配置项时需要添加使能参数
|
|
401
509
|
if not skynet_with_enable_luajit and self.config.enable_luajit:
|
|
402
510
|
self.board_option += f" -o skynet:enable_luajit={self.config.enable_luajit}"
|
|
403
|
-
self.merge_0502_default_options()
|
|
404
511
|
sha256 = Tools.sha256sum(SDK_ROOT_MODULE_SYMVERS)
|
|
405
512
|
self.board_option += " -o *:module_symvers={}".format(sha256)
|
|
406
513
|
new_fd.close()
|
|
407
514
|
|
|
515
|
+
def merge_dep_options(self):
|
|
516
|
+
self.merge_0502_default_options()
|
|
517
|
+
|
|
408
518
|
def merge_0502_default_options(self):
|
|
519
|
+
if misc.conan_v1():
|
|
520
|
+
prefix = "*"
|
|
521
|
+
else:
|
|
522
|
+
prefix = "*/*"
|
|
409
523
|
if self.config.manufacture_code:
|
|
410
524
|
default_options_path = f"manufacture/{self.config.manufacture_code}/default_options"
|
|
411
525
|
default_options = self.get_manufacture_config(default_options_path, {})
|
|
412
526
|
for key, val in default_options.items():
|
|
413
|
-
self.board_option += f" -o
|
|
527
|
+
self.board_option += f" -o {prefix}:{key}={val}"
|
|
528
|
+
module_symvers_path = os.path.join(SDK_PATH, MODULE_SYMVERS)
|
|
529
|
+
option, value = self.tools.get_hi171x_module_symver_option(module_symvers_path)
|
|
530
|
+
self.board_option += f" -o {prefix}:{option}={value}"
|
|
414
531
|
|
|
415
532
|
def package_info_gen(self, bundle_file, dst_file):
|
|
416
533
|
with open(bundle_file, "r") as fp:
|
|
@@ -419,8 +536,8 @@ class TaskClass(Task):
|
|
|
419
536
|
list.sort(require_list)
|
|
420
537
|
package_info_list = deepcopy(require_list)
|
|
421
538
|
self.run_command(f"rm -rf {dst_file}", sudo=True)
|
|
422
|
-
|
|
423
|
-
with os.fdopen(os.open(
|
|
539
|
+
self.package_info = f"{self.config.rootfs_path}/{os.path.basename(dst_file)}"
|
|
540
|
+
with os.fdopen(os.open(self.package_info, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, stat.S_IWUSR | stat.S_IRUSR |
|
|
424
541
|
stat.S_IWGRP | stat.S_IRGRP | stat.S_IWOTH | stat.S_IROTH), 'w') as fp:
|
|
425
542
|
for package in require_list:
|
|
426
543
|
if "openubmc/" not in package:
|
|
@@ -428,8 +545,7 @@ class TaskClass(Task):
|
|
|
428
545
|
else:
|
|
429
546
|
package_info_list.remove(package)
|
|
430
547
|
fp.close()
|
|
431
|
-
self.run_command(f"cp -df {
|
|
432
|
-
self.run_command(f"rm -rf {temp_dst_file}", sudo=True)
|
|
548
|
+
self.run_command(f"cp -df {self.package_info} {dst_file}", sudo=True)
|
|
433
549
|
|
|
434
550
|
def sensitive_data_conf_gen(self, comps, dst_file):
|
|
435
551
|
self.run_command(f"rm -rf {dst_file}", sudo=True)
|
|
@@ -500,6 +616,79 @@ class TaskClass(Task):
|
|
|
500
616
|
options += " -o *:gcov=True"
|
|
501
617
|
return options
|
|
502
618
|
|
|
619
|
+
def proc_sensitive_v2(self, table_type, props, table_name, output):
|
|
620
|
+
for prop_name, prop_data in props.items():
|
|
621
|
+
if prop_data.get("primaryKey"):
|
|
622
|
+
output[PRIMARY_KEYS][table_name] = output[PRIMARY_KEYS].get(table_name, {})
|
|
623
|
+
output[PRIMARY_KEYS][table_name][prop_name] = prop_data.get("sensitive", False)
|
|
624
|
+
continue
|
|
625
|
+
per_type = self.match_persist_type(prop_data.get("usage", [])) or table_type
|
|
626
|
+
if not per_type:
|
|
627
|
+
continue
|
|
628
|
+
output[per_type][table_name] = output[per_type].get(table_name, {})
|
|
629
|
+
output[per_type][table_name][prop_name] = prop_data.get("sensitive", False)
|
|
630
|
+
return output
|
|
631
|
+
|
|
632
|
+
def proc_comp_v2(self, package_folder, output):
|
|
633
|
+
model_file = os.path.join(package_folder, "include", "mds", "model.json")
|
|
634
|
+
if not os.path.isfile(model_file):
|
|
635
|
+
return output
|
|
636
|
+
with open(model_file, "r") as fp:
|
|
637
|
+
content = json.load(fp)
|
|
638
|
+
for class_data in content.values():
|
|
639
|
+
table_type = PERSIST_TYPES.get(class_data.get("tableType", ""))
|
|
640
|
+
table_name = class_data.get("tableName", "")
|
|
641
|
+
if not table_name or class_data.get("tableLocation") == "Local":
|
|
642
|
+
continue
|
|
643
|
+
class_props = [class_data.get("properties", {})]
|
|
644
|
+
for intf_data in class_data.get("interfaces", {}).values():
|
|
645
|
+
class_props.append(intf_data.get("properties", {}))
|
|
646
|
+
for props in class_props:
|
|
647
|
+
output = self.proc_sensitive_v2(table_type, props, table_name, output)
|
|
648
|
+
return output
|
|
649
|
+
|
|
650
|
+
def calc_options_v2(self):
|
|
651
|
+
if self.config.build_type == "debug":
|
|
652
|
+
options = "-s build_type=Debug"
|
|
653
|
+
else:
|
|
654
|
+
options = "-s build_type=Release"
|
|
655
|
+
|
|
656
|
+
if self.config.enable_arm_gcov:
|
|
657
|
+
options += " -o */*:gcov=True"
|
|
658
|
+
return options
|
|
659
|
+
|
|
660
|
+
def install_openubmc_v2(self):
|
|
661
|
+
profile_file = os.path.join(self.tools.conan_profiles_dir, self.config.profile)
|
|
662
|
+
if not os.path.isfile(profile_file):
|
|
663
|
+
raise BmcGoException(f"{profile_file} 文件不存在")
|
|
664
|
+
|
|
665
|
+
options = self.calc_options_v2()
|
|
666
|
+
append_cmd = f"-r {self.config.remote}" if self.config.remote else ""
|
|
667
|
+
# 依据选项生成openubmc.lock和openubmc.bundle文件
|
|
668
|
+
base_cmd = f"-pr={self.config.profile} "
|
|
669
|
+
base_cmd += f"-pr:b profile.dt.ini {append_cmd} {options} {self.board_option}"
|
|
670
|
+
lockfile = os.path.join(self.config.build_path, "conan.lock")
|
|
671
|
+
graphfile = os.path.join(self.config.build_path, "graph.info")
|
|
672
|
+
lock_cmd = f"conan lock create . {base_cmd} --lockfile-out={lockfile}"
|
|
673
|
+
self.run_command(lock_cmd, capture_output=False)
|
|
674
|
+
graph_cmd = f"conan graph info . {base_cmd} -f json --lockfile={lockfile} --out-file={graphfile}"
|
|
675
|
+
self.run_command(graph_cmd)
|
|
676
|
+
self.success(f"start build dependency packages of {self.config.board_name}")
|
|
677
|
+
bcp = BuildConans(graphfile, lockfile, base_cmd, self.config.from_source, self.config.log_path)
|
|
678
|
+
bcp.build()
|
|
679
|
+
|
|
680
|
+
cmd = f"conan create . {base_cmd} --build=missing"
|
|
681
|
+
self.info(f"start build {self.config.board_name}: {cmd}")
|
|
682
|
+
self.run_command(cmd, capture_output=False)
|
|
683
|
+
|
|
684
|
+
self.graphfile = graphfile
|
|
685
|
+
|
|
686
|
+
shutil.copyfile(lockfile, f"{self.conan_install}/conan.lock")
|
|
687
|
+
# 检查使用到的组件是否都在单板目录 manifest.yml 中配置了
|
|
688
|
+
component_check = ComponentVersionCheck("manifest.yml", lockfile, "openubmc")
|
|
689
|
+
component_check.run()
|
|
690
|
+
self.clean_luac_out()
|
|
691
|
+
|
|
503
692
|
def install_ibmc(self):
|
|
504
693
|
profile_file = os.path.join(self.tools.conan_profiles_dir, self.config.profile)
|
|
505
694
|
if not os.path.isfile(profile_file):
|
|
@@ -539,21 +728,30 @@ class TaskClass(Task):
|
|
|
539
728
|
self.run_command(install_cmd, command_echo=True)
|
|
540
729
|
shutil.copyfile(IBMC_LOCK_FILE, f"{self.conan_install}/conan.lock")
|
|
541
730
|
# 检查使用到的组件是否都在单板目录 manifest.yml 中配置了
|
|
542
|
-
component_check = ComponentVersionCheck(
|
|
731
|
+
component_check = ComponentVersionCheck("manifest.yml", IBMC_LOCK_FILE, "openubmc")
|
|
543
732
|
component_check.run()
|
|
544
733
|
self.clean_luac_out()
|
|
545
734
|
|
|
546
735
|
def deploy(self):
|
|
547
|
-
|
|
548
|
-
shutil.copytree(src, self.conan_source, dirs_exist_ok=True)
|
|
736
|
+
conanfile_dir = os.path.join(cwd_dir, "conan")
|
|
549
737
|
openubmc_dir = os.path.join(self.conan_source, "all")
|
|
738
|
+
os.makedirs(openubmc_dir, exist_ok=True)
|
|
739
|
+
shutil.copytree(conanfile_dir, openubmc_dir, dirs_exist_ok=True)
|
|
550
740
|
self.chdir(openubmc_dir)
|
|
551
741
|
# 复制manifest.yml文件
|
|
552
|
-
|
|
742
|
+
if misc.conan_v2():
|
|
743
|
+
self.merge_manifest_v2()
|
|
744
|
+
else:
|
|
745
|
+
self.merge_manifest()
|
|
746
|
+
if self.only_manifest_yml:
|
|
747
|
+
return
|
|
748
|
+
if not self.skip_install_comp:
|
|
749
|
+
self.merge_dep_options()
|
|
553
750
|
# 下载组件构建脚本
|
|
554
751
|
ComponentHelper.download_recipes(self.depdencies, self.tools, self.config.remote_list)
|
|
555
752
|
# 下载skynet
|
|
556
|
-
|
|
753
|
+
if misc.conan_v1():
|
|
754
|
+
self.install_luac_or_luajit()
|
|
557
755
|
|
|
558
756
|
# 复制全局定制rootfs到conan install目录
|
|
559
757
|
top_rootfs = os.path.join(self.config.code_path, "rootfs")
|
|
@@ -578,7 +776,10 @@ class TaskClass(Task):
|
|
|
578
776
|
if os.path.isfile(per_file):
|
|
579
777
|
shutil.copy(per_file, self.openubmc_ins_dir)
|
|
580
778
|
|
|
581
|
-
|
|
779
|
+
if misc.conan_v2():
|
|
780
|
+
self.install_openubmc_v2()
|
|
781
|
+
else:
|
|
782
|
+
self.install_ibmc()
|
|
582
783
|
|
|
583
784
|
def clean_luac_out(self):
|
|
584
785
|
# 清理冗余文件luac.out
|
|
@@ -600,15 +801,15 @@ class TaskClass(Task):
|
|
|
600
801
|
self.run_command(f'cp -f conan.lock {os.path.join(inner_path, f"package_{self.config.board_name}.lock")}')
|
|
601
802
|
self.run_command(f'cp -f conan.lock {os.path.join(self.config.output_path, f"package.lock")}')
|
|
602
803
|
|
|
603
|
-
def copy_components(self, comps: list, profile
|
|
804
|
+
def copy_components(self, comps: list, profile):
|
|
604
805
|
# 优先处理rootfs
|
|
605
|
-
p = CopyComponent(self, "rootfs", profile)
|
|
806
|
+
p = CopyComponent(self, "rootfs", os.path.join(self.conan_install, "rootfs"), profile)
|
|
606
807
|
p.run()
|
|
607
808
|
# 打印组件清单
|
|
608
809
|
self.info(f"组件列表: {comps}")
|
|
609
810
|
pools = []
|
|
610
811
|
for comp in comps:
|
|
611
|
-
p = CopyComponent(self, comp, profile)
|
|
812
|
+
p = CopyComponent(self, comp, os.path.join(self.conan_install, comp), profile)
|
|
612
813
|
p.start()
|
|
613
814
|
pools.append(p)
|
|
614
815
|
|
|
@@ -621,11 +822,161 @@ class TaskClass(Task):
|
|
|
621
822
|
raise errors.BmcGoException(f"复制组件 ({p.comp}) 失败, 退出码: {p.exitcode}")
|
|
622
823
|
pools.remove(p)
|
|
623
824
|
# 最后处理openubmc
|
|
624
|
-
p = CopyComponent(self,
|
|
825
|
+
p = CopyComponent(self, misc.community_name(), os.path.join(self.conan_install, misc.community_name()), profile)
|
|
826
|
+
p.run()
|
|
827
|
+
|
|
828
|
+
def package_info_gen_v2(self, bmc_lock, dst_file):
|
|
829
|
+
with open(bmc_lock, "r") as bmc_lock_fp:
|
|
830
|
+
lock_info = json.load(bmc_lock_fp)
|
|
831
|
+
require_list = lock_info.get("requires")
|
|
832
|
+
self.run_command(f"rm -rf {dst_file}", sudo=True)
|
|
833
|
+
self.package_info = f"{self.config.rootfs_path}/{os.path.basename(dst_file)}"
|
|
834
|
+
with os.fdopen(os.open(self.package_info, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, stat.S_IWUSR | stat.S_IRUSR |
|
|
835
|
+
stat.S_IWGRP | stat.S_IRGRP | stat.S_IWOTH | stat.S_IROTH), 'w') as fp:
|
|
836
|
+
for package in require_list:
|
|
837
|
+
fp.write(f"{package}\n")
|
|
838
|
+
fp.close()
|
|
839
|
+
self.run_command(f"cp -df {self.package_info} {dst_file}", sudo=True)
|
|
840
|
+
|
|
841
|
+
def sensitive_data_conf_gen_v2(self, dst_file):
|
|
842
|
+
self.run_command(f"rm -rf {dst_file}", sudo=True)
|
|
843
|
+
temp_dst_file = f"{self.config.rootfs_path}/{os.path.basename(dst_file)}"
|
|
844
|
+
output = {
|
|
845
|
+
"TemporaryPer": {},
|
|
846
|
+
"ResetPer": {},
|
|
847
|
+
"PoweroffPer": {},
|
|
848
|
+
"PermanentPer": {},
|
|
849
|
+
"PrimaryKeys": {}
|
|
850
|
+
}
|
|
851
|
+
for name, node in self.graph_nodes.items():
|
|
852
|
+
try:
|
|
853
|
+
output = self.proc_comp_v2(node.package_folder, output)
|
|
854
|
+
except Exception as e:
|
|
855
|
+
raise errors.BmcGoException(f"分析组件 {name} 的 model.json 失败, 失败信息:{e}")
|
|
856
|
+
|
|
857
|
+
with os.fdopen(os.open(temp_dst_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC,
|
|
858
|
+
stat.S_IWUSR | stat.S_IRUSR), 'w') as fp:
|
|
859
|
+
json.dump(output, fp, sort_keys=True)
|
|
860
|
+
self.run_command(f"mkdir -p {os.path.dirname(dst_file)}", sudo=True)
|
|
861
|
+
self.run_command(f"cp -df {temp_dst_file} {dst_file}", sudo=True)
|
|
862
|
+
self.run_command(f"rm -rf {temp_dst_file}", sudo=True)
|
|
863
|
+
|
|
864
|
+
def get_all_subdirectories(self, dirpath):
|
|
865
|
+
exclude_files = ["permissions.ini", "conaninfo.txt", "conanmanifest.txt"]
|
|
866
|
+
out = []
|
|
867
|
+
for root, _, files in os.walk(dirpath):
|
|
868
|
+
for file in files:
|
|
869
|
+
file = os.path.join(root, file)
|
|
870
|
+
relative_path = os.path.relpath(file, dirpath)
|
|
871
|
+
if not relative_path.startswith("include") and relative_path not in exclude_files:
|
|
872
|
+
out.append(relative_path)
|
|
873
|
+
return out
|
|
874
|
+
|
|
875
|
+
def print_dup_files(self, comps, paths):
|
|
876
|
+
dup_paths = {}
|
|
877
|
+
for comp, files in comps.items():
|
|
878
|
+
for path in paths:
|
|
879
|
+
if path in files:
|
|
880
|
+
dup_paths.setdefault(path, []).append(comp)
|
|
881
|
+
|
|
882
|
+
for path, comp in dup_paths.items():
|
|
883
|
+
self.error(f"{comp} 存在重复文件 {path}")
|
|
884
|
+
|
|
885
|
+
def copy_components_v2(self, profile):
|
|
886
|
+
# 优先处理rootfs
|
|
887
|
+
p = CopyComponent(self, "rootfs", os.path.join(self.conan_install, "rootfs"), profile)
|
|
888
|
+
p.run()
|
|
889
|
+
# 打印组件清单
|
|
890
|
+
self.info(f"组件列表: {self.graph_nodes.keys()}")
|
|
891
|
+
pools = []
|
|
892
|
+
# 记录全部的文件信息
|
|
893
|
+
all_path = []
|
|
894
|
+
# 记录组件有哪些文件
|
|
895
|
+
comp_path = {}
|
|
896
|
+
for name, node in self.graph_nodes.items():
|
|
897
|
+
if name in ["rootfs", misc.community_name()]:
|
|
898
|
+
continue
|
|
899
|
+
file_paths = self.get_all_subdirectories(node.package_folder)
|
|
900
|
+
all_path.extend(file_paths)
|
|
901
|
+
comp_path[name] = file_paths
|
|
902
|
+
p = CopyComponent(self, node.name, node.package_folder, profile)
|
|
903
|
+
p.start()
|
|
904
|
+
pools.append(p)
|
|
905
|
+
|
|
906
|
+
while pools:
|
|
907
|
+
time.sleep(0.01)
|
|
908
|
+
for p in pools:
|
|
909
|
+
if p.is_alive():
|
|
910
|
+
continue
|
|
911
|
+
if p.exitcode is not None and p.exitcode != 0:
|
|
912
|
+
raise errors.BmcGoException(f"复制组件 ({p.name}) 失败, 退出码: {p.exitcode}")
|
|
913
|
+
pools.remove(p)
|
|
914
|
+
counter = Counter(all_path)
|
|
915
|
+
duplicates = {path for path, count in counter.items() if count > 1}
|
|
916
|
+
if len(duplicates):
|
|
917
|
+
self.print_dup_files(comp_path, duplicates)
|
|
918
|
+
raise errors.BmcGoException("请检查组件的打包逻辑,检测到重复文件: " + ".".join(duplicates))
|
|
919
|
+
# 最后处理openubmc
|
|
920
|
+
p = CopyComponent(self, misc.community_name(), os.path.join(self.conan_install, misc.community_name()), profile)
|
|
625
921
|
p.run()
|
|
922
|
+
self.run_command(f"sudo rm -f {self.config.rootfs_path}/conaninfo.txt")
|
|
923
|
+
self.run_command(f"sudo rm -f {self.config.rootfs_path}/conanmanifest.txt")
|
|
924
|
+
|
|
925
|
+
def update_graph_nodes(self):
|
|
926
|
+
rootfs = GraphNode(None)
|
|
927
|
+
rootfs.name = "rootfs"
|
|
928
|
+
rootfs.package_folder = os.path.join(self.conan_install, "rootfs")
|
|
929
|
+
self.graph_nodes["rootfs"] = rootfs
|
|
930
|
+
|
|
931
|
+
with open(self.graphfile, "r") as fp:
|
|
932
|
+
graph = json.load(fp)
|
|
933
|
+
nodes = graph.get("graph", {}).get("nodes", {})
|
|
934
|
+
for _, info in nodes.items():
|
|
935
|
+
node = GraphNode(info)
|
|
936
|
+
context = info.get("context")
|
|
937
|
+
if context != "host":
|
|
938
|
+
continue
|
|
939
|
+
cmd = f"conan cache path {node.ref}:{node.package_id}"
|
|
940
|
+
node.package_folder = tools.run_command(cmd, capture_output=True).stdout.strip()
|
|
941
|
+
self.graph_nodes[node.name] = node
|
|
942
|
+
|
|
943
|
+
rootfs = GraphNode(None)
|
|
944
|
+
rootfs.name = misc.community_name()
|
|
945
|
+
rootfs.package_folder = os.path.join(self.conan_install, misc.community_name())
|
|
946
|
+
self.graph_nodes[misc.community_name()] = rootfs
|
|
947
|
+
|
|
948
|
+
def make_customization_symlinks(self):
|
|
949
|
+
if os.path.isdir(self.customization_dir):
|
|
950
|
+
self.run_command(f"rm -rf {self.customization_dir}")
|
|
951
|
+
self.run_command(f"mkdir -p {self.customization_dir}")
|
|
952
|
+
for name, node in self.graph_nodes.items():
|
|
953
|
+
cust = os.path.join(node.package_folder, "include", "customization.py")
|
|
954
|
+
if not os.path.isfile(cust):
|
|
955
|
+
continue
|
|
956
|
+
os.symlink(node.package_folder, os.path.join(self.customization_dir, name))
|
|
957
|
+
|
|
958
|
+
def package_v2(self):
|
|
959
|
+
self.update_graph_nodes()
|
|
960
|
+
self.make_customization_symlinks()
|
|
961
|
+
self.run_command(f"sudo rm -rf {self.config.rootfs_path}")
|
|
962
|
+
os.makedirs(self.config.rootfs_path)
|
|
963
|
+
profile, _ = self.get_profile_config()
|
|
964
|
+
self.copy_components_v2(profile)
|
|
965
|
+
|
|
966
|
+
self.chdir(self.config.rootfs_path)
|
|
967
|
+
self._component_cust_action("post_image")
|
|
968
|
+
|
|
969
|
+
self.chdir(self.config.rootfs_path)
|
|
970
|
+
self.package_info_gen_v2(f"{self.conan_install}/conan.lock", "etc/package_info")
|
|
971
|
+
self.sensitive_data_conf_gen_v2("opt/bmc/trust/sensitive_data.json")
|
|
972
|
+
if os.path.isfile("permissions.ini"):
|
|
973
|
+
os.unlink("permissions.ini")
|
|
626
974
|
|
|
627
975
|
def package(self):
|
|
628
976
|
self.chdir(self.conan_install)
|
|
977
|
+
if misc.conan_v2():
|
|
978
|
+
self.package_v2()
|
|
979
|
+
return
|
|
629
980
|
self.run_command(f"sudo rm -rf {self.config.rootfs_path}")
|
|
630
981
|
os.makedirs(self.config.rootfs_path)
|
|
631
982
|
profile, _ = self.get_profile_config()
|
|
@@ -638,14 +989,8 @@ class TaskClass(Task):
|
|
|
638
989
|
|
|
639
990
|
self.copy_components(comps, profile)
|
|
640
991
|
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
cust = os.path.join(self.conan_install, comp, "include", "customization.py")
|
|
644
|
-
if not os.path.isfile(cust):
|
|
645
|
-
continue
|
|
646
|
-
self.info(f"开始执行 {comp}/include/customization.py")
|
|
647
|
-
post = ComponentPost(self.config, os.path.join(self.conan_install, comp), profile)
|
|
648
|
-
post.post_work(self.config.rootfs_path, "post_image")
|
|
992
|
+
self.chdir(self.config.rootfs_path)
|
|
993
|
+
self._component_cust_action("post_image")
|
|
649
994
|
|
|
650
995
|
self.chdir(self.config.rootfs_path)
|
|
651
996
|
self.package_info_gen(f"{self.conan_source}/all/openubmc.bundle", "etc/package_info")
|
|
@@ -721,6 +1066,8 @@ class TaskClass(Task):
|
|
|
721
1066
|
self.mkdir_work_path()
|
|
722
1067
|
self.tools.clean_locks()
|
|
723
1068
|
self.deploy()
|
|
1069
|
+
if self.only_manifest_yml:
|
|
1070
|
+
return
|
|
724
1071
|
self.package_lock()
|
|
725
1072
|
if not self.skip_package:
|
|
726
1073
|
self.package()
|