openubmc-bingo 0.5.277__py3-none-any.whl → 0.6.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of openubmc-bingo might be problematic. Click here for more details.
- bmcgo/__init__.py +1 -1
- bmcgo/bmcgo_config.py +22 -10
- bmcgo/cli/cli.py +95 -39
- bmcgo/cli/config.conan2.yaml +9 -0
- bmcgo/codegen/lua/codegen.py +2 -2
- bmcgo/codegen/lua/script/gen_intf_rpc_json.py +15 -14
- bmcgo/component/analysis/analysis.py +8 -8
- bmcgo/component/analysis/intf_validation.py +23 -12
- bmcgo/component/build.py +76 -14
- bmcgo/component/component_dt_version_parse.py +3 -2
- bmcgo/component/component_helper.py +43 -15
- bmcgo/component/coverage/incremental_cov.py +2 -2
- bmcgo/component/deploy.py +68 -14
- bmcgo/component/gen.py +1 -1
- bmcgo/component/package_info.py +128 -38
- bmcgo/component/template/conanbase.py.mako +1 -0
- bmcgo/component/template_v2/conanbase.py.mako +383 -0
- bmcgo/component/template_v2/conanfile.deploy.py.mako +26 -0
- bmcgo/component/test.py +53 -20
- bmcgo/frame.py +7 -3
- bmcgo/functional/analysis.py +3 -2
- bmcgo/functional/check.py +10 -6
- bmcgo/functional/conan_index_build.py +96 -20
- bmcgo/functional/csr_build.py +1 -1
- bmcgo/functional/diff.py +1 -1
- bmcgo/functional/fetch.py +1 -1
- bmcgo/functional/full_component.py +32 -24
- bmcgo/functional/git_history.py +220 -0
- bmcgo/functional/maintain.py +55 -12
- bmcgo/functional/new.py +1 -1
- bmcgo/functional/schema_valid.py +2 -2
- bmcgo/logger.py +2 -3
- bmcgo/misc.py +130 -9
- bmcgo/tasks/conan/__init__.py +10 -0
- bmcgo/tasks/conan/conanfile.py +45 -0
- bmcgo/tasks/task.py +27 -4
- bmcgo/tasks/task_build_conan.py +425 -53
- bmcgo/tasks/task_buildgppbin.py +8 -2
- bmcgo/tasks/task_download_buildtools.py +76 -0
- bmcgo/tasks/task_download_dependency.py +1 -0
- bmcgo/tasks/task_hpm_envir_prepare.py +1 -1
- bmcgo/utils/build_conans.py +231 -0
- bmcgo/utils/component_post.py +6 -4
- bmcgo/utils/component_version_check.py +75 -16
- bmcgo/utils/config.py +76 -40
- bmcgo/utils/fetch_component_code.py +44 -25
- bmcgo/utils/tools.py +239 -117
- bmcgo/worker.py +2 -2
- {openubmc_bingo-0.5.277.dist-info → openubmc_bingo-0.6.2.dist-info}/METADATA +4 -2
- {openubmc_bingo-0.5.277.dist-info → openubmc_bingo-0.6.2.dist-info}/RECORD +53 -46
- {openubmc_bingo-0.5.277.dist-info → openubmc_bingo-0.6.2.dist-info}/WHEEL +0 -0
- {openubmc_bingo-0.5.277.dist-info → openubmc_bingo-0.6.2.dist-info}/entry_points.txt +0 -0
- {openubmc_bingo-0.5.277.dist-info → openubmc_bingo-0.6.2.dist-info}/top_level.txt +0 -0
bmcgo/tasks/task_build_conan.py
CHANGED
|
@@ -27,10 +27,10 @@ import random
|
|
|
27
27
|
import pathlib
|
|
28
28
|
from multiprocessing import Process, Queue
|
|
29
29
|
from copy import deepcopy
|
|
30
|
+
from collections import Counter
|
|
30
31
|
|
|
31
32
|
import yaml
|
|
32
33
|
from colorama import Fore, Back, Style
|
|
33
|
-
from conans.model.profile import Profile
|
|
34
34
|
|
|
35
35
|
from bmcgo.tasks.task import Task
|
|
36
36
|
from bmcgo.utils.config import Config
|
|
@@ -40,6 +40,9 @@ from bmcgo import errors, misc
|
|
|
40
40
|
from bmcgo.utils.tools import Tools
|
|
41
41
|
from bmcgo.errors import BmcGoException
|
|
42
42
|
from bmcgo.component.component_helper import ComponentHelper
|
|
43
|
+
from bmcgo.utils.build_conans import BuildConans
|
|
44
|
+
from bmcgo.component.deploy import GraphNode
|
|
45
|
+
from bmcgo.tasks.misc import MODULE_SYMVERS, SDK_PATH
|
|
43
46
|
|
|
44
47
|
IBMC_LOCK_FILE = "openubmc.lock"
|
|
45
48
|
SDK_ROOT = "/opt/hi1711sdk"
|
|
@@ -56,14 +59,30 @@ PERSIST_TYPES = {
|
|
|
56
59
|
PRIMARY_KEYS = "PrimaryKeys"
|
|
57
60
|
tools = Tools()
|
|
58
61
|
|
|
62
|
+
PERSIST_TYPES = {
|
|
63
|
+
"TemporaryPer": "TemporaryPer",
|
|
64
|
+
"TemporaryPerRetain": "TemporaryPer",
|
|
65
|
+
"ResetPer": "ResetPer",
|
|
66
|
+
"ResetPerRetain": "ResetPer",
|
|
67
|
+
"PoweroffPer": "PoweroffPer",
|
|
68
|
+
"PoweroffPerRetain": "PoweroffPer",
|
|
69
|
+
"PermanentPer": "PermanentPer"
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
PRIMARY_KEYS = "PrimaryKeys"
|
|
73
|
+
|
|
74
|
+
cwd_dir = os.path.dirname(os.path.realpath(__file__))
|
|
75
|
+
|
|
59
76
|
|
|
60
77
|
class CopyComponent(Process):
|
|
61
|
-
def __init__(self, work: Task, comp: str,
|
|
78
|
+
def __init__(self, work: Task, comp: str, package_folder: str, profile):
|
|
62
79
|
super().__init__()
|
|
63
80
|
self.work = work
|
|
64
81
|
self.config = self.work.config
|
|
65
82
|
self.comp = comp
|
|
83
|
+
self.package_folder = package_folder
|
|
66
84
|
self.profile = profile
|
|
85
|
+
self.graphfile = None
|
|
67
86
|
|
|
68
87
|
def link_recursive_deal(self, file_name, ownership):
|
|
69
88
|
self.work.run_command(f"chown -h {ownership} {file_name}", sudo=True)
|
|
@@ -74,16 +93,16 @@ class CopyComponent(Process):
|
|
|
74
93
|
self.work.work_name = self.comp
|
|
75
94
|
# 复制组件文件到rootfs中
|
|
76
95
|
# rc.sysinit脚本只允许使用rootfs_user仓库的
|
|
77
|
-
rc_sysinit_path = os.path.join(self.
|
|
96
|
+
rc_sysinit_path = os.path.join(self.package_folder, "etc/rc.d/rc.sysinit")
|
|
78
97
|
if self.comp != "rootfs_user" and os.path.isfile(rc_sysinit_path):
|
|
79
98
|
shutil.rmtree(rc_sysinit_path, ignore_errors=True)
|
|
80
99
|
# 由于复制时有很多同名路径,cp命令有概率失败,10次复制尝试
|
|
81
100
|
self._copy_files()
|
|
82
101
|
|
|
83
102
|
# 执行权限控制逻辑
|
|
84
|
-
per_cfg = os.path.join(self.
|
|
103
|
+
per_cfg = os.path.join(self.package_folder, "permissions.ini")
|
|
85
104
|
if not os.path.isfile(per_cfg):
|
|
86
|
-
self.work.warning("权限文件 {} 不存在, 所以无法设置组件 {} 的权限".format(per_cfg, self.
|
|
105
|
+
self.work.warning("权限文件 {} 不存在, 所以无法设置组件 {} 的权限".format(per_cfg, self.package_folder))
|
|
87
106
|
return
|
|
88
107
|
self.work.info("依据配置文件: {}, 修改权限".format(per_cfg))
|
|
89
108
|
# NOTE 这里的配置时间, 偶发复制未完成, 就开始修改权限
|
|
@@ -130,7 +149,7 @@ class CopyComponent(Process):
|
|
|
130
149
|
|
|
131
150
|
def _copy_files(self):
|
|
132
151
|
# 由于复制时有很多同名路径,cp命令有概率失败,10次复制尝试
|
|
133
|
-
cmd = "sudo cp -dfr {}/. {}".format(self.
|
|
152
|
+
cmd = "sudo cp -dfr {}/. {}".format(self.package_folder, self.config.rootfs_path)
|
|
134
153
|
for _ in range(0, 10):
|
|
135
154
|
try:
|
|
136
155
|
# 尝试复制到目标目录,成功则break退出循环
|
|
@@ -165,22 +184,33 @@ class ConanLockParse:
|
|
|
165
184
|
self._process_list = []
|
|
166
185
|
|
|
167
186
|
def conan_install(self, cmd, com):
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
187
|
+
com_name = com[:com.index("/")]
|
|
188
|
+
com_install_log = os.path.join(self.work.config.temp_path, f"log/conan_install/{com_name}.log")
|
|
189
|
+
with os.fdopen(os.open(com_install_log, os.O_RDWR | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), "a+") as f:
|
|
190
|
+
self.work.info(f"命令 {cmd} " + Fore.GREEN + "开始" + Style.RESET_ALL)
|
|
191
|
+
real_cmd = shlex.split(cmd)
|
|
192
|
+
pipe = subprocess.Popen(real_cmd, stdout=f, stderr=f)
|
|
193
|
+
start_time = time.time()
|
|
194
|
+
while True:
|
|
195
|
+
if pipe.poll() is not None:
|
|
196
|
+
break
|
|
197
|
+
# 1800秒超时
|
|
198
|
+
if time.time() - start_time > 1800:
|
|
199
|
+
pipe.kill()
|
|
200
|
+
self.work.error(f"================== {com} 构建超时 ==================")
|
|
201
|
+
self._queue.put(self.error_str)
|
|
202
|
+
return
|
|
203
|
+
time.sleep(1)
|
|
204
|
+
if pipe.returncode != 0:
|
|
205
|
+
self.work.error(f"================== {com} 构建失败日志起始位置 ==================")
|
|
206
|
+
f.seek(0)
|
|
207
|
+
conan_log = f.read()
|
|
208
|
+
self.work.info(conan_log)
|
|
209
|
+
self.work.error(f"================== {com} 构建失败日志结束位置 ==================")
|
|
210
|
+
self._queue.put(self.error_str)
|
|
211
|
+
return
|
|
212
|
+
self._queue.put(com)
|
|
213
|
+
self.work.info(f"命令 {cmd} " + Fore.GREEN + "执行完成" + Style.RESET_ALL)
|
|
184
214
|
|
|
185
215
|
def wait_task_finished(self):
|
|
186
216
|
# 所有入度为0任务拉起后等待完成消息
|
|
@@ -210,6 +240,9 @@ class ConanLockParse:
|
|
|
210
240
|
self._degree = {x: len(self._bundle[x].get('requires')) if self._bundle[x].get('requires') is not None else 0
|
|
211
241
|
for x in self._bundle.keys()}
|
|
212
242
|
|
|
243
|
+
conan_log_path = os.path.join(self.work.config.temp_path, "log/conan_install")
|
|
244
|
+
os.makedirs(conan_log_path, exist_ok=True)
|
|
245
|
+
|
|
213
246
|
self.work.success(">>>>>>>>>>>>>>>>> 开始构建所有组件 <<<<<<<<<<<<<<<<")
|
|
214
247
|
self.work.info("这些进程将要花费几分钟(2-3分钟), 请耐心等待")
|
|
215
248
|
while self._degree:
|
|
@@ -226,7 +259,6 @@ class ConanLockParse:
|
|
|
226
259
|
t = Process(target=self.conan_install, args=(cmd_tmp, com))
|
|
227
260
|
t.start()
|
|
228
261
|
self._process_list.append(t)
|
|
229
|
-
self.work.info(f"命令 {cmd_tmp} " + Fore.GREEN + "开始" + Style.RESET_ALL)
|
|
230
262
|
# 入度为0的任务构建拉起后弹出
|
|
231
263
|
self._degree.pop(com)
|
|
232
264
|
else:
|
|
@@ -243,7 +275,12 @@ class TaskClass(Task):
|
|
|
243
275
|
|
|
244
276
|
def __init__(self, config: Config, work_name=""):
|
|
245
277
|
super(TaskClass, self).__init__(config, work_name)
|
|
278
|
+
self.skip_install_comp = False
|
|
279
|
+
self.only_manifest_yml = False
|
|
246
280
|
self.update_path()
|
|
281
|
+
self.component_check = ""
|
|
282
|
+
# 记录每个组件的GraphNode信息
|
|
283
|
+
self.package_info = os.path.join(self.config.rootfs_path, "package_info")
|
|
247
284
|
|
|
248
285
|
@property
|
|
249
286
|
def subsys_dir(self):
|
|
@@ -261,6 +298,13 @@ class TaskClass(Task):
|
|
|
261
298
|
return PERSIST_TYPES[item]
|
|
262
299
|
return None
|
|
263
300
|
|
|
301
|
+
@staticmethod
|
|
302
|
+
def match_persist_type(usage):
|
|
303
|
+
for item in usage:
|
|
304
|
+
if item in PERSIST_TYPES:
|
|
305
|
+
return PERSIST_TYPES[item]
|
|
306
|
+
return None
|
|
307
|
+
|
|
264
308
|
def find_conan_package_and_write(self, search, comps, file_handler, stage):
|
|
265
309
|
if search.find("/") < 0:
|
|
266
310
|
search += "/"
|
|
@@ -273,18 +317,25 @@ class TaskClass(Task):
|
|
|
273
317
|
if stage != misc.StageEnum.STAGE_DEV.value and comp.endswith(f"/{misc.StageEnum.STAGE_DEV.value}"):
|
|
274
318
|
err_msg = f"组件包 {comp} user/channel 配置错误, 必须以 \"{stage}\" 作为结尾, 终止构建"
|
|
275
319
|
raise errors.BmcGoException(err_msg)
|
|
320
|
+
if misc.conan_v2():
|
|
321
|
+
comp = comp.lower()
|
|
276
322
|
self.add_new_dependencies(comp, file_handler)
|
|
277
323
|
return
|
|
324
|
+
# 判断组件是否在openubmcsdk中定义
|
|
325
|
+
if search[:-1] in self.component_check.openubmcsdk_dict.keys():
|
|
326
|
+
if misc.conan_v2():
|
|
327
|
+
comp = comp.lower()
|
|
328
|
+
self.add_new_dependencies(comp, file_handler)
|
|
329
|
+
return
|
|
278
330
|
raise errors.ConfigException(f"未知组件: {search}, 请检查配置 !!!")
|
|
279
331
|
|
|
280
332
|
def update_path(self):
|
|
281
333
|
# conan source folder
|
|
282
|
-
self.conan_source = os.path.join(self.config.
|
|
283
|
-
f"conan_source_{self.config.build_type}_{self.config.stage}/openubmc")
|
|
334
|
+
self.conan_source = os.path.join(self.config.build_path, misc.community_name())
|
|
284
335
|
# conan install folder
|
|
285
336
|
self.conan_install = os.path.join(self.config.build_path, "conan_install")
|
|
286
337
|
# openubmc install folder
|
|
287
|
-
self.openubmc_ins_dir = os.path.join(self.conan_install,
|
|
338
|
+
self.openubmc_ins_dir = os.path.join(self.conan_install, misc.community_name())
|
|
288
339
|
# openubmc top rootfs folder
|
|
289
340
|
self.top_rootfs_dir = os.path.join(self.conan_install, "rootfs")
|
|
290
341
|
self.board_option = ""
|
|
@@ -320,6 +371,18 @@ class TaskClass(Task):
|
|
|
320
371
|
else:
|
|
321
372
|
self.skip_package = False
|
|
322
373
|
|
|
374
|
+
def set_skip_install_comp(self, value):
|
|
375
|
+
if value:
|
|
376
|
+
self.skip_install_comp = True
|
|
377
|
+
else:
|
|
378
|
+
self.skip_install_comp = False
|
|
379
|
+
|
|
380
|
+
def set_only_manifest_yml(self, value):
|
|
381
|
+
if value:
|
|
382
|
+
self.only_manifest_yml = True
|
|
383
|
+
else:
|
|
384
|
+
self.only_manifest_yml = False
|
|
385
|
+
|
|
323
386
|
def package_dependencies_parse(self, default_component: list):
|
|
324
387
|
"""解析manufacture或者tosupporte配置,不同的包设置不同组件的不同编译选项
|
|
325
388
|
参数:
|
|
@@ -346,6 +409,68 @@ class TaskClass(Task):
|
|
|
346
409
|
self.success(f"获取到依赖: {conan}")
|
|
347
410
|
self.depdencies.append(conan)
|
|
348
411
|
|
|
412
|
+
def merge_manifest_v2(self):
|
|
413
|
+
comps = []
|
|
414
|
+
for f in os.listdir(self.subsys_dir):
|
|
415
|
+
with open(os.path.join(self.subsys_dir, f)) as fp:
|
|
416
|
+
yml = yaml.safe_load(fp)
|
|
417
|
+
deps = yml.get('dependencies')
|
|
418
|
+
for dep in deps:
|
|
419
|
+
conan = dep.get(misc.CONAN)
|
|
420
|
+
if conan:
|
|
421
|
+
comps.append(conan)
|
|
422
|
+
self.debug("依赖列表: {}".format(comps))
|
|
423
|
+
# 重建新的依赖关系
|
|
424
|
+
# 从单板目录manifest.yml,与subsys/<stage>目录下的组件合并
|
|
425
|
+
new_fd = os.fdopen(os.open("manifest.yml", os.O_WRONLY | os.O_CREAT,
|
|
426
|
+
stat.S_IWUSR | stat.S_IRUSR), 'w')
|
|
427
|
+
new_fd.write("base:\n")
|
|
428
|
+
ver = self.get_manufacture_config("base/version")
|
|
429
|
+
|
|
430
|
+
new_fd.write(f" version: \"{ver}\"\n")
|
|
431
|
+
self.info(f"包版本号: {ver}")
|
|
432
|
+
|
|
433
|
+
new_fd.write("dependencies:\n")
|
|
434
|
+
deps = self._get_dependencies_pkg()
|
|
435
|
+
|
|
436
|
+
# 如果是打包,依赖信息调整
|
|
437
|
+
deps = self.package_dependencies_parse(deps)
|
|
438
|
+
# 由于manifest.yml当中有对于此的新的配置,此处将配置读出,并重新分配
|
|
439
|
+
skynet_with_enable_luajit = False
|
|
440
|
+
# 获取manifest.yml中"base/sdk"字段值
|
|
441
|
+
openubmcsdk = self.get_manufacture_config("base/sdk", "")
|
|
442
|
+
if openubmcsdk:
|
|
443
|
+
self.add_new_dependencies(openubmcsdk, new_fd)
|
|
444
|
+
self.component_check = ComponentVersionCheck(
|
|
445
|
+
manifest_yml="manifest.yml",
|
|
446
|
+
ibmc_lock=IBMC_LOCK_FILE,
|
|
447
|
+
community_name="openubmc",
|
|
448
|
+
openubmcsdk=openubmcsdk
|
|
449
|
+
)
|
|
450
|
+
for dep in deps:
|
|
451
|
+
conan = dep.get(misc.CONAN)
|
|
452
|
+
if not conan:
|
|
453
|
+
continue
|
|
454
|
+
if conan.find("@") > 0:
|
|
455
|
+
self.add_new_dependencies(conan.lower(), new_fd)
|
|
456
|
+
else:
|
|
457
|
+
self.find_conan_package_and_write(conan, comps, new_fd, self.config.stage)
|
|
458
|
+
options = dep.get("options", {})
|
|
459
|
+
name = conan.split("/")[0]
|
|
460
|
+
for key, val in options.items():
|
|
461
|
+
# skynet特殊处理:manifest.yml指定enable_luajit特性时需要覆盖用户输入
|
|
462
|
+
if name == "skynet" and key == "enable_luajit":
|
|
463
|
+
self.warning(f"根据manifest.yml配置,当前产品的enable_luajit配置为{val},忽略命令行指定的-jit参数")
|
|
464
|
+
skynet_with_enable_luajit = True
|
|
465
|
+
self.config.set_enable_luajit(val)
|
|
466
|
+
self.board_option = self.board_option + " -o */*:{}={}".format(key, val)
|
|
467
|
+
else:
|
|
468
|
+
self.board_option = self.board_option + " -o {}/*:{}={}".format(name, key, val)
|
|
469
|
+
# 当使能Luajit又未向skynet传递enable_luajit配置项时需要添加使能参数
|
|
470
|
+
if not skynet_with_enable_luajit and self.config.enable_luajit:
|
|
471
|
+
self.board_option += f" -o */*:enable_luajit={self.config.enable_luajit}"
|
|
472
|
+
new_fd.close()
|
|
473
|
+
|
|
349
474
|
def merge_manifest(self):
|
|
350
475
|
comps = []
|
|
351
476
|
for f in os.listdir(self.subsys_dir):
|
|
@@ -364,7 +489,7 @@ class TaskClass(Task):
|
|
|
364
489
|
new_fd.write("base:\n")
|
|
365
490
|
ver = self.get_manufacture_config("base/version")
|
|
366
491
|
|
|
367
|
-
new_fd.write(f" version: \"{ver}@{
|
|
492
|
+
new_fd.write(f" version: \"{ver}@{misc.conan_user()}/{misc.StageEnum.STAGE_RC.value}\"\n")
|
|
368
493
|
self.info(f"包版本号: {ver}")
|
|
369
494
|
|
|
370
495
|
new_fd.write("dependencies:\n")
|
|
@@ -374,6 +499,16 @@ class TaskClass(Task):
|
|
|
374
499
|
deps = self.package_dependencies_parse(deps)
|
|
375
500
|
# 由于manifest.yml当中有对于此的新的配置,此处将配置读出,并重新分配
|
|
376
501
|
skynet_with_enable_luajit = False
|
|
502
|
+
# 获取manifest.yml中定义"base/sdk"字段值
|
|
503
|
+
openubmcsdk = self.get_manufacture_config("base/sdk", "")
|
|
504
|
+
if openubmcsdk:
|
|
505
|
+
self.add_new_dependencies(openubmcsdk, new_fd)
|
|
506
|
+
self.component_check = ComponentVersionCheck(
|
|
507
|
+
manifest_yml="manifest.yml",
|
|
508
|
+
ibmc_lock=IBMC_LOCK_FILE,
|
|
509
|
+
community_name="openubmc",
|
|
510
|
+
openubmcsdk=openubmcsdk
|
|
511
|
+
)
|
|
377
512
|
for dep in deps:
|
|
378
513
|
conan = dep.get(misc.CONAN)
|
|
379
514
|
if not conan:
|
|
@@ -384,7 +519,7 @@ class TaskClass(Task):
|
|
|
384
519
|
stage = self.config.stage
|
|
385
520
|
if stage != misc.StageEnum.STAGE_STABLE.value:
|
|
386
521
|
stage = misc.StageEnum.STAGE_RC.value
|
|
387
|
-
conan += f"@{
|
|
522
|
+
conan += f"@{misc.conan_user()}/{stage}"
|
|
388
523
|
self.add_new_dependencies(conan, new_fd)
|
|
389
524
|
else:
|
|
390
525
|
self.find_conan_package_and_write(conan, comps, new_fd, self.config.stage)
|
|
@@ -400,17 +535,26 @@ class TaskClass(Task):
|
|
|
400
535
|
# 当使能Luajit又未向skynet传递enable_luajit配置项时需要添加使能参数
|
|
401
536
|
if not skynet_with_enable_luajit and self.config.enable_luajit:
|
|
402
537
|
self.board_option += f" -o skynet:enable_luajit={self.config.enable_luajit}"
|
|
403
|
-
self.merge_0502_default_options()
|
|
404
538
|
sha256 = Tools.sha256sum(SDK_ROOT_MODULE_SYMVERS)
|
|
405
539
|
self.board_option += " -o *:module_symvers={}".format(sha256)
|
|
406
540
|
new_fd.close()
|
|
407
541
|
|
|
542
|
+
def merge_dep_options(self):
|
|
543
|
+
self.merge_0502_default_options()
|
|
544
|
+
|
|
408
545
|
def merge_0502_default_options(self):
|
|
546
|
+
if misc.conan_v1():
|
|
547
|
+
prefix = "*"
|
|
548
|
+
else:
|
|
549
|
+
prefix = "*/*"
|
|
409
550
|
if self.config.manufacture_code:
|
|
410
551
|
default_options_path = f"manufacture/{self.config.manufacture_code}/default_options"
|
|
411
552
|
default_options = self.get_manufacture_config(default_options_path, {})
|
|
412
553
|
for key, val in default_options.items():
|
|
413
|
-
self.board_option += f" -o
|
|
554
|
+
self.board_option += f" -o {prefix}:{key}={val}"
|
|
555
|
+
module_symvers_path = os.path.join(SDK_PATH, MODULE_SYMVERS)
|
|
556
|
+
option, value = self.tools.get_hi171x_module_symver_option(module_symvers_path)
|
|
557
|
+
self.board_option += f" -o {prefix}:{option}={value}"
|
|
414
558
|
|
|
415
559
|
def package_info_gen(self, bundle_file, dst_file):
|
|
416
560
|
with open(bundle_file, "r") as fp:
|
|
@@ -419,8 +563,8 @@ class TaskClass(Task):
|
|
|
419
563
|
list.sort(require_list)
|
|
420
564
|
package_info_list = deepcopy(require_list)
|
|
421
565
|
self.run_command(f"rm -rf {dst_file}", sudo=True)
|
|
422
|
-
|
|
423
|
-
with os.fdopen(os.open(
|
|
566
|
+
self.package_info = f"{self.config.rootfs_path}/{os.path.basename(dst_file)}"
|
|
567
|
+
with os.fdopen(os.open(self.package_info, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, stat.S_IWUSR | stat.S_IRUSR |
|
|
424
568
|
stat.S_IWGRP | stat.S_IRGRP | stat.S_IWOTH | stat.S_IROTH), 'w') as fp:
|
|
425
569
|
for package in require_list:
|
|
426
570
|
if "openubmc/" not in package:
|
|
@@ -428,8 +572,7 @@ class TaskClass(Task):
|
|
|
428
572
|
else:
|
|
429
573
|
package_info_list.remove(package)
|
|
430
574
|
fp.close()
|
|
431
|
-
self.run_command(f"cp -df {
|
|
432
|
-
self.run_command(f"rm -rf {temp_dst_file}", sudo=True)
|
|
575
|
+
self.run_command(f"cp -df {self.package_info} {dst_file}", sudo=True)
|
|
433
576
|
|
|
434
577
|
def sensitive_data_conf_gen(self, comps, dst_file):
|
|
435
578
|
self.run_command(f"rm -rf {dst_file}", sudo=True)
|
|
@@ -500,6 +643,78 @@ class TaskClass(Task):
|
|
|
500
643
|
options += " -o *:gcov=True"
|
|
501
644
|
return options
|
|
502
645
|
|
|
646
|
+
def proc_sensitive_v2(self, table_type, props, table_name, output):
|
|
647
|
+
for prop_name, prop_data in props.items():
|
|
648
|
+
if prop_data.get("primaryKey"):
|
|
649
|
+
output[PRIMARY_KEYS][table_name] = output[PRIMARY_KEYS].get(table_name, {})
|
|
650
|
+
output[PRIMARY_KEYS][table_name][prop_name] = prop_data.get("sensitive", False)
|
|
651
|
+
continue
|
|
652
|
+
per_type = self.match_persist_type(prop_data.get("usage", [])) or table_type
|
|
653
|
+
if not per_type:
|
|
654
|
+
continue
|
|
655
|
+
output[per_type][table_name] = output[per_type].get(table_name, {})
|
|
656
|
+
output[per_type][table_name][prop_name] = prop_data.get("sensitive", False)
|
|
657
|
+
return output
|
|
658
|
+
|
|
659
|
+
def proc_comp_v2(self, package_folder, output):
|
|
660
|
+
model_file = os.path.join(package_folder, "include", "mds", "model.json")
|
|
661
|
+
if not os.path.isfile(model_file):
|
|
662
|
+
return output
|
|
663
|
+
with open(model_file, "r") as fp:
|
|
664
|
+
content = json.load(fp)
|
|
665
|
+
for class_data in content.values():
|
|
666
|
+
table_type = PERSIST_TYPES.get(class_data.get("tableType", ""))
|
|
667
|
+
table_name = class_data.get("tableName", "")
|
|
668
|
+
if not table_name or class_data.get("tableLocation") == "Local":
|
|
669
|
+
continue
|
|
670
|
+
class_props = [class_data.get("properties", {})]
|
|
671
|
+
for intf_data in class_data.get("interfaces", {}).values():
|
|
672
|
+
class_props.append(intf_data.get("properties", {}))
|
|
673
|
+
for props in class_props:
|
|
674
|
+
output = self.proc_sensitive_v2(table_type, props, table_name, output)
|
|
675
|
+
return output
|
|
676
|
+
|
|
677
|
+
def calc_options_v2(self):
|
|
678
|
+
if self.config.build_type == "debug":
|
|
679
|
+
options = "-s build_type=Debug"
|
|
680
|
+
else:
|
|
681
|
+
options = "-s build_type=Release"
|
|
682
|
+
|
|
683
|
+
if self.config.enable_arm_gcov:
|
|
684
|
+
options += " -o */*:gcov=True"
|
|
685
|
+
return options
|
|
686
|
+
|
|
687
|
+
def install_openubmc_v2(self):
|
|
688
|
+
profile_file = os.path.join(self.tools.conan_profiles_dir, self.config.profile)
|
|
689
|
+
if not os.path.isfile(profile_file):
|
|
690
|
+
raise BmcGoException(f"{profile_file} 文件不存在")
|
|
691
|
+
|
|
692
|
+
options = self.calc_options_v2()
|
|
693
|
+
append_cmd = f"-r {self.config.remote}" if self.config.remote else ""
|
|
694
|
+
# 依据选项生成openubmc.lock和openubmc.bundle文件
|
|
695
|
+
base_cmd = f"-pr={self.config.profile} "
|
|
696
|
+
base_cmd += f"-pr:b profile.dt.ini {append_cmd} {options} {self.board_option}"
|
|
697
|
+
lockfile = os.path.join(self.config.build_path, "conan.lock")
|
|
698
|
+
graphfile = os.path.join(self.config.build_path, "graph.info")
|
|
699
|
+
lock_cmd = f"conan lock create . {base_cmd} --lockfile-out={lockfile}"
|
|
700
|
+
self.run_command(lock_cmd, capture_output=False)
|
|
701
|
+
graph_cmd = f"conan graph info . {base_cmd} -f json --lockfile={lockfile} --out-file={graphfile}"
|
|
702
|
+
self.run_command(graph_cmd)
|
|
703
|
+
self.success(f"start build dependency packages of {self.config.board_name}")
|
|
704
|
+
bcp = BuildConans(graphfile, lockfile, base_cmd, self.config.from_source, self.config.log_path)
|
|
705
|
+
bcp.build()
|
|
706
|
+
|
|
707
|
+
cmd = f"conan create . {base_cmd} --build=missing"
|
|
708
|
+
self.info(f"start build {self.config.board_name}: {cmd}")
|
|
709
|
+
self.run_command(cmd, capture_output=False)
|
|
710
|
+
|
|
711
|
+
self.graphfile = graphfile
|
|
712
|
+
|
|
713
|
+
shutil.copyfile(lockfile, f"{self.conan_install}/conan.lock")
|
|
714
|
+
# 检查使用到的组件是否都在单板目录 manifest.yml 中配置了
|
|
715
|
+
self.component_check.run()
|
|
716
|
+
self.clean_luac_out()
|
|
717
|
+
|
|
503
718
|
def install_ibmc(self):
|
|
504
719
|
profile_file = os.path.join(self.tools.conan_profiles_dir, self.config.profile)
|
|
505
720
|
if not os.path.isfile(profile_file):
|
|
@@ -539,21 +754,29 @@ class TaskClass(Task):
|
|
|
539
754
|
self.run_command(install_cmd, command_echo=True)
|
|
540
755
|
shutil.copyfile(IBMC_LOCK_FILE, f"{self.conan_install}/conan.lock")
|
|
541
756
|
# 检查使用到的组件是否都在单板目录 manifest.yml 中配置了
|
|
542
|
-
component_check
|
|
543
|
-
component_check.run()
|
|
757
|
+
self.component_check.run()
|
|
544
758
|
self.clean_luac_out()
|
|
545
759
|
|
|
546
760
|
def deploy(self):
|
|
547
|
-
|
|
548
|
-
shutil.copytree(src, self.conan_source, dirs_exist_ok=True)
|
|
761
|
+
conanfile_dir = os.path.join(cwd_dir, "conan")
|
|
549
762
|
openubmc_dir = os.path.join(self.conan_source, "all")
|
|
763
|
+
os.makedirs(openubmc_dir, exist_ok=True)
|
|
764
|
+
shutil.copytree(conanfile_dir, openubmc_dir, dirs_exist_ok=True)
|
|
550
765
|
self.chdir(openubmc_dir)
|
|
551
766
|
# 复制manifest.yml文件
|
|
552
|
-
|
|
767
|
+
if misc.conan_v2():
|
|
768
|
+
self.merge_manifest_v2()
|
|
769
|
+
else:
|
|
770
|
+
self.merge_manifest()
|
|
771
|
+
if self.only_manifest_yml:
|
|
772
|
+
return
|
|
773
|
+
if not self.skip_install_comp:
|
|
774
|
+
self.merge_dep_options()
|
|
553
775
|
# 下载组件构建脚本
|
|
554
776
|
ComponentHelper.download_recipes(self.depdencies, self.tools, self.config.remote_list)
|
|
555
777
|
# 下载skynet
|
|
556
|
-
|
|
778
|
+
if misc.conan_v1():
|
|
779
|
+
self.install_luac_or_luajit()
|
|
557
780
|
|
|
558
781
|
# 复制全局定制rootfs到conan install目录
|
|
559
782
|
top_rootfs = os.path.join(self.config.code_path, "rootfs")
|
|
@@ -578,7 +801,10 @@ class TaskClass(Task):
|
|
|
578
801
|
if os.path.isfile(per_file):
|
|
579
802
|
shutil.copy(per_file, self.openubmc_ins_dir)
|
|
580
803
|
|
|
581
|
-
|
|
804
|
+
if misc.conan_v2():
|
|
805
|
+
self.install_openubmc_v2()
|
|
806
|
+
else:
|
|
807
|
+
self.install_ibmc()
|
|
582
808
|
|
|
583
809
|
def clean_luac_out(self):
|
|
584
810
|
# 清理冗余文件luac.out
|
|
@@ -600,15 +826,15 @@ class TaskClass(Task):
|
|
|
600
826
|
self.run_command(f'cp -f conan.lock {os.path.join(inner_path, f"package_{self.config.board_name}.lock")}')
|
|
601
827
|
self.run_command(f'cp -f conan.lock {os.path.join(self.config.output_path, f"package.lock")}')
|
|
602
828
|
|
|
603
|
-
def copy_components(self, comps: list, profile
|
|
829
|
+
def copy_components(self, comps: list, profile):
|
|
604
830
|
# 优先处理rootfs
|
|
605
|
-
p = CopyComponent(self, "rootfs", profile)
|
|
831
|
+
p = CopyComponent(self, "rootfs", os.path.join(self.conan_install, "rootfs"), profile)
|
|
606
832
|
p.run()
|
|
607
833
|
# 打印组件清单
|
|
608
834
|
self.info(f"组件列表: {comps}")
|
|
609
835
|
pools = []
|
|
610
836
|
for comp in comps:
|
|
611
|
-
p = CopyComponent(self, comp, profile)
|
|
837
|
+
p = CopyComponent(self, comp, os.path.join(self.conan_install, comp), profile)
|
|
612
838
|
p.start()
|
|
613
839
|
pools.append(p)
|
|
614
840
|
|
|
@@ -621,11 +847,161 @@ class TaskClass(Task):
|
|
|
621
847
|
raise errors.BmcGoException(f"复制组件 ({p.comp}) 失败, 退出码: {p.exitcode}")
|
|
622
848
|
pools.remove(p)
|
|
623
849
|
# 最后处理openubmc
|
|
624
|
-
p = CopyComponent(self,
|
|
850
|
+
p = CopyComponent(self, misc.community_name(), os.path.join(self.conan_install, misc.community_name()), profile)
|
|
851
|
+
p.run()
|
|
852
|
+
|
|
853
|
+
def package_info_gen_v2(self, bmc_lock, dst_file):
|
|
854
|
+
with open(bmc_lock, "r") as bmc_lock_fp:
|
|
855
|
+
lock_info = json.load(bmc_lock_fp)
|
|
856
|
+
require_list = lock_info.get("requires")
|
|
857
|
+
self.run_command(f"rm -rf {dst_file}", sudo=True)
|
|
858
|
+
self.package_info = f"{self.config.rootfs_path}/{os.path.basename(dst_file)}"
|
|
859
|
+
with os.fdopen(os.open(self.package_info, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, stat.S_IWUSR | stat.S_IRUSR |
|
|
860
|
+
stat.S_IWGRP | stat.S_IRGRP | stat.S_IWOTH | stat.S_IROTH), 'w') as fp:
|
|
861
|
+
for package in require_list:
|
|
862
|
+
fp.write(f"{package}\n")
|
|
863
|
+
fp.close()
|
|
864
|
+
self.run_command(f"cp -df {self.package_info} {dst_file}", sudo=True)
|
|
865
|
+
|
|
866
|
+
def sensitive_data_conf_gen_v2(self, dst_file):
|
|
867
|
+
self.run_command(f"rm -rf {dst_file}", sudo=True)
|
|
868
|
+
temp_dst_file = f"{self.config.rootfs_path}/{os.path.basename(dst_file)}"
|
|
869
|
+
output = {
|
|
870
|
+
"TemporaryPer": {},
|
|
871
|
+
"ResetPer": {},
|
|
872
|
+
"PoweroffPer": {},
|
|
873
|
+
"PermanentPer": {},
|
|
874
|
+
"PrimaryKeys": {}
|
|
875
|
+
}
|
|
876
|
+
for name, node in self.graph_nodes.items():
|
|
877
|
+
try:
|
|
878
|
+
output = self.proc_comp_v2(node.package_folder, output)
|
|
879
|
+
except Exception as e:
|
|
880
|
+
raise errors.BmcGoException(f"分析组件 {name} 的 model.json 失败, 失败信息:{e}")
|
|
881
|
+
|
|
882
|
+
with os.fdopen(os.open(temp_dst_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC,
|
|
883
|
+
stat.S_IWUSR | stat.S_IRUSR), 'w') as fp:
|
|
884
|
+
json.dump(output, fp, sort_keys=True)
|
|
885
|
+
self.run_command(f"mkdir -p {os.path.dirname(dst_file)}", sudo=True)
|
|
886
|
+
self.run_command(f"cp -df {temp_dst_file} {dst_file}", sudo=True)
|
|
887
|
+
self.run_command(f"rm -rf {temp_dst_file}", sudo=True)
|
|
888
|
+
|
|
889
|
+
def get_all_subdirectories(self, dirpath):
|
|
890
|
+
exclude_files = ["permissions.ini", "conaninfo.txt", "conanmanifest.txt"]
|
|
891
|
+
out = []
|
|
892
|
+
for root, _, files in os.walk(dirpath):
|
|
893
|
+
for file in files:
|
|
894
|
+
file = os.path.join(root, file)
|
|
895
|
+
relative_path = os.path.relpath(file, dirpath)
|
|
896
|
+
if not relative_path.startswith("include") and relative_path not in exclude_files:
|
|
897
|
+
out.append(relative_path)
|
|
898
|
+
return out
|
|
899
|
+
|
|
900
|
+
def print_dup_files(self, comps, paths):
|
|
901
|
+
dup_paths = {}
|
|
902
|
+
for comp, files in comps.items():
|
|
903
|
+
for path in paths:
|
|
904
|
+
if path in files:
|
|
905
|
+
dup_paths.setdefault(path, []).append(comp)
|
|
906
|
+
|
|
907
|
+
for path, comp in dup_paths.items():
|
|
908
|
+
self.error(f"{comp} 存在重复文件 {path}")
|
|
909
|
+
|
|
910
|
+
def copy_components_v2(self, profile):
|
|
911
|
+
# 优先处理rootfs
|
|
912
|
+
p = CopyComponent(self, "rootfs", os.path.join(self.conan_install, "rootfs"), profile)
|
|
913
|
+
p.run()
|
|
914
|
+
# 打印组件清单
|
|
915
|
+
self.info(f"组件列表: {self.graph_nodes.keys()}")
|
|
916
|
+
pools = []
|
|
917
|
+
# 记录全部的文件信息
|
|
918
|
+
all_path = []
|
|
919
|
+
# 记录组件有哪些文件
|
|
920
|
+
comp_path = {}
|
|
921
|
+
for name, node in self.graph_nodes.items():
|
|
922
|
+
if name in ["rootfs", misc.community_name()]:
|
|
923
|
+
continue
|
|
924
|
+
file_paths = self.get_all_subdirectories(node.package_folder)
|
|
925
|
+
all_path.extend(file_paths)
|
|
926
|
+
comp_path[name] = file_paths
|
|
927
|
+
p = CopyComponent(self, node.name, node.package_folder, profile)
|
|
928
|
+
p.start()
|
|
929
|
+
pools.append(p)
|
|
930
|
+
|
|
931
|
+
while pools:
|
|
932
|
+
time.sleep(0.01)
|
|
933
|
+
for p in pools:
|
|
934
|
+
if p.is_alive():
|
|
935
|
+
continue
|
|
936
|
+
if p.exitcode is not None and p.exitcode != 0:
|
|
937
|
+
raise errors.BmcGoException(f"复制组件 ({p.name}) 失败, 退出码: {p.exitcode}")
|
|
938
|
+
pools.remove(p)
|
|
939
|
+
counter = Counter(all_path)
|
|
940
|
+
duplicates = {path for path, count in counter.items() if count > 1}
|
|
941
|
+
if len(duplicates):
|
|
942
|
+
self.print_dup_files(comp_path, duplicates)
|
|
943
|
+
raise errors.BmcGoException("请检查组件的打包逻辑,检测到重复文件: " + ".".join(duplicates))
|
|
944
|
+
# 最后处理openubmc
|
|
945
|
+
p = CopyComponent(self, misc.community_name(), os.path.join(self.conan_install, misc.community_name()), profile)
|
|
625
946
|
p.run()
|
|
947
|
+
self.run_command(f"sudo rm -f {self.config.rootfs_path}/conaninfo.txt")
|
|
948
|
+
self.run_command(f"sudo rm -f {self.config.rootfs_path}/conanmanifest.txt")
|
|
949
|
+
|
|
950
|
+
def update_graph_nodes(self):
|
|
951
|
+
rootfs = GraphNode(None)
|
|
952
|
+
rootfs.name = "rootfs"
|
|
953
|
+
rootfs.package_folder = os.path.join(self.conan_install, "rootfs")
|
|
954
|
+
self.graph_nodes["rootfs"] = rootfs
|
|
955
|
+
|
|
956
|
+
with open(self.graphfile, "r") as fp:
|
|
957
|
+
graph = json.load(fp)
|
|
958
|
+
nodes = graph.get("graph", {}).get("nodes", {})
|
|
959
|
+
for _, info in nodes.items():
|
|
960
|
+
node = GraphNode(info)
|
|
961
|
+
context = info.get("context")
|
|
962
|
+
if context != "host":
|
|
963
|
+
continue
|
|
964
|
+
cmd = f"conan cache path {node.ref}:{node.package_id}"
|
|
965
|
+
node.package_folder = tools.run_command(cmd, capture_output=True).stdout.strip()
|
|
966
|
+
self.graph_nodes[node.name] = node
|
|
967
|
+
|
|
968
|
+
rootfs = GraphNode(None)
|
|
969
|
+
rootfs.name = misc.community_name()
|
|
970
|
+
rootfs.package_folder = os.path.join(self.conan_install, misc.community_name())
|
|
971
|
+
self.graph_nodes[misc.community_name()] = rootfs
|
|
972
|
+
|
|
973
|
+
def make_customization_symlinks(self):
|
|
974
|
+
if os.path.isdir(self.customization_dir):
|
|
975
|
+
self.run_command(f"rm -rf {self.customization_dir}")
|
|
976
|
+
self.run_command(f"mkdir -p {self.customization_dir}")
|
|
977
|
+
for name, node in self.graph_nodes.items():
|
|
978
|
+
cust = os.path.join(node.package_folder, "include", "customization.py")
|
|
979
|
+
if not os.path.isfile(cust):
|
|
980
|
+
continue
|
|
981
|
+
os.symlink(node.package_folder, os.path.join(self.customization_dir, name))
|
|
982
|
+
|
|
983
|
+
def package_v2(self):
|
|
984
|
+
self.update_graph_nodes()
|
|
985
|
+
self.make_customization_symlinks()
|
|
986
|
+
self.run_command(f"sudo rm -rf {self.config.rootfs_path}")
|
|
987
|
+
os.makedirs(self.config.rootfs_path)
|
|
988
|
+
profile, _ = self.get_profile_config()
|
|
989
|
+
self.copy_components_v2(profile)
|
|
990
|
+
|
|
991
|
+
self.chdir(self.config.rootfs_path)
|
|
992
|
+
self._component_cust_action("post_image")
|
|
993
|
+
|
|
994
|
+
self.chdir(self.config.rootfs_path)
|
|
995
|
+
self.package_info_gen_v2(f"{self.conan_install}/conan.lock", "etc/package_info")
|
|
996
|
+
self.sensitive_data_conf_gen_v2("opt/bmc/trust/sensitive_data.json")
|
|
997
|
+
if os.path.isfile("permissions.ini"):
|
|
998
|
+
os.unlink("permissions.ini")
|
|
626
999
|
|
|
627
1000
|
def package(self):
|
|
628
1001
|
self.chdir(self.conan_install)
|
|
1002
|
+
if misc.conan_v2():
|
|
1003
|
+
self.package_v2()
|
|
1004
|
+
return
|
|
629
1005
|
self.run_command(f"sudo rm -rf {self.config.rootfs_path}")
|
|
630
1006
|
os.makedirs(self.config.rootfs_path)
|
|
631
1007
|
profile, _ = self.get_profile_config()
|
|
@@ -638,14 +1014,8 @@ class TaskClass(Task):
|
|
|
638
1014
|
|
|
639
1015
|
self.copy_components(comps, profile)
|
|
640
1016
|
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
cust = os.path.join(self.conan_install, comp, "include", "customization.py")
|
|
644
|
-
if not os.path.isfile(cust):
|
|
645
|
-
continue
|
|
646
|
-
self.info(f"开始执行 {comp}/include/customization.py")
|
|
647
|
-
post = ComponentPost(self.config, os.path.join(self.conan_install, comp), profile)
|
|
648
|
-
post.post_work(self.config.rootfs_path, "post_image")
|
|
1017
|
+
self.chdir(self.config.rootfs_path)
|
|
1018
|
+
self._component_cust_action("post_image")
|
|
649
1019
|
|
|
650
1020
|
self.chdir(self.config.rootfs_path)
|
|
651
1021
|
self.package_info_gen(f"{self.conan_source}/all/openubmc.bundle", "etc/package_info")
|
|
@@ -721,6 +1091,8 @@ class TaskClass(Task):
|
|
|
721
1091
|
self.mkdir_work_path()
|
|
722
1092
|
self.tools.clean_locks()
|
|
723
1093
|
self.deploy()
|
|
1094
|
+
if self.only_manifest_yml:
|
|
1095
|
+
return
|
|
724
1096
|
self.package_lock()
|
|
725
1097
|
if not self.skip_package:
|
|
726
1098
|
self.package()
|