openubmc-bingo 0.5.277__py3-none-any.whl → 0.6.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of openubmc-bingo might be problematic. Click here for more details.

Files changed (53) hide show
  1. bmcgo/__init__.py +1 -1
  2. bmcgo/bmcgo_config.py +22 -10
  3. bmcgo/cli/cli.py +95 -39
  4. bmcgo/cli/config.conan2.yaml +9 -0
  5. bmcgo/codegen/lua/codegen.py +2 -2
  6. bmcgo/codegen/lua/script/gen_intf_rpc_json.py +15 -14
  7. bmcgo/component/analysis/analysis.py +8 -8
  8. bmcgo/component/analysis/intf_validation.py +23 -12
  9. bmcgo/component/build.py +76 -14
  10. bmcgo/component/component_dt_version_parse.py +3 -2
  11. bmcgo/component/component_helper.py +43 -15
  12. bmcgo/component/coverage/incremental_cov.py +2 -2
  13. bmcgo/component/deploy.py +68 -14
  14. bmcgo/component/gen.py +1 -1
  15. bmcgo/component/package_info.py +128 -38
  16. bmcgo/component/template/conanbase.py.mako +1 -0
  17. bmcgo/component/template_v2/conanbase.py.mako +383 -0
  18. bmcgo/component/template_v2/conanfile.deploy.py.mako +26 -0
  19. bmcgo/component/test.py +53 -20
  20. bmcgo/frame.py +7 -3
  21. bmcgo/functional/analysis.py +3 -2
  22. bmcgo/functional/check.py +10 -6
  23. bmcgo/functional/conan_index_build.py +96 -20
  24. bmcgo/functional/csr_build.py +1 -1
  25. bmcgo/functional/diff.py +1 -1
  26. bmcgo/functional/fetch.py +1 -1
  27. bmcgo/functional/full_component.py +32 -24
  28. bmcgo/functional/git_history.py +220 -0
  29. bmcgo/functional/maintain.py +55 -12
  30. bmcgo/functional/new.py +1 -1
  31. bmcgo/functional/schema_valid.py +2 -2
  32. bmcgo/logger.py +2 -3
  33. bmcgo/misc.py +130 -9
  34. bmcgo/tasks/conan/__init__.py +10 -0
  35. bmcgo/tasks/conan/conanfile.py +45 -0
  36. bmcgo/tasks/task.py +27 -4
  37. bmcgo/tasks/task_build_conan.py +425 -53
  38. bmcgo/tasks/task_buildgppbin.py +8 -2
  39. bmcgo/tasks/task_download_buildtools.py +76 -0
  40. bmcgo/tasks/task_download_dependency.py +1 -0
  41. bmcgo/tasks/task_hpm_envir_prepare.py +1 -1
  42. bmcgo/utils/build_conans.py +231 -0
  43. bmcgo/utils/component_post.py +6 -4
  44. bmcgo/utils/component_version_check.py +75 -16
  45. bmcgo/utils/config.py +76 -40
  46. bmcgo/utils/fetch_component_code.py +44 -25
  47. bmcgo/utils/tools.py +239 -117
  48. bmcgo/worker.py +2 -2
  49. {openubmc_bingo-0.5.277.dist-info → openubmc_bingo-0.6.2.dist-info}/METADATA +4 -2
  50. {openubmc_bingo-0.5.277.dist-info → openubmc_bingo-0.6.2.dist-info}/RECORD +53 -46
  51. {openubmc_bingo-0.5.277.dist-info → openubmc_bingo-0.6.2.dist-info}/WHEEL +0 -0
  52. {openubmc_bingo-0.5.277.dist-info → openubmc_bingo-0.6.2.dist-info}/entry_points.txt +0 -0
  53. {openubmc_bingo-0.5.277.dist-info → openubmc_bingo-0.6.2.dist-info}/top_level.txt +0 -0
@@ -27,10 +27,10 @@ import random
27
27
  import pathlib
28
28
  from multiprocessing import Process, Queue
29
29
  from copy import deepcopy
30
+ from collections import Counter
30
31
 
31
32
  import yaml
32
33
  from colorama import Fore, Back, Style
33
- from conans.model.profile import Profile
34
34
 
35
35
  from bmcgo.tasks.task import Task
36
36
  from bmcgo.utils.config import Config
@@ -40,6 +40,9 @@ from bmcgo import errors, misc
40
40
  from bmcgo.utils.tools import Tools
41
41
  from bmcgo.errors import BmcGoException
42
42
  from bmcgo.component.component_helper import ComponentHelper
43
+ from bmcgo.utils.build_conans import BuildConans
44
+ from bmcgo.component.deploy import GraphNode
45
+ from bmcgo.tasks.misc import MODULE_SYMVERS, SDK_PATH
43
46
 
44
47
  IBMC_LOCK_FILE = "openubmc.lock"
45
48
  SDK_ROOT = "/opt/hi1711sdk"
@@ -56,14 +59,30 @@ PERSIST_TYPES = {
56
59
  PRIMARY_KEYS = "PrimaryKeys"
57
60
  tools = Tools()
58
61
 
62
+ PERSIST_TYPES = {
63
+ "TemporaryPer": "TemporaryPer",
64
+ "TemporaryPerRetain": "TemporaryPer",
65
+ "ResetPer": "ResetPer",
66
+ "ResetPerRetain": "ResetPer",
67
+ "PoweroffPer": "PoweroffPer",
68
+ "PoweroffPerRetain": "PoweroffPer",
69
+ "PermanentPer": "PermanentPer"
70
+ }
71
+
72
+ PRIMARY_KEYS = "PrimaryKeys"
73
+
74
+ cwd_dir = os.path.dirname(os.path.realpath(__file__))
75
+
59
76
 
60
77
  class CopyComponent(Process):
61
- def __init__(self, work: Task, comp: str, profile: Profile):
78
+ def __init__(self, work: Task, comp: str, package_folder: str, profile):
62
79
  super().__init__()
63
80
  self.work = work
64
81
  self.config = self.work.config
65
82
  self.comp = comp
83
+ self.package_folder = package_folder
66
84
  self.profile = profile
85
+ self.graphfile = None
67
86
 
68
87
  def link_recursive_deal(self, file_name, ownership):
69
88
  self.work.run_command(f"chown -h {ownership} {file_name}", sudo=True)
@@ -74,16 +93,16 @@ class CopyComponent(Process):
74
93
  self.work.work_name = self.comp
75
94
  # 复制组件文件到rootfs中
76
95
  # rc.sysinit脚本只允许使用rootfs_user仓库的
77
- rc_sysinit_path = os.path.join(self.comp, "etc/rc.d/rc.sysinit")
96
+ rc_sysinit_path = os.path.join(self.package_folder, "etc/rc.d/rc.sysinit")
78
97
  if self.comp != "rootfs_user" and os.path.isfile(rc_sysinit_path):
79
98
  shutil.rmtree(rc_sysinit_path, ignore_errors=True)
80
99
  # 由于复制时有很多同名路径,cp命令有概率失败,10次复制尝试
81
100
  self._copy_files()
82
101
 
83
102
  # 执行权限控制逻辑
84
- per_cfg = os.path.join(self.comp, "permissions.ini")
103
+ per_cfg = os.path.join(self.package_folder, "permissions.ini")
85
104
  if not os.path.isfile(per_cfg):
86
- self.work.warning("权限文件 {} 不存在, 所以无法设置组件 {} 的权限".format(per_cfg, self.comp))
105
+ self.work.warning("权限文件 {} 不存在, 所以无法设置组件 {} 的权限".format(per_cfg, self.package_folder))
87
106
  return
88
107
  self.work.info("依据配置文件: {}, 修改权限".format(per_cfg))
89
108
  # NOTE 这里的配置时间, 偶发复制未完成, 就开始修改权限
@@ -130,7 +149,7 @@ class CopyComponent(Process):
130
149
 
131
150
  def _copy_files(self):
132
151
  # 由于复制时有很多同名路径,cp命令有概率失败,10次复制尝试
133
- cmd = "sudo cp -dfr {}/. {}".format(self.comp, self.config.rootfs_path)
152
+ cmd = "sudo cp -dfr {}/. {}".format(self.package_folder, self.config.rootfs_path)
134
153
  for _ in range(0, 10):
135
154
  try:
136
155
  # 尝试复制到目标目录,成功则break退出循环
@@ -165,22 +184,33 @@ class ConanLockParse:
165
184
  self._process_list = []
166
185
 
167
186
  def conan_install(self, cmd, com):
168
- real_cmd = shlex.split(cmd)
169
- pipe = subprocess.Popen(real_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
170
- # 超时时长1800s
171
- out, _ = pipe.communicate(timeout=1800)
172
- if pipe.returncode != 0:
173
- self.work.error(f"================== {com} 构建失败日志起始位置 ==================")
174
- self.work.info(out.decode())
175
- self.work.error(f"================== {com} 构建失败日志结束位置 ==================")
176
- self._queue.put(self.error_str)
177
- self._queue.put(self.error_str)
178
- self._queue.put(self.error_str)
179
- return
180
- self._queue.put(com)
181
- self._queue.put(com)
182
- self._queue.put(com)
183
- self.work.info(f"命令 {cmd} " + Fore.GREEN + "执行完成" + Style.RESET_ALL)
187
+ com_name = com[:com.index("/")]
188
+ com_install_log = os.path.join(self.work.config.temp_path, f"log/conan_install/{com_name}.log")
189
+ with os.fdopen(os.open(com_install_log, os.O_RDWR | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), "a+") as f:
190
+ self.work.info(f"命令 {cmd} " + Fore.GREEN + "开始" + Style.RESET_ALL)
191
+ real_cmd = shlex.split(cmd)
192
+ pipe = subprocess.Popen(real_cmd, stdout=f, stderr=f)
193
+ start_time = time.time()
194
+ while True:
195
+ if pipe.poll() is not None:
196
+ break
197
+ # 1800秒超时
198
+ if time.time() - start_time > 1800:
199
+ pipe.kill()
200
+ self.work.error(f"================== {com} 构建超时 ==================")
201
+ self._queue.put(self.error_str)
202
+ return
203
+ time.sleep(1)
204
+ if pipe.returncode != 0:
205
+ self.work.error(f"================== {com} 构建失败日志起始位置 ==================")
206
+ f.seek(0)
207
+ conan_log = f.read()
208
+ self.work.info(conan_log)
209
+ self.work.error(f"================== {com} 构建失败日志结束位置 ==================")
210
+ self._queue.put(self.error_str)
211
+ return
212
+ self._queue.put(com)
213
+ self.work.info(f"命令 {cmd} " + Fore.GREEN + "执行完成" + Style.RESET_ALL)
184
214
 
185
215
  def wait_task_finished(self):
186
216
  # 所有入度为0任务拉起后等待完成消息
@@ -210,6 +240,9 @@ class ConanLockParse:
210
240
  self._degree = {x: len(self._bundle[x].get('requires')) if self._bundle[x].get('requires') is not None else 0
211
241
  for x in self._bundle.keys()}
212
242
 
243
+ conan_log_path = os.path.join(self.work.config.temp_path, "log/conan_install")
244
+ os.makedirs(conan_log_path, exist_ok=True)
245
+
213
246
  self.work.success(">>>>>>>>>>>>>>>>> 开始构建所有组件 <<<<<<<<<<<<<<<<")
214
247
  self.work.info("这些进程将要花费几分钟(2-3分钟), 请耐心等待")
215
248
  while self._degree:
@@ -226,7 +259,6 @@ class ConanLockParse:
226
259
  t = Process(target=self.conan_install, args=(cmd_tmp, com))
227
260
  t.start()
228
261
  self._process_list.append(t)
229
- self.work.info(f"命令 {cmd_tmp} " + Fore.GREEN + "开始" + Style.RESET_ALL)
230
262
  # 入度为0的任务构建拉起后弹出
231
263
  self._degree.pop(com)
232
264
  else:
@@ -243,7 +275,12 @@ class TaskClass(Task):
243
275
 
244
276
  def __init__(self, config: Config, work_name=""):
245
277
  super(TaskClass, self).__init__(config, work_name)
278
+ self.skip_install_comp = False
279
+ self.only_manifest_yml = False
246
280
  self.update_path()
281
+ self.component_check = ""
282
+ # 记录每个组件的GraphNode信息
283
+ self.package_info = os.path.join(self.config.rootfs_path, "package_info")
247
284
 
248
285
  @property
249
286
  def subsys_dir(self):
@@ -261,6 +298,13 @@ class TaskClass(Task):
261
298
  return PERSIST_TYPES[item]
262
299
  return None
263
300
 
301
+ @staticmethod
302
+ def match_persist_type(usage):
303
+ for item in usage:
304
+ if item in PERSIST_TYPES:
305
+ return PERSIST_TYPES[item]
306
+ return None
307
+
264
308
  def find_conan_package_and_write(self, search, comps, file_handler, stage):
265
309
  if search.find("/") < 0:
266
310
  search += "/"
@@ -273,18 +317,25 @@ class TaskClass(Task):
273
317
  if stage != misc.StageEnum.STAGE_DEV.value and comp.endswith(f"/{misc.StageEnum.STAGE_DEV.value}"):
274
318
  err_msg = f"组件包 {comp} user/channel 配置错误, 必须以 \"{stage}\" 作为结尾, 终止构建"
275
319
  raise errors.BmcGoException(err_msg)
320
+ if misc.conan_v2():
321
+ comp = comp.lower()
276
322
  self.add_new_dependencies(comp, file_handler)
277
323
  return
324
+ # 判断组件是否在openubmcsdk中定义
325
+ if search[:-1] in self.component_check.openubmcsdk_dict.keys():
326
+ if misc.conan_v2():
327
+ comp = comp.lower()
328
+ self.add_new_dependencies(comp, file_handler)
329
+ return
278
330
  raise errors.ConfigException(f"未知组件: {search}, 请检查配置 !!!")
279
331
 
280
332
  def update_path(self):
281
333
  # conan source folder
282
- self.conan_source = os.path.join(self.config.temp_path,
283
- f"conan_source_{self.config.build_type}_{self.config.stage}/openubmc")
334
+ self.conan_source = os.path.join(self.config.build_path, misc.community_name())
284
335
  # conan install folder
285
336
  self.conan_install = os.path.join(self.config.build_path, "conan_install")
286
337
  # openubmc install folder
287
- self.openubmc_ins_dir = os.path.join(self.conan_install, "openubmc")
338
+ self.openubmc_ins_dir = os.path.join(self.conan_install, misc.community_name())
288
339
  # openubmc top rootfs folder
289
340
  self.top_rootfs_dir = os.path.join(self.conan_install, "rootfs")
290
341
  self.board_option = ""
@@ -320,6 +371,18 @@ class TaskClass(Task):
320
371
  else:
321
372
  self.skip_package = False
322
373
 
374
+ def set_skip_install_comp(self, value):
375
+ if value:
376
+ self.skip_install_comp = True
377
+ else:
378
+ self.skip_install_comp = False
379
+
380
+ def set_only_manifest_yml(self, value):
381
+ if value:
382
+ self.only_manifest_yml = True
383
+ else:
384
+ self.only_manifest_yml = False
385
+
323
386
  def package_dependencies_parse(self, default_component: list):
324
387
  """解析manufacture或者tosupporte配置,不同的包设置不同组件的不同编译选项
325
388
  参数:
@@ -346,6 +409,68 @@ class TaskClass(Task):
346
409
  self.success(f"获取到依赖: {conan}")
347
410
  self.depdencies.append(conan)
348
411
 
412
+ def merge_manifest_v2(self):
413
+ comps = []
414
+ for f in os.listdir(self.subsys_dir):
415
+ with open(os.path.join(self.subsys_dir, f)) as fp:
416
+ yml = yaml.safe_load(fp)
417
+ deps = yml.get('dependencies')
418
+ for dep in deps:
419
+ conan = dep.get(misc.CONAN)
420
+ if conan:
421
+ comps.append(conan)
422
+ self.debug("依赖列表: {}".format(comps))
423
+ # 重建新的依赖关系
424
+ # 从单板目录manifest.yml,与subsys/<stage>目录下的组件合并
425
+ new_fd = os.fdopen(os.open("manifest.yml", os.O_WRONLY | os.O_CREAT,
426
+ stat.S_IWUSR | stat.S_IRUSR), 'w')
427
+ new_fd.write("base:\n")
428
+ ver = self.get_manufacture_config("base/version")
429
+
430
+ new_fd.write(f" version: \"{ver}\"\n")
431
+ self.info(f"包版本号: {ver}")
432
+
433
+ new_fd.write("dependencies:\n")
434
+ deps = self._get_dependencies_pkg()
435
+
436
+ # 如果是打包,依赖信息调整
437
+ deps = self.package_dependencies_parse(deps)
438
+ # 由于manifest.yml当中有对于此的新的配置,此处将配置读出,并重新分配
439
+ skynet_with_enable_luajit = False
440
+ # 获取manifest.yml中"base/sdk"字段值
441
+ openubmcsdk = self.get_manufacture_config("base/sdk", "")
442
+ if openubmcsdk:
443
+ self.add_new_dependencies(openubmcsdk, new_fd)
444
+ self.component_check = ComponentVersionCheck(
445
+ manifest_yml="manifest.yml",
446
+ ibmc_lock=IBMC_LOCK_FILE,
447
+ community_name="openubmc",
448
+ openubmcsdk=openubmcsdk
449
+ )
450
+ for dep in deps:
451
+ conan = dep.get(misc.CONAN)
452
+ if not conan:
453
+ continue
454
+ if conan.find("@") > 0:
455
+ self.add_new_dependencies(conan.lower(), new_fd)
456
+ else:
457
+ self.find_conan_package_and_write(conan, comps, new_fd, self.config.stage)
458
+ options = dep.get("options", {})
459
+ name = conan.split("/")[0]
460
+ for key, val in options.items():
461
+ # skynet特殊处理:manifest.yml指定enable_luajit特性时需要覆盖用户输入
462
+ if name == "skynet" and key == "enable_luajit":
463
+ self.warning(f"根据manifest.yml配置,当前产品的enable_luajit配置为{val},忽略命令行指定的-jit参数")
464
+ skynet_with_enable_luajit = True
465
+ self.config.set_enable_luajit(val)
466
+ self.board_option = self.board_option + " -o */*:{}={}".format(key, val)
467
+ else:
468
+ self.board_option = self.board_option + " -o {}/*:{}={}".format(name, key, val)
469
+ # 当使能Luajit又未向skynet传递enable_luajit配置项时需要添加使能参数
470
+ if not skynet_with_enable_luajit and self.config.enable_luajit:
471
+ self.board_option += f" -o */*:enable_luajit={self.config.enable_luajit}"
472
+ new_fd.close()
473
+
349
474
  def merge_manifest(self):
350
475
  comps = []
351
476
  for f in os.listdir(self.subsys_dir):
@@ -364,7 +489,7 @@ class TaskClass(Task):
364
489
  new_fd.write("base:\n")
365
490
  ver = self.get_manufacture_config("base/version")
366
491
 
367
- new_fd.write(f" version: \"{ver}@{tools.conan_user}/{misc.StageEnum.STAGE_RC.value}\"\n")
492
+ new_fd.write(f" version: \"{ver}@{misc.conan_user()}/{misc.StageEnum.STAGE_RC.value}\"\n")
368
493
  self.info(f"包版本号: {ver}")
369
494
 
370
495
  new_fd.write("dependencies:\n")
@@ -374,6 +499,16 @@ class TaskClass(Task):
374
499
  deps = self.package_dependencies_parse(deps)
375
500
  # 由于manifest.yml当中有对于此的新的配置,此处将配置读出,并重新分配
376
501
  skynet_with_enable_luajit = False
502
+ # 获取manifest.yml中定义"base/sdk"字段值
503
+ openubmcsdk = self.get_manufacture_config("base/sdk", "")
504
+ if openubmcsdk:
505
+ self.add_new_dependencies(openubmcsdk, new_fd)
506
+ self.component_check = ComponentVersionCheck(
507
+ manifest_yml="manifest.yml",
508
+ ibmc_lock=IBMC_LOCK_FILE,
509
+ community_name="openubmc",
510
+ openubmcsdk=openubmcsdk
511
+ )
377
512
  for dep in deps:
378
513
  conan = dep.get(misc.CONAN)
379
514
  if not conan:
@@ -384,7 +519,7 @@ class TaskClass(Task):
384
519
  stage = self.config.stage
385
520
  if stage != misc.StageEnum.STAGE_STABLE.value:
386
521
  stage = misc.StageEnum.STAGE_RC.value
387
- conan += f"@{tools.conan_user}/{stage}"
522
+ conan += f"@{misc.conan_user()}/{stage}"
388
523
  self.add_new_dependencies(conan, new_fd)
389
524
  else:
390
525
  self.find_conan_package_and_write(conan, comps, new_fd, self.config.stage)
@@ -400,17 +535,26 @@ class TaskClass(Task):
400
535
  # 当使能Luajit又未向skynet传递enable_luajit配置项时需要添加使能参数
401
536
  if not skynet_with_enable_luajit and self.config.enable_luajit:
402
537
  self.board_option += f" -o skynet:enable_luajit={self.config.enable_luajit}"
403
- self.merge_0502_default_options()
404
538
  sha256 = Tools.sha256sum(SDK_ROOT_MODULE_SYMVERS)
405
539
  self.board_option += " -o *:module_symvers={}".format(sha256)
406
540
  new_fd.close()
407
541
 
542
+ def merge_dep_options(self):
543
+ self.merge_0502_default_options()
544
+
408
545
  def merge_0502_default_options(self):
546
+ if misc.conan_v1():
547
+ prefix = "*"
548
+ else:
549
+ prefix = "*/*"
409
550
  if self.config.manufacture_code:
410
551
  default_options_path = f"manufacture/{self.config.manufacture_code}/default_options"
411
552
  default_options = self.get_manufacture_config(default_options_path, {})
412
553
  for key, val in default_options.items():
413
- self.board_option += f" -o *:{key}={val}"
554
+ self.board_option += f" -o {prefix}:{key}={val}"
555
+ module_symvers_path = os.path.join(SDK_PATH, MODULE_SYMVERS)
556
+ option, value = self.tools.get_hi171x_module_symver_option(module_symvers_path)
557
+ self.board_option += f" -o {prefix}:{option}={value}"
414
558
 
415
559
  def package_info_gen(self, bundle_file, dst_file):
416
560
  with open(bundle_file, "r") as fp:
@@ -419,8 +563,8 @@ class TaskClass(Task):
419
563
  list.sort(require_list)
420
564
  package_info_list = deepcopy(require_list)
421
565
  self.run_command(f"rm -rf {dst_file}", sudo=True)
422
- temp_dst_file = f"{self.config.rootfs_path}/{os.path.basename(dst_file)}"
423
- with os.fdopen(os.open(temp_dst_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, stat.S_IWUSR | stat.S_IRUSR |
566
+ self.package_info = f"{self.config.rootfs_path}/{os.path.basename(dst_file)}"
567
+ with os.fdopen(os.open(self.package_info, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, stat.S_IWUSR | stat.S_IRUSR |
424
568
  stat.S_IWGRP | stat.S_IRGRP | stat.S_IWOTH | stat.S_IROTH), 'w') as fp:
425
569
  for package in require_list:
426
570
  if "openubmc/" not in package:
@@ -428,8 +572,7 @@ class TaskClass(Task):
428
572
  else:
429
573
  package_info_list.remove(package)
430
574
  fp.close()
431
- self.run_command(f"cp -df {temp_dst_file} {dst_file}", sudo=True)
432
- self.run_command(f"rm -rf {temp_dst_file}", sudo=True)
575
+ self.run_command(f"cp -df {self.package_info} {dst_file}", sudo=True)
433
576
 
434
577
  def sensitive_data_conf_gen(self, comps, dst_file):
435
578
  self.run_command(f"rm -rf {dst_file}", sudo=True)
@@ -500,6 +643,78 @@ class TaskClass(Task):
500
643
  options += " -o *:gcov=True"
501
644
  return options
502
645
 
646
+ def proc_sensitive_v2(self, table_type, props, table_name, output):
647
+ for prop_name, prop_data in props.items():
648
+ if prop_data.get("primaryKey"):
649
+ output[PRIMARY_KEYS][table_name] = output[PRIMARY_KEYS].get(table_name, {})
650
+ output[PRIMARY_KEYS][table_name][prop_name] = prop_data.get("sensitive", False)
651
+ continue
652
+ per_type = self.match_persist_type(prop_data.get("usage", [])) or table_type
653
+ if not per_type:
654
+ continue
655
+ output[per_type][table_name] = output[per_type].get(table_name, {})
656
+ output[per_type][table_name][prop_name] = prop_data.get("sensitive", False)
657
+ return output
658
+
659
+ def proc_comp_v2(self, package_folder, output):
660
+ model_file = os.path.join(package_folder, "include", "mds", "model.json")
661
+ if not os.path.isfile(model_file):
662
+ return output
663
+ with open(model_file, "r") as fp:
664
+ content = json.load(fp)
665
+ for class_data in content.values():
666
+ table_type = PERSIST_TYPES.get(class_data.get("tableType", ""))
667
+ table_name = class_data.get("tableName", "")
668
+ if not table_name or class_data.get("tableLocation") == "Local":
669
+ continue
670
+ class_props = [class_data.get("properties", {})]
671
+ for intf_data in class_data.get("interfaces", {}).values():
672
+ class_props.append(intf_data.get("properties", {}))
673
+ for props in class_props:
674
+ output = self.proc_sensitive_v2(table_type, props, table_name, output)
675
+ return output
676
+
677
+ def calc_options_v2(self):
678
+ if self.config.build_type == "debug":
679
+ options = "-s build_type=Debug"
680
+ else:
681
+ options = "-s build_type=Release"
682
+
683
+ if self.config.enable_arm_gcov:
684
+ options += " -o */*:gcov=True"
685
+ return options
686
+
687
+ def install_openubmc_v2(self):
688
+ profile_file = os.path.join(self.tools.conan_profiles_dir, self.config.profile)
689
+ if not os.path.isfile(profile_file):
690
+ raise BmcGoException(f"{profile_file} 文件不存在")
691
+
692
+ options = self.calc_options_v2()
693
+ append_cmd = f"-r {self.config.remote}" if self.config.remote else ""
694
+ # 依据选项生成openubmc.lock和openubmc.bundle文件
695
+ base_cmd = f"-pr={self.config.profile} "
696
+ base_cmd += f"-pr:b profile.dt.ini {append_cmd} {options} {self.board_option}"
697
+ lockfile = os.path.join(self.config.build_path, "conan.lock")
698
+ graphfile = os.path.join(self.config.build_path, "graph.info")
699
+ lock_cmd = f"conan lock create . {base_cmd} --lockfile-out={lockfile}"
700
+ self.run_command(lock_cmd, capture_output=False)
701
+ graph_cmd = f"conan graph info . {base_cmd} -f json --lockfile={lockfile} --out-file={graphfile}"
702
+ self.run_command(graph_cmd)
703
+ self.success(f"start build dependency packages of {self.config.board_name}")
704
+ bcp = BuildConans(graphfile, lockfile, base_cmd, self.config.from_source, self.config.log_path)
705
+ bcp.build()
706
+
707
+ cmd = f"conan create . {base_cmd} --build=missing"
708
+ self.info(f"start build {self.config.board_name}: {cmd}")
709
+ self.run_command(cmd, capture_output=False)
710
+
711
+ self.graphfile = graphfile
712
+
713
+ shutil.copyfile(lockfile, f"{self.conan_install}/conan.lock")
714
+ # 检查使用到的组件是否都在单板目录 manifest.yml 中配置了
715
+ self.component_check.run()
716
+ self.clean_luac_out()
717
+
503
718
  def install_ibmc(self):
504
719
  profile_file = os.path.join(self.tools.conan_profiles_dir, self.config.profile)
505
720
  if not os.path.isfile(profile_file):
@@ -539,21 +754,29 @@ class TaskClass(Task):
539
754
  self.run_command(install_cmd, command_echo=True)
540
755
  shutil.copyfile(IBMC_LOCK_FILE, f"{self.conan_install}/conan.lock")
541
756
  # 检查使用到的组件是否都在单板目录 manifest.yml 中配置了
542
- component_check = ComponentVersionCheck(manifest_yml="manifest.yml", ibmc_lock=IBMC_LOCK_FILE)
543
- component_check.run()
757
+ self.component_check.run()
544
758
  self.clean_luac_out()
545
759
 
546
760
  def deploy(self):
547
- src = os.path.join(self.config.code_path, "conan_index/openubmc")
548
- shutil.copytree(src, self.conan_source, dirs_exist_ok=True)
761
+ conanfile_dir = os.path.join(cwd_dir, "conan")
549
762
  openubmc_dir = os.path.join(self.conan_source, "all")
763
+ os.makedirs(openubmc_dir, exist_ok=True)
764
+ shutil.copytree(conanfile_dir, openubmc_dir, dirs_exist_ok=True)
550
765
  self.chdir(openubmc_dir)
551
766
  # 复制manifest.yml文件
552
- self.merge_manifest()
767
+ if misc.conan_v2():
768
+ self.merge_manifest_v2()
769
+ else:
770
+ self.merge_manifest()
771
+ if self.only_manifest_yml:
772
+ return
773
+ if not self.skip_install_comp:
774
+ self.merge_dep_options()
553
775
  # 下载组件构建脚本
554
776
  ComponentHelper.download_recipes(self.depdencies, self.tools, self.config.remote_list)
555
777
  # 下载skynet
556
- self.install_luac_or_luajit()
778
+ if misc.conan_v1():
779
+ self.install_luac_or_luajit()
557
780
 
558
781
  # 复制全局定制rootfs到conan install目录
559
782
  top_rootfs = os.path.join(self.config.code_path, "rootfs")
@@ -578,7 +801,10 @@ class TaskClass(Task):
578
801
  if os.path.isfile(per_file):
579
802
  shutil.copy(per_file, self.openubmc_ins_dir)
580
803
 
581
- self.install_ibmc()
804
+ if misc.conan_v2():
805
+ self.install_openubmc_v2()
806
+ else:
807
+ self.install_ibmc()
582
808
 
583
809
  def clean_luac_out(self):
584
810
  # 清理冗余文件luac.out
@@ -600,15 +826,15 @@ class TaskClass(Task):
600
826
  self.run_command(f'cp -f conan.lock {os.path.join(inner_path, f"package_{self.config.board_name}.lock")}')
601
827
  self.run_command(f'cp -f conan.lock {os.path.join(self.config.output_path, f"package.lock")}')
602
828
 
603
- def copy_components(self, comps: list, profile: Profile):
829
+ def copy_components(self, comps: list, profile):
604
830
  # 优先处理rootfs
605
- p = CopyComponent(self, "rootfs", profile)
831
+ p = CopyComponent(self, "rootfs", os.path.join(self.conan_install, "rootfs"), profile)
606
832
  p.run()
607
833
  # 打印组件清单
608
834
  self.info(f"组件列表: {comps}")
609
835
  pools = []
610
836
  for comp in comps:
611
- p = CopyComponent(self, comp, profile)
837
+ p = CopyComponent(self, comp, os.path.join(self.conan_install, comp), profile)
612
838
  p.start()
613
839
  pools.append(p)
614
840
 
@@ -621,11 +847,161 @@ class TaskClass(Task):
621
847
  raise errors.BmcGoException(f"复制组件 ({p.comp}) 失败, 退出码: {p.exitcode}")
622
848
  pools.remove(p)
623
849
  # 最后处理openubmc
624
- p = CopyComponent(self, "openubmc", profile)
850
+ p = CopyComponent(self, misc.community_name(), os.path.join(self.conan_install, misc.community_name()), profile)
851
+ p.run()
852
+
853
+ def package_info_gen_v2(self, bmc_lock, dst_file):
854
+ with open(bmc_lock, "r") as bmc_lock_fp:
855
+ lock_info = json.load(bmc_lock_fp)
856
+ require_list = lock_info.get("requires")
857
+ self.run_command(f"rm -rf {dst_file}", sudo=True)
858
+ self.package_info = f"{self.config.rootfs_path}/{os.path.basename(dst_file)}"
859
+ with os.fdopen(os.open(self.package_info, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, stat.S_IWUSR | stat.S_IRUSR |
860
+ stat.S_IWGRP | stat.S_IRGRP | stat.S_IWOTH | stat.S_IROTH), 'w') as fp:
861
+ for package in require_list:
862
+ fp.write(f"{package}\n")
863
+ fp.close()
864
+ self.run_command(f"cp -df {self.package_info} {dst_file}", sudo=True)
865
+
866
+ def sensitive_data_conf_gen_v2(self, dst_file):
867
+ self.run_command(f"rm -rf {dst_file}", sudo=True)
868
+ temp_dst_file = f"{self.config.rootfs_path}/{os.path.basename(dst_file)}"
869
+ output = {
870
+ "TemporaryPer": {},
871
+ "ResetPer": {},
872
+ "PoweroffPer": {},
873
+ "PermanentPer": {},
874
+ "PrimaryKeys": {}
875
+ }
876
+ for name, node in self.graph_nodes.items():
877
+ try:
878
+ output = self.proc_comp_v2(node.package_folder, output)
879
+ except Exception as e:
880
+ raise errors.BmcGoException(f"分析组件 {name} 的 model.json 失败, 失败信息:{e}")
881
+
882
+ with os.fdopen(os.open(temp_dst_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC,
883
+ stat.S_IWUSR | stat.S_IRUSR), 'w') as fp:
884
+ json.dump(output, fp, sort_keys=True)
885
+ self.run_command(f"mkdir -p {os.path.dirname(dst_file)}", sudo=True)
886
+ self.run_command(f"cp -df {temp_dst_file} {dst_file}", sudo=True)
887
+ self.run_command(f"rm -rf {temp_dst_file}", sudo=True)
888
+
889
+ def get_all_subdirectories(self, dirpath):
890
+ exclude_files = ["permissions.ini", "conaninfo.txt", "conanmanifest.txt"]
891
+ out = []
892
+ for root, _, files in os.walk(dirpath):
893
+ for file in files:
894
+ file = os.path.join(root, file)
895
+ relative_path = os.path.relpath(file, dirpath)
896
+ if not relative_path.startswith("include") and relative_path not in exclude_files:
897
+ out.append(relative_path)
898
+ return out
899
+
900
+ def print_dup_files(self, comps, paths):
901
+ dup_paths = {}
902
+ for comp, files in comps.items():
903
+ for path in paths:
904
+ if path in files:
905
+ dup_paths.setdefault(path, []).append(comp)
906
+
907
+ for path, comp in dup_paths.items():
908
+ self.error(f"{comp} 存在重复文件 {path}")
909
+
910
+ def copy_components_v2(self, profile):
911
+ # 优先处理rootfs
912
+ p = CopyComponent(self, "rootfs", os.path.join(self.conan_install, "rootfs"), profile)
913
+ p.run()
914
+ # 打印组件清单
915
+ self.info(f"组件列表: {self.graph_nodes.keys()}")
916
+ pools = []
917
+ # 记录全部的文件信息
918
+ all_path = []
919
+ # 记录组件有哪些文件
920
+ comp_path = {}
921
+ for name, node in self.graph_nodes.items():
922
+ if name in ["rootfs", misc.community_name()]:
923
+ continue
924
+ file_paths = self.get_all_subdirectories(node.package_folder)
925
+ all_path.extend(file_paths)
926
+ comp_path[name] = file_paths
927
+ p = CopyComponent(self, node.name, node.package_folder, profile)
928
+ p.start()
929
+ pools.append(p)
930
+
931
+ while pools:
932
+ time.sleep(0.01)
933
+ for p in pools:
934
+ if p.is_alive():
935
+ continue
936
+ if p.exitcode is not None and p.exitcode != 0:
937
+ raise errors.BmcGoException(f"复制组件 ({p.name}) 失败, 退出码: {p.exitcode}")
938
+ pools.remove(p)
939
+ counter = Counter(all_path)
940
+ duplicates = {path for path, count in counter.items() if count > 1}
941
+ if len(duplicates):
942
+ self.print_dup_files(comp_path, duplicates)
943
+ raise errors.BmcGoException("请检查组件的打包逻辑,检测到重复文件: " + ".".join(duplicates))
944
+ # 最后处理openubmc
945
+ p = CopyComponent(self, misc.community_name(), os.path.join(self.conan_install, misc.community_name()), profile)
625
946
  p.run()
947
+ self.run_command(f"sudo rm -f {self.config.rootfs_path}/conaninfo.txt")
948
+ self.run_command(f"sudo rm -f {self.config.rootfs_path}/conanmanifest.txt")
949
+
950
+ def update_graph_nodes(self):
951
+ rootfs = GraphNode(None)
952
+ rootfs.name = "rootfs"
953
+ rootfs.package_folder = os.path.join(self.conan_install, "rootfs")
954
+ self.graph_nodes["rootfs"] = rootfs
955
+
956
+ with open(self.graphfile, "r") as fp:
957
+ graph = json.load(fp)
958
+ nodes = graph.get("graph", {}).get("nodes", {})
959
+ for _, info in nodes.items():
960
+ node = GraphNode(info)
961
+ context = info.get("context")
962
+ if context != "host":
963
+ continue
964
+ cmd = f"conan cache path {node.ref}:{node.package_id}"
965
+ node.package_folder = tools.run_command(cmd, capture_output=True).stdout.strip()
966
+ self.graph_nodes[node.name] = node
967
+
968
+ rootfs = GraphNode(None)
969
+ rootfs.name = misc.community_name()
970
+ rootfs.package_folder = os.path.join(self.conan_install, misc.community_name())
971
+ self.graph_nodes[misc.community_name()] = rootfs
972
+
973
+ def make_customization_symlinks(self):
974
+ if os.path.isdir(self.customization_dir):
975
+ self.run_command(f"rm -rf {self.customization_dir}")
976
+ self.run_command(f"mkdir -p {self.customization_dir}")
977
+ for name, node in self.graph_nodes.items():
978
+ cust = os.path.join(node.package_folder, "include", "customization.py")
979
+ if not os.path.isfile(cust):
980
+ continue
981
+ os.symlink(node.package_folder, os.path.join(self.customization_dir, name))
982
+
983
+ def package_v2(self):
984
+ self.update_graph_nodes()
985
+ self.make_customization_symlinks()
986
+ self.run_command(f"sudo rm -rf {self.config.rootfs_path}")
987
+ os.makedirs(self.config.rootfs_path)
988
+ profile, _ = self.get_profile_config()
989
+ self.copy_components_v2(profile)
990
+
991
+ self.chdir(self.config.rootfs_path)
992
+ self._component_cust_action("post_image")
993
+
994
+ self.chdir(self.config.rootfs_path)
995
+ self.package_info_gen_v2(f"{self.conan_install}/conan.lock", "etc/package_info")
996
+ self.sensitive_data_conf_gen_v2("opt/bmc/trust/sensitive_data.json")
997
+ if os.path.isfile("permissions.ini"):
998
+ os.unlink("permissions.ini")
626
999
 
627
1000
  def package(self):
628
1001
  self.chdir(self.conan_install)
1002
+ if misc.conan_v2():
1003
+ self.package_v2()
1004
+ return
629
1005
  self.run_command(f"sudo rm -rf {self.config.rootfs_path}")
630
1006
  os.makedirs(self.config.rootfs_path)
631
1007
  profile, _ = self.get_profile_config()
@@ -638,14 +1014,8 @@ class TaskClass(Task):
638
1014
 
639
1015
  self.copy_components(comps, profile)
640
1016
 
641
- # 运行组件定制化文件
642
- for comp in comps:
643
- cust = os.path.join(self.conan_install, comp, "include", "customization.py")
644
- if not os.path.isfile(cust):
645
- continue
646
- self.info(f"开始执行 {comp}/include/customization.py")
647
- post = ComponentPost(self.config, os.path.join(self.conan_install, comp), profile)
648
- post.post_work(self.config.rootfs_path, "post_image")
1017
+ self.chdir(self.config.rootfs_path)
1018
+ self._component_cust_action("post_image")
649
1019
 
650
1020
  self.chdir(self.config.rootfs_path)
651
1021
  self.package_info_gen(f"{self.conan_source}/all/openubmc.bundle", "etc/package_info")
@@ -721,6 +1091,8 @@ class TaskClass(Task):
721
1091
  self.mkdir_work_path()
722
1092
  self.tools.clean_locks()
723
1093
  self.deploy()
1094
+ if self.only_manifest_yml:
1095
+ return
724
1096
  self.package_lock()
725
1097
  if not self.skip_package:
726
1098
  self.package()