openubmc-bingo 0.5.277__py3-none-any.whl → 0.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of openubmc-bingo might be problematic. Click here for more details.

Files changed (52) hide show
  1. bmcgo/__init__.py +1 -1
  2. bmcgo/bmcgo_config.py +22 -10
  3. bmcgo/cli/cli.py +86 -39
  4. bmcgo/cli/config.conan2.yaml +9 -0
  5. bmcgo/codegen/lua/codegen.py +1 -1
  6. bmcgo/codegen/lua/script/gen_intf_rpc_json.py +15 -14
  7. bmcgo/component/analysis/analysis.py +8 -8
  8. bmcgo/component/analysis/intf_validation.py +23 -12
  9. bmcgo/component/build.py +76 -14
  10. bmcgo/component/component_dt_version_parse.py +3 -2
  11. bmcgo/component/component_helper.py +43 -15
  12. bmcgo/component/coverage/incremental_cov.py +2 -2
  13. bmcgo/component/deploy.py +68 -14
  14. bmcgo/component/gen.py +1 -1
  15. bmcgo/component/package_info.py +128 -38
  16. bmcgo/component/template_v2/conanbase.py.mako +352 -0
  17. bmcgo/component/template_v2/conanfile.deploy.py.mako +26 -0
  18. bmcgo/component/test.py +53 -20
  19. bmcgo/frame.py +7 -3
  20. bmcgo/functional/analysis.py +3 -2
  21. bmcgo/functional/check.py +10 -6
  22. bmcgo/functional/conan_index_build.py +79 -20
  23. bmcgo/functional/csr_build.py +1 -1
  24. bmcgo/functional/diff.py +1 -1
  25. bmcgo/functional/fetch.py +1 -1
  26. bmcgo/functional/full_component.py +32 -24
  27. bmcgo/functional/git_history.py +220 -0
  28. bmcgo/functional/maintain.py +55 -12
  29. bmcgo/functional/new.py +1 -1
  30. bmcgo/functional/schema_valid.py +2 -2
  31. bmcgo/logger.py +2 -3
  32. bmcgo/misc.py +130 -9
  33. bmcgo/tasks/conan/__init__.py +10 -0
  34. bmcgo/tasks/conan/conanfile.py +45 -0
  35. bmcgo/tasks/task.py +27 -4
  36. bmcgo/tasks/task_build_conan.py +399 -52
  37. bmcgo/tasks/task_buildgppbin.py +8 -2
  38. bmcgo/tasks/task_download_buildtools.py +76 -0
  39. bmcgo/tasks/task_download_dependency.py +1 -0
  40. bmcgo/tasks/task_hpm_envir_prepare.py +1 -1
  41. bmcgo/utils/build_conans.py +231 -0
  42. bmcgo/utils/component_post.py +6 -4
  43. bmcgo/utils/component_version_check.py +10 -5
  44. bmcgo/utils/config.py +76 -40
  45. bmcgo/utils/fetch_component_code.py +44 -25
  46. bmcgo/utils/tools.py +239 -117
  47. bmcgo/worker.py +2 -2
  48. {openubmc_bingo-0.5.277.dist-info → openubmc_bingo-0.6.0.dist-info}/METADATA +4 -2
  49. {openubmc_bingo-0.5.277.dist-info → openubmc_bingo-0.6.0.dist-info}/RECORD +52 -45
  50. {openubmc_bingo-0.5.277.dist-info → openubmc_bingo-0.6.0.dist-info}/WHEEL +0 -0
  51. {openubmc_bingo-0.5.277.dist-info → openubmc_bingo-0.6.0.dist-info}/entry_points.txt +0 -0
  52. {openubmc_bingo-0.5.277.dist-info → openubmc_bingo-0.6.0.dist-info}/top_level.txt +0 -0
@@ -27,10 +27,10 @@ import random
27
27
  import pathlib
28
28
  from multiprocessing import Process, Queue
29
29
  from copy import deepcopy
30
+ from collections import Counter
30
31
 
31
32
  import yaml
32
33
  from colorama import Fore, Back, Style
33
- from conans.model.profile import Profile
34
34
 
35
35
  from bmcgo.tasks.task import Task
36
36
  from bmcgo.utils.config import Config
@@ -40,6 +40,9 @@ from bmcgo import errors, misc
40
40
  from bmcgo.utils.tools import Tools
41
41
  from bmcgo.errors import BmcGoException
42
42
  from bmcgo.component.component_helper import ComponentHelper
43
+ from bmcgo.utils.build_conans import BuildConans
44
+ from bmcgo.component.deploy import GraphNode
45
+ from bmcgo.tasks.misc import MODULE_SYMVERS, SDK_PATH
43
46
 
44
47
  IBMC_LOCK_FILE = "openubmc.lock"
45
48
  SDK_ROOT = "/opt/hi1711sdk"
@@ -56,14 +59,30 @@ PERSIST_TYPES = {
56
59
  PRIMARY_KEYS = "PrimaryKeys"
57
60
  tools = Tools()
58
61
 
62
+ PERSIST_TYPES = {
63
+ "TemporaryPer": "TemporaryPer",
64
+ "TemporaryPerRetain": "TemporaryPer",
65
+ "ResetPer": "ResetPer",
66
+ "ResetPerRetain": "ResetPer",
67
+ "PoweroffPer": "PoweroffPer",
68
+ "PoweroffPerRetain": "PoweroffPer",
69
+ "PermanentPer": "PermanentPer"
70
+ }
71
+
72
+ PRIMARY_KEYS = "PrimaryKeys"
73
+
74
+ cwd_dir = os.path.dirname(os.path.realpath(__file__))
75
+
59
76
 
60
77
  class CopyComponent(Process):
61
- def __init__(self, work: Task, comp: str, profile: Profile):
78
+ def __init__(self, work: Task, comp: str, package_folder: str, profile):
62
79
  super().__init__()
63
80
  self.work = work
64
81
  self.config = self.work.config
65
82
  self.comp = comp
83
+ self.package_folder = package_folder
66
84
  self.profile = profile
85
+ self.graphfile = None
67
86
 
68
87
  def link_recursive_deal(self, file_name, ownership):
69
88
  self.work.run_command(f"chown -h {ownership} {file_name}", sudo=True)
@@ -74,16 +93,16 @@ class CopyComponent(Process):
74
93
  self.work.work_name = self.comp
75
94
  # 复制组件文件到rootfs中
76
95
  # rc.sysinit脚本只允许使用rootfs_user仓库的
77
- rc_sysinit_path = os.path.join(self.comp, "etc/rc.d/rc.sysinit")
96
+ rc_sysinit_path = os.path.join(self.package_folder, "etc/rc.d/rc.sysinit")
78
97
  if self.comp != "rootfs_user" and os.path.isfile(rc_sysinit_path):
79
98
  shutil.rmtree(rc_sysinit_path, ignore_errors=True)
80
99
  # 由于复制时有很多同名路径,cp命令有概率失败,10次复制尝试
81
100
  self._copy_files()
82
101
 
83
102
  # 执行权限控制逻辑
84
- per_cfg = os.path.join(self.comp, "permissions.ini")
103
+ per_cfg = os.path.join(self.package_folder, "permissions.ini")
85
104
  if not os.path.isfile(per_cfg):
86
- self.work.warning("权限文件 {} 不存在, 所以无法设置组件 {} 的权限".format(per_cfg, self.comp))
105
+ self.work.warning("权限文件 {} 不存在, 所以无法设置组件 {} 的权限".format(per_cfg, self.package_folder))
87
106
  return
88
107
  self.work.info("依据配置文件: {}, 修改权限".format(per_cfg))
89
108
  # NOTE 这里的配置时间, 偶发复制未完成, 就开始修改权限
@@ -130,7 +149,7 @@ class CopyComponent(Process):
130
149
 
131
150
  def _copy_files(self):
132
151
  # 由于复制时有很多同名路径,cp命令有概率失败,10次复制尝试
133
- cmd = "sudo cp -dfr {}/. {}".format(self.comp, self.config.rootfs_path)
152
+ cmd = "sudo cp -dfr {}/. {}".format(self.package_folder, self.config.rootfs_path)
134
153
  for _ in range(0, 10):
135
154
  try:
136
155
  # 尝试复制到目标目录,成功则break退出循环
@@ -165,22 +184,33 @@ class ConanLockParse:
165
184
  self._process_list = []
166
185
 
167
186
  def conan_install(self, cmd, com):
168
- real_cmd = shlex.split(cmd)
169
- pipe = subprocess.Popen(real_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
170
- # 超时时长1800s
171
- out, _ = pipe.communicate(timeout=1800)
172
- if pipe.returncode != 0:
173
- self.work.error(f"================== {com} 构建失败日志起始位置 ==================")
174
- self.work.info(out.decode())
175
- self.work.error(f"================== {com} 构建失败日志结束位置 ==================")
176
- self._queue.put(self.error_str)
177
- self._queue.put(self.error_str)
178
- self._queue.put(self.error_str)
179
- return
180
- self._queue.put(com)
181
- self._queue.put(com)
182
- self._queue.put(com)
183
- self.work.info(f"命令 {cmd} " + Fore.GREEN + "执行完成" + Style.RESET_ALL)
187
+ com_name = com[:com.index("/")]
188
+ com_install_log = os.path.join(self.work.config.temp_path, f"log/conan_install/{com_name}.log")
189
+ with os.fdopen(os.open(com_install_log, os.O_RDWR | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), "a+") as f:
190
+ self.work.info(f"命令 {cmd} " + Fore.GREEN + "开始" + Style.RESET_ALL)
191
+ real_cmd = shlex.split(cmd)
192
+ pipe = subprocess.Popen(real_cmd, stdout=f, stderr=f)
193
+ start_time = time.time()
194
+ while True:
195
+ if pipe.poll() is not None:
196
+ break
197
+ # 1800秒超时
198
+ if time.time() - start_time > 1800:
199
+ pipe.kill()
200
+ self.work.error(f"================== {com} 构建超时 ==================")
201
+ self._queue.put(self.error_str)
202
+ return
203
+ time.sleep(1)
204
+ if pipe.returncode != 0:
205
+ self.work.error(f"================== {com} 构建失败日志起始位置 ==================")
206
+ f.seek(0)
207
+ conan_log = f.read()
208
+ self.work.info(conan_log)
209
+ self.work.error(f"================== {com} 构建失败日志结束位置 ==================")
210
+ self._queue.put(self.error_str)
211
+ return
212
+ self._queue.put(com)
213
+ self.work.info(f"命令 {cmd} " + Fore.GREEN + "执行完成" + Style.RESET_ALL)
184
214
 
185
215
  def wait_task_finished(self):
186
216
  # 所有入度为0任务拉起后等待完成消息
@@ -210,6 +240,9 @@ class ConanLockParse:
210
240
  self._degree = {x: len(self._bundle[x].get('requires')) if self._bundle[x].get('requires') is not None else 0
211
241
  for x in self._bundle.keys()}
212
242
 
243
+ conan_log_path = os.path.join(self.work.config.temp_path, "log/conan_install")
244
+ os.makedirs(conan_log_path, exist_ok=True)
245
+
213
246
  self.work.success(">>>>>>>>>>>>>>>>> 开始构建所有组件 <<<<<<<<<<<<<<<<")
214
247
  self.work.info("这些进程将要花费几分钟(2-3分钟), 请耐心等待")
215
248
  while self._degree:
@@ -226,7 +259,6 @@ class ConanLockParse:
226
259
  t = Process(target=self.conan_install, args=(cmd_tmp, com))
227
260
  t.start()
228
261
  self._process_list.append(t)
229
- self.work.info(f"命令 {cmd_tmp} " + Fore.GREEN + "开始" + Style.RESET_ALL)
230
262
  # 入度为0的任务构建拉起后弹出
231
263
  self._degree.pop(com)
232
264
  else:
@@ -243,7 +275,11 @@ class TaskClass(Task):
243
275
 
244
276
  def __init__(self, config: Config, work_name=""):
245
277
  super(TaskClass, self).__init__(config, work_name)
278
+ self.skip_install_comp = False
279
+ self.only_manifest_yml = False
246
280
  self.update_path()
281
+ # 记录每个组件的GraphNode信息
282
+ self.package_info = os.path.join(self.config.rootfs_path, "package_info")
247
283
 
248
284
  @property
249
285
  def subsys_dir(self):
@@ -261,6 +297,13 @@ class TaskClass(Task):
261
297
  return PERSIST_TYPES[item]
262
298
  return None
263
299
 
300
+ @staticmethod
301
+ def match_persist_type(usage):
302
+ for item in usage:
303
+ if item in PERSIST_TYPES:
304
+ return PERSIST_TYPES[item]
305
+ return None
306
+
264
307
  def find_conan_package_and_write(self, search, comps, file_handler, stage):
265
308
  if search.find("/") < 0:
266
309
  search += "/"
@@ -273,18 +316,19 @@ class TaskClass(Task):
273
316
  if stage != misc.StageEnum.STAGE_DEV.value and comp.endswith(f"/{misc.StageEnum.STAGE_DEV.value}"):
274
317
  err_msg = f"组件包 {comp} user/channel 配置错误, 必须以 \"{stage}\" 作为结尾, 终止构建"
275
318
  raise errors.BmcGoException(err_msg)
319
+ if misc.conan_v2():
320
+ comp = comp.lower()
276
321
  self.add_new_dependencies(comp, file_handler)
277
322
  return
278
323
  raise errors.ConfigException(f"未知组件: {search}, 请检查配置 !!!")
279
324
 
280
325
  def update_path(self):
281
326
  # conan source folder
282
- self.conan_source = os.path.join(self.config.temp_path,
283
- f"conan_source_{self.config.build_type}_{self.config.stage}/openubmc")
327
+ self.conan_source = os.path.join(self.config.build_path, misc.community_name())
284
328
  # conan install folder
285
329
  self.conan_install = os.path.join(self.config.build_path, "conan_install")
286
330
  # openubmc install folder
287
- self.openubmc_ins_dir = os.path.join(self.conan_install, "openubmc")
331
+ self.openubmc_ins_dir = os.path.join(self.conan_install, misc.community_name())
288
332
  # openubmc top rootfs folder
289
333
  self.top_rootfs_dir = os.path.join(self.conan_install, "rootfs")
290
334
  self.board_option = ""
@@ -320,6 +364,18 @@ class TaskClass(Task):
320
364
  else:
321
365
  self.skip_package = False
322
366
 
367
+ def set_skip_install_comp(self, value):
368
+ if value:
369
+ self.skip_install_comp = True
370
+ else:
371
+ self.skip_install_comp = False
372
+
373
+ def set_only_manifest_yml(self, value):
374
+ if value:
375
+ self.only_manifest_yml = True
376
+ else:
377
+ self.only_manifest_yml = False
378
+
323
379
  def package_dependencies_parse(self, default_component: list):
324
380
  """解析manufacture或者tosupporte配置,不同的包设置不同组件的不同编译选项
325
381
  参数:
@@ -346,6 +402,58 @@ class TaskClass(Task):
346
402
  self.success(f"获取到依赖: {conan}")
347
403
  self.depdencies.append(conan)
348
404
 
405
+ def merge_manifest_v2(self):
406
+ comps = []
407
+ for f in os.listdir(self.subsys_dir):
408
+ with open(os.path.join(self.subsys_dir, f)) as fp:
409
+ yml = yaml.safe_load(fp)
410
+ deps = yml.get('dependencies')
411
+ for dep in deps:
412
+ conan = dep.get(misc.CONAN)
413
+ if conan:
414
+ comps.append(conan)
415
+ self.debug("依赖列表: {}".format(comps))
416
+ # 重建新的依赖关系
417
+ # 从单板目录manifest.yml,与subsys/<stage>目录下的组件合并
418
+ new_fd = os.fdopen(os.open("manifest.yml", os.O_WRONLY | os.O_CREAT,
419
+ stat.S_IWUSR | stat.S_IRUSR), 'w')
420
+ new_fd.write("base:\n")
421
+ ver = self.get_manufacture_config("base/version")
422
+
423
+ new_fd.write(f" version: \"{ver}\"\n")
424
+ self.info(f"包版本号: {ver}")
425
+
426
+ new_fd.write("dependencies:\n")
427
+ deps = self._get_dependencies_pkg()
428
+
429
+ # 如果是打包,依赖信息调整
430
+ deps = self.package_dependencies_parse(deps)
431
+ # 由于manifest.yml当中有对于此的新的配置,此处将配置读出,并重新分配
432
+ skynet_with_enable_luajit = False
433
+ for dep in deps:
434
+ conan = dep.get(misc.CONAN)
435
+ if not conan:
436
+ continue
437
+ if conan.find("@") > 0:
438
+ self.add_new_dependencies(conan.lower(), new_fd)
439
+ else:
440
+ self.find_conan_package_and_write(conan, comps, new_fd, self.config.stage)
441
+ options = dep.get("options", {})
442
+ name = conan.split("/")[0]
443
+ for key, val in options.items():
444
+ # skynet特殊处理:manifest.yml指定enable_luajit特性时需要覆盖用户输入
445
+ if name == "skynet" and key == "enable_luajit":
446
+ self.warning(f"根据manifest.yml配置,当前产品的enable_luajit配置为{val},忽略命令行指定的-jit参数")
447
+ skynet_with_enable_luajit = True
448
+ self.config.set_enable_luajit(val)
449
+ self.board_option = self.board_option + " -o */*:{}={}".format(key, val)
450
+ else:
451
+ self.board_option = self.board_option + " -o {}/*:{}={}".format(name, key, val)
452
+ # 当使能Luajit又未向skynet传递enable_luajit配置项时需要添加使能参数
453
+ if not skynet_with_enable_luajit and self.config.enable_luajit:
454
+ self.board_option += f" -o */*:enable_luajit={self.config.enable_luajit}"
455
+ new_fd.close()
456
+
349
457
  def merge_manifest(self):
350
458
  comps = []
351
459
  for f in os.listdir(self.subsys_dir):
@@ -364,7 +472,7 @@ class TaskClass(Task):
364
472
  new_fd.write("base:\n")
365
473
  ver = self.get_manufacture_config("base/version")
366
474
 
367
- new_fd.write(f" version: \"{ver}@{tools.conan_user}/{misc.StageEnum.STAGE_RC.value}\"\n")
475
+ new_fd.write(f" version: \"{ver}@{misc.conan_user()}/{misc.StageEnum.STAGE_RC.value}\"\n")
368
476
  self.info(f"包版本号: {ver}")
369
477
 
370
478
  new_fd.write("dependencies:\n")
@@ -384,7 +492,7 @@ class TaskClass(Task):
384
492
  stage = self.config.stage
385
493
  if stage != misc.StageEnum.STAGE_STABLE.value:
386
494
  stage = misc.StageEnum.STAGE_RC.value
387
- conan += f"@{tools.conan_user}/{stage}"
495
+ conan += f"@{misc.conan_user()}/{stage}"
388
496
  self.add_new_dependencies(conan, new_fd)
389
497
  else:
390
498
  self.find_conan_package_and_write(conan, comps, new_fd, self.config.stage)
@@ -400,17 +508,26 @@ class TaskClass(Task):
400
508
  # 当使能Luajit又未向skynet传递enable_luajit配置项时需要添加使能参数
401
509
  if not skynet_with_enable_luajit and self.config.enable_luajit:
402
510
  self.board_option += f" -o skynet:enable_luajit={self.config.enable_luajit}"
403
- self.merge_0502_default_options()
404
511
  sha256 = Tools.sha256sum(SDK_ROOT_MODULE_SYMVERS)
405
512
  self.board_option += " -o *:module_symvers={}".format(sha256)
406
513
  new_fd.close()
407
514
 
515
+ def merge_dep_options(self):
516
+ self.merge_0502_default_options()
517
+
408
518
  def merge_0502_default_options(self):
519
+ if misc.conan_v1():
520
+ prefix = "*"
521
+ else:
522
+ prefix = "*/*"
409
523
  if self.config.manufacture_code:
410
524
  default_options_path = f"manufacture/{self.config.manufacture_code}/default_options"
411
525
  default_options = self.get_manufacture_config(default_options_path, {})
412
526
  for key, val in default_options.items():
413
- self.board_option += f" -o *:{key}={val}"
527
+ self.board_option += f" -o {prefix}:{key}={val}"
528
+ module_symvers_path = os.path.join(SDK_PATH, MODULE_SYMVERS)
529
+ option, value = self.tools.get_hi171x_module_symver_option(module_symvers_path)
530
+ self.board_option += f" -o {prefix}:{option}={value}"
414
531
 
415
532
  def package_info_gen(self, bundle_file, dst_file):
416
533
  with open(bundle_file, "r") as fp:
@@ -419,8 +536,8 @@ class TaskClass(Task):
419
536
  list.sort(require_list)
420
537
  package_info_list = deepcopy(require_list)
421
538
  self.run_command(f"rm -rf {dst_file}", sudo=True)
422
- temp_dst_file = f"{self.config.rootfs_path}/{os.path.basename(dst_file)}"
423
- with os.fdopen(os.open(temp_dst_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, stat.S_IWUSR | stat.S_IRUSR |
539
+ self.package_info = f"{self.config.rootfs_path}/{os.path.basename(dst_file)}"
540
+ with os.fdopen(os.open(self.package_info, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, stat.S_IWUSR | stat.S_IRUSR |
424
541
  stat.S_IWGRP | stat.S_IRGRP | stat.S_IWOTH | stat.S_IROTH), 'w') as fp:
425
542
  for package in require_list:
426
543
  if "openubmc/" not in package:
@@ -428,8 +545,7 @@ class TaskClass(Task):
428
545
  else:
429
546
  package_info_list.remove(package)
430
547
  fp.close()
431
- self.run_command(f"cp -df {temp_dst_file} {dst_file}", sudo=True)
432
- self.run_command(f"rm -rf {temp_dst_file}", sudo=True)
548
+ self.run_command(f"cp -df {self.package_info} {dst_file}", sudo=True)
433
549
 
434
550
  def sensitive_data_conf_gen(self, comps, dst_file):
435
551
  self.run_command(f"rm -rf {dst_file}", sudo=True)
@@ -500,6 +616,79 @@ class TaskClass(Task):
500
616
  options += " -o *:gcov=True"
501
617
  return options
502
618
 
619
+ def proc_sensitive_v2(self, table_type, props, table_name, output):
620
+ for prop_name, prop_data in props.items():
621
+ if prop_data.get("primaryKey"):
622
+ output[PRIMARY_KEYS][table_name] = output[PRIMARY_KEYS].get(table_name, {})
623
+ output[PRIMARY_KEYS][table_name][prop_name] = prop_data.get("sensitive", False)
624
+ continue
625
+ per_type = self.match_persist_type(prop_data.get("usage", [])) or table_type
626
+ if not per_type:
627
+ continue
628
+ output[per_type][table_name] = output[per_type].get(table_name, {})
629
+ output[per_type][table_name][prop_name] = prop_data.get("sensitive", False)
630
+ return output
631
+
632
+ def proc_comp_v2(self, package_folder, output):
633
+ model_file = os.path.join(package_folder, "include", "mds", "model.json")
634
+ if not os.path.isfile(model_file):
635
+ return output
636
+ with open(model_file, "r") as fp:
637
+ content = json.load(fp)
638
+ for class_data in content.values():
639
+ table_type = PERSIST_TYPES.get(class_data.get("tableType", ""))
640
+ table_name = class_data.get("tableName", "")
641
+ if not table_name or class_data.get("tableLocation") == "Local":
642
+ continue
643
+ class_props = [class_data.get("properties", {})]
644
+ for intf_data in class_data.get("interfaces", {}).values():
645
+ class_props.append(intf_data.get("properties", {}))
646
+ for props in class_props:
647
+ output = self.proc_sensitive_v2(table_type, props, table_name, output)
648
+ return output
649
+
650
+ def calc_options_v2(self):
651
+ if self.config.build_type == "debug":
652
+ options = "-s build_type=Debug"
653
+ else:
654
+ options = "-s build_type=Release"
655
+
656
+ if self.config.enable_arm_gcov:
657
+ options += " -o */*:gcov=True"
658
+ return options
659
+
660
+ def install_openubmc_v2(self):
661
+ profile_file = os.path.join(self.tools.conan_profiles_dir, self.config.profile)
662
+ if not os.path.isfile(profile_file):
663
+ raise BmcGoException(f"{profile_file} 文件不存在")
664
+
665
+ options = self.calc_options_v2()
666
+ append_cmd = f"-r {self.config.remote}" if self.config.remote else ""
667
+ # 依据选项生成openubmc.lock和openubmc.bundle文件
668
+ base_cmd = f"-pr={self.config.profile} "
669
+ base_cmd += f"-pr:b profile.dt.ini {append_cmd} {options} {self.board_option}"
670
+ lockfile = os.path.join(self.config.build_path, "conan.lock")
671
+ graphfile = os.path.join(self.config.build_path, "graph.info")
672
+ lock_cmd = f"conan lock create . {base_cmd} --lockfile-out={lockfile}"
673
+ self.run_command(lock_cmd, capture_output=False)
674
+ graph_cmd = f"conan graph info . {base_cmd} -f json --lockfile={lockfile} --out-file={graphfile}"
675
+ self.run_command(graph_cmd)
676
+ self.success(f"start build dependency packages of {self.config.board_name}")
677
+ bcp = BuildConans(graphfile, lockfile, base_cmd, self.config.from_source, self.config.log_path)
678
+ bcp.build()
679
+
680
+ cmd = f"conan create . {base_cmd} --build=missing"
681
+ self.info(f"start build {self.config.board_name}: {cmd}")
682
+ self.run_command(cmd, capture_output=False)
683
+
684
+ self.graphfile = graphfile
685
+
686
+ shutil.copyfile(lockfile, f"{self.conan_install}/conan.lock")
687
+ # 检查使用到的组件是否都在单板目录 manifest.yml 中配置了
688
+ component_check = ComponentVersionCheck("manifest.yml", lockfile, "openubmc")
689
+ component_check.run()
690
+ self.clean_luac_out()
691
+
503
692
  def install_ibmc(self):
504
693
  profile_file = os.path.join(self.tools.conan_profiles_dir, self.config.profile)
505
694
  if not os.path.isfile(profile_file):
@@ -539,21 +728,30 @@ class TaskClass(Task):
539
728
  self.run_command(install_cmd, command_echo=True)
540
729
  shutil.copyfile(IBMC_LOCK_FILE, f"{self.conan_install}/conan.lock")
541
730
  # 检查使用到的组件是否都在单板目录 manifest.yml 中配置了
542
- component_check = ComponentVersionCheck(manifest_yml="manifest.yml", ibmc_lock=IBMC_LOCK_FILE)
731
+ component_check = ComponentVersionCheck("manifest.yml", IBMC_LOCK_FILE, "openubmc")
543
732
  component_check.run()
544
733
  self.clean_luac_out()
545
734
 
546
735
  def deploy(self):
547
- src = os.path.join(self.config.code_path, "conan_index/openubmc")
548
- shutil.copytree(src, self.conan_source, dirs_exist_ok=True)
736
+ conanfile_dir = os.path.join(cwd_dir, "conan")
549
737
  openubmc_dir = os.path.join(self.conan_source, "all")
738
+ os.makedirs(openubmc_dir, exist_ok=True)
739
+ shutil.copytree(conanfile_dir, openubmc_dir, dirs_exist_ok=True)
550
740
  self.chdir(openubmc_dir)
551
741
  # 复制manifest.yml文件
552
- self.merge_manifest()
742
+ if misc.conan_v2():
743
+ self.merge_manifest_v2()
744
+ else:
745
+ self.merge_manifest()
746
+ if self.only_manifest_yml:
747
+ return
748
+ if not self.skip_install_comp:
749
+ self.merge_dep_options()
553
750
  # 下载组件构建脚本
554
751
  ComponentHelper.download_recipes(self.depdencies, self.tools, self.config.remote_list)
555
752
  # 下载skynet
556
- self.install_luac_or_luajit()
753
+ if misc.conan_v1():
754
+ self.install_luac_or_luajit()
557
755
 
558
756
  # 复制全局定制rootfs到conan install目录
559
757
  top_rootfs = os.path.join(self.config.code_path, "rootfs")
@@ -578,7 +776,10 @@ class TaskClass(Task):
578
776
  if os.path.isfile(per_file):
579
777
  shutil.copy(per_file, self.openubmc_ins_dir)
580
778
 
581
- self.install_ibmc()
779
+ if misc.conan_v2():
780
+ self.install_openubmc_v2()
781
+ else:
782
+ self.install_ibmc()
582
783
 
583
784
  def clean_luac_out(self):
584
785
  # 清理冗余文件luac.out
@@ -600,15 +801,15 @@ class TaskClass(Task):
600
801
  self.run_command(f'cp -f conan.lock {os.path.join(inner_path, f"package_{self.config.board_name}.lock")}')
601
802
  self.run_command(f'cp -f conan.lock {os.path.join(self.config.output_path, f"package.lock")}')
602
803
 
603
- def copy_components(self, comps: list, profile: Profile):
804
+ def copy_components(self, comps: list, profile):
604
805
  # 优先处理rootfs
605
- p = CopyComponent(self, "rootfs", profile)
806
+ p = CopyComponent(self, "rootfs", os.path.join(self.conan_install, "rootfs"), profile)
606
807
  p.run()
607
808
  # 打印组件清单
608
809
  self.info(f"组件列表: {comps}")
609
810
  pools = []
610
811
  for comp in comps:
611
- p = CopyComponent(self, comp, profile)
812
+ p = CopyComponent(self, comp, os.path.join(self.conan_install, comp), profile)
612
813
  p.start()
613
814
  pools.append(p)
614
815
 
@@ -621,11 +822,161 @@ class TaskClass(Task):
621
822
  raise errors.BmcGoException(f"复制组件 ({p.comp}) 失败, 退出码: {p.exitcode}")
622
823
  pools.remove(p)
623
824
  # 最后处理openubmc
624
- p = CopyComponent(self, "openubmc", profile)
825
+ p = CopyComponent(self, misc.community_name(), os.path.join(self.conan_install, misc.community_name()), profile)
826
+ p.run()
827
+
828
+ def package_info_gen_v2(self, bmc_lock, dst_file):
829
+ with open(bmc_lock, "r") as bmc_lock_fp:
830
+ lock_info = json.load(bmc_lock_fp)
831
+ require_list = lock_info.get("requires")
832
+ self.run_command(f"rm -rf {dst_file}", sudo=True)
833
+ self.package_info = f"{self.config.rootfs_path}/{os.path.basename(dst_file)}"
834
+ with os.fdopen(os.open(self.package_info, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, stat.S_IWUSR | stat.S_IRUSR |
835
+ stat.S_IWGRP | stat.S_IRGRP | stat.S_IWOTH | stat.S_IROTH), 'w') as fp:
836
+ for package in require_list:
837
+ fp.write(f"{package}\n")
838
+ fp.close()
839
+ self.run_command(f"cp -df {self.package_info} {dst_file}", sudo=True)
840
+
841
+ def sensitive_data_conf_gen_v2(self, dst_file):
842
+ self.run_command(f"rm -rf {dst_file}", sudo=True)
843
+ temp_dst_file = f"{self.config.rootfs_path}/{os.path.basename(dst_file)}"
844
+ output = {
845
+ "TemporaryPer": {},
846
+ "ResetPer": {},
847
+ "PoweroffPer": {},
848
+ "PermanentPer": {},
849
+ "PrimaryKeys": {}
850
+ }
851
+ for name, node in self.graph_nodes.items():
852
+ try:
853
+ output = self.proc_comp_v2(node.package_folder, output)
854
+ except Exception as e:
855
+ raise errors.BmcGoException(f"分析组件 {name} 的 model.json 失败, 失败信息:{e}")
856
+
857
+ with os.fdopen(os.open(temp_dst_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC,
858
+ stat.S_IWUSR | stat.S_IRUSR), 'w') as fp:
859
+ json.dump(output, fp, sort_keys=True)
860
+ self.run_command(f"mkdir -p {os.path.dirname(dst_file)}", sudo=True)
861
+ self.run_command(f"cp -df {temp_dst_file} {dst_file}", sudo=True)
862
+ self.run_command(f"rm -rf {temp_dst_file}", sudo=True)
863
+
864
+ def get_all_subdirectories(self, dirpath):
865
+ exclude_files = ["permissions.ini", "conaninfo.txt", "conanmanifest.txt"]
866
+ out = []
867
+ for root, _, files in os.walk(dirpath):
868
+ for file in files:
869
+ file = os.path.join(root, file)
870
+ relative_path = os.path.relpath(file, dirpath)
871
+ if not relative_path.startswith("include") and relative_path not in exclude_files:
872
+ out.append(relative_path)
873
+ return out
874
+
875
+ def print_dup_files(self, comps, paths):
876
+ dup_paths = {}
877
+ for comp, files in comps.items():
878
+ for path in paths:
879
+ if path in files:
880
+ dup_paths.setdefault(path, []).append(comp)
881
+
882
+ for path, comp in dup_paths.items():
883
+ self.error(f"{comp} 存在重复文件 {path}")
884
+
885
+ def copy_components_v2(self, profile):
886
+ # 优先处理rootfs
887
+ p = CopyComponent(self, "rootfs", os.path.join(self.conan_install, "rootfs"), profile)
888
+ p.run()
889
+ # 打印组件清单
890
+ self.info(f"组件列表: {self.graph_nodes.keys()}")
891
+ pools = []
892
+ # 记录全部的文件信息
893
+ all_path = []
894
+ # 记录组件有哪些文件
895
+ comp_path = {}
896
+ for name, node in self.graph_nodes.items():
897
+ if name in ["rootfs", misc.community_name()]:
898
+ continue
899
+ file_paths = self.get_all_subdirectories(node.package_folder)
900
+ all_path.extend(file_paths)
901
+ comp_path[name] = file_paths
902
+ p = CopyComponent(self, node.name, node.package_folder, profile)
903
+ p.start()
904
+ pools.append(p)
905
+
906
+ while pools:
907
+ time.sleep(0.01)
908
+ for p in pools:
909
+ if p.is_alive():
910
+ continue
911
+ if p.exitcode is not None and p.exitcode != 0:
912
+ raise errors.BmcGoException(f"复制组件 ({p.name}) 失败, 退出码: {p.exitcode}")
913
+ pools.remove(p)
914
+ counter = Counter(all_path)
915
+ duplicates = {path for path, count in counter.items() if count > 1}
916
+ if len(duplicates):
917
+ self.print_dup_files(comp_path, duplicates)
918
+ raise errors.BmcGoException("请检查组件的打包逻辑,检测到重复文件: " + ".".join(duplicates))
919
+ # 最后处理openubmc
920
+ p = CopyComponent(self, misc.community_name(), os.path.join(self.conan_install, misc.community_name()), profile)
625
921
  p.run()
922
+ self.run_command(f"sudo rm -f {self.config.rootfs_path}/conaninfo.txt")
923
+ self.run_command(f"sudo rm -f {self.config.rootfs_path}/conanmanifest.txt")
924
+
925
+ def update_graph_nodes(self):
926
+ rootfs = GraphNode(None)
927
+ rootfs.name = "rootfs"
928
+ rootfs.package_folder = os.path.join(self.conan_install, "rootfs")
929
+ self.graph_nodes["rootfs"] = rootfs
930
+
931
+ with open(self.graphfile, "r") as fp:
932
+ graph = json.load(fp)
933
+ nodes = graph.get("graph", {}).get("nodes", {})
934
+ for _, info in nodes.items():
935
+ node = GraphNode(info)
936
+ context = info.get("context")
937
+ if context != "host":
938
+ continue
939
+ cmd = f"conan cache path {node.ref}:{node.package_id}"
940
+ node.package_folder = tools.run_command(cmd, capture_output=True).stdout.strip()
941
+ self.graph_nodes[node.name] = node
942
+
943
+ rootfs = GraphNode(None)
944
+ rootfs.name = misc.community_name()
945
+ rootfs.package_folder = os.path.join(self.conan_install, misc.community_name())
946
+ self.graph_nodes[misc.community_name()] = rootfs
947
+
948
+ def make_customization_symlinks(self):
949
+ if os.path.isdir(self.customization_dir):
950
+ self.run_command(f"rm -rf {self.customization_dir}")
951
+ self.run_command(f"mkdir -p {self.customization_dir}")
952
+ for name, node in self.graph_nodes.items():
953
+ cust = os.path.join(node.package_folder, "include", "customization.py")
954
+ if not os.path.isfile(cust):
955
+ continue
956
+ os.symlink(node.package_folder, os.path.join(self.customization_dir, name))
957
+
958
+ def package_v2(self):
959
+ self.update_graph_nodes()
960
+ self.make_customization_symlinks()
961
+ self.run_command(f"sudo rm -rf {self.config.rootfs_path}")
962
+ os.makedirs(self.config.rootfs_path)
963
+ profile, _ = self.get_profile_config()
964
+ self.copy_components_v2(profile)
965
+
966
+ self.chdir(self.config.rootfs_path)
967
+ self._component_cust_action("post_image")
968
+
969
+ self.chdir(self.config.rootfs_path)
970
+ self.package_info_gen_v2(f"{self.conan_install}/conan.lock", "etc/package_info")
971
+ self.sensitive_data_conf_gen_v2("opt/bmc/trust/sensitive_data.json")
972
+ if os.path.isfile("permissions.ini"):
973
+ os.unlink("permissions.ini")
626
974
 
627
975
  def package(self):
628
976
  self.chdir(self.conan_install)
977
+ if misc.conan_v2():
978
+ self.package_v2()
979
+ return
629
980
  self.run_command(f"sudo rm -rf {self.config.rootfs_path}")
630
981
  os.makedirs(self.config.rootfs_path)
631
982
  profile, _ = self.get_profile_config()
@@ -638,14 +989,8 @@ class TaskClass(Task):
638
989
 
639
990
  self.copy_components(comps, profile)
640
991
 
641
- # 运行组件定制化文件
642
- for comp in comps:
643
- cust = os.path.join(self.conan_install, comp, "include", "customization.py")
644
- if not os.path.isfile(cust):
645
- continue
646
- self.info(f"开始执行 {comp}/include/customization.py")
647
- post = ComponentPost(self.config, os.path.join(self.conan_install, comp), profile)
648
- post.post_work(self.config.rootfs_path, "post_image")
992
+ self.chdir(self.config.rootfs_path)
993
+ self._component_cust_action("post_image")
649
994
 
650
995
  self.chdir(self.config.rootfs_path)
651
996
  self.package_info_gen(f"{self.conan_source}/all/openubmc.bundle", "etc/package_info")
@@ -721,6 +1066,8 @@ class TaskClass(Task):
721
1066
  self.mkdir_work_path()
722
1067
  self.tools.clean_locks()
723
1068
  self.deploy()
1069
+ if self.only_manifest_yml:
1070
+ return
724
1071
  self.package_lock()
725
1072
  if not self.skip_package:
726
1073
  self.package()