openubmc-bingo 0.5.275__py3-none-any.whl → 0.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of openubmc-bingo might be problematic. Click here for more details.

Files changed (56) hide show
  1. bmcgo/__init__.py +1 -1
  2. bmcgo/bmcgo_config.py +22 -10
  3. bmcgo/cli/cli.py +86 -39
  4. bmcgo/cli/config.conan2.yaml +9 -0
  5. bmcgo/codegen/lua/codegen.py +1 -1
  6. bmcgo/codegen/lua/script/gen_intf_rpc_json.py +15 -14
  7. bmcgo/component/analysis/analysis.py +8 -8
  8. bmcgo/component/analysis/intf_validation.py +23 -12
  9. bmcgo/component/build.py +76 -14
  10. bmcgo/component/component_dt_version_parse.py +3 -2
  11. bmcgo/component/component_helper.py +43 -15
  12. bmcgo/component/coverage/incremental_cov.py +2 -2
  13. bmcgo/component/deploy.py +68 -14
  14. bmcgo/component/gen.py +1 -1
  15. bmcgo/component/package_info.py +128 -38
  16. bmcgo/component/template_v2/conanbase.py.mako +352 -0
  17. bmcgo/component/template_v2/conanfile.deploy.py.mako +26 -0
  18. bmcgo/component/test.py +53 -20
  19. bmcgo/frame.py +7 -3
  20. bmcgo/functional/analysis.py +3 -2
  21. bmcgo/functional/check.py +10 -6
  22. bmcgo/functional/conan_index_build.py +79 -20
  23. bmcgo/functional/csr_build.py +11 -3
  24. bmcgo/functional/diff.py +1 -1
  25. bmcgo/functional/fetch.py +1 -1
  26. bmcgo/functional/full_component.py +32 -24
  27. bmcgo/functional/git_history.py +220 -0
  28. bmcgo/functional/maintain.py +55 -12
  29. bmcgo/functional/new.py +1 -1
  30. bmcgo/functional/schema_valid.py +2 -2
  31. bmcgo/logger.py +2 -3
  32. bmcgo/misc.py +130 -9
  33. bmcgo/tasks/conan/__init__.py +10 -0
  34. bmcgo/tasks/conan/conanfile.py +45 -0
  35. bmcgo/tasks/task.py +27 -4
  36. bmcgo/tasks/task_build_conan.py +433 -65
  37. bmcgo/tasks/task_buildgppbin.py +8 -2
  38. bmcgo/tasks/task_download_buildtools.py +76 -0
  39. bmcgo/tasks/task_download_dependency.py +1 -0
  40. bmcgo/tasks/task_hpm_envir_prepare.py +1 -1
  41. bmcgo/utils/build_conans.py +231 -0
  42. bmcgo/utils/component_post.py +6 -4
  43. bmcgo/utils/component_version_check.py +10 -5
  44. bmcgo/utils/config.py +76 -40
  45. bmcgo/utils/fetch_component_code.py +44 -25
  46. bmcgo/utils/installations/install_plans/qemu.yml +6 -0
  47. bmcgo/utils/installations/install_plans/studio.yml +6 -0
  48. bmcgo/utils/installations/install_workflow.py +28 -2
  49. bmcgo/utils/merge_csr.py +124 -0
  50. bmcgo/utils/tools.py +239 -117
  51. bmcgo/worker.py +2 -2
  52. {openubmc_bingo-0.5.275.dist-info → openubmc_bingo-0.6.0.dist-info}/METADATA +4 -2
  53. {openubmc_bingo-0.5.275.dist-info → openubmc_bingo-0.6.0.dist-info}/RECORD +56 -46
  54. {openubmc_bingo-0.5.275.dist-info → openubmc_bingo-0.6.0.dist-info}/WHEEL +0 -0
  55. {openubmc_bingo-0.5.275.dist-info → openubmc_bingo-0.6.0.dist-info}/entry_points.txt +0 -0
  56. {openubmc_bingo-0.5.275.dist-info → openubmc_bingo-0.6.0.dist-info}/top_level.txt +0 -0
@@ -27,10 +27,10 @@ import random
27
27
  import pathlib
28
28
  from multiprocessing import Process, Queue
29
29
  from copy import deepcopy
30
+ from collections import Counter
30
31
 
31
32
  import yaml
32
33
  from colorama import Fore, Back, Style
33
- from conans.model.profile import Profile
34
34
 
35
35
  from bmcgo.tasks.task import Task
36
36
  from bmcgo.utils.config import Config
@@ -40,20 +40,49 @@ from bmcgo import errors, misc
40
40
  from bmcgo.utils.tools import Tools
41
41
  from bmcgo.errors import BmcGoException
42
42
  from bmcgo.component.component_helper import ComponentHelper
43
+ from bmcgo.utils.build_conans import BuildConans
44
+ from bmcgo.component.deploy import GraphNode
45
+ from bmcgo.tasks.misc import MODULE_SYMVERS, SDK_PATH
43
46
 
44
47
  IBMC_LOCK_FILE = "openubmc.lock"
45
48
  SDK_ROOT = "/opt/hi1711sdk"
46
49
  SDK_ROOT_MODULE_SYMVERS = os.path.join(SDK_ROOT, "Module.symvers")
50
+ PERSIST_TYPES = {
51
+ "TemporaryPer": "TemporaryPer",
52
+ "TemporaryPerRetain": "TemporaryPer",
53
+ "ResetPer": "ResetPer",
54
+ "ResetPerRetain": "ResetPer",
55
+ "PoweroffPer": "PoweroffPer",
56
+ "PoweroffPerRetain": "PoweroffPer",
57
+ "PermanentPer": "PermanentPer"
58
+ }
59
+ PRIMARY_KEYS = "PrimaryKeys"
47
60
  tools = Tools()
48
61
 
62
+ PERSIST_TYPES = {
63
+ "TemporaryPer": "TemporaryPer",
64
+ "TemporaryPerRetain": "TemporaryPer",
65
+ "ResetPer": "ResetPer",
66
+ "ResetPerRetain": "ResetPer",
67
+ "PoweroffPer": "PoweroffPer",
68
+ "PoweroffPerRetain": "PoweroffPer",
69
+ "PermanentPer": "PermanentPer"
70
+ }
71
+
72
+ PRIMARY_KEYS = "PrimaryKeys"
73
+
74
+ cwd_dir = os.path.dirname(os.path.realpath(__file__))
75
+
49
76
 
50
77
  class CopyComponent(Process):
51
- def __init__(self, work: Task, comp: str, profile: Profile):
78
+ def __init__(self, work: Task, comp: str, package_folder: str, profile):
52
79
  super().__init__()
53
80
  self.work = work
54
81
  self.config = self.work.config
55
82
  self.comp = comp
83
+ self.package_folder = package_folder
56
84
  self.profile = profile
85
+ self.graphfile = None
57
86
 
58
87
  def link_recursive_deal(self, file_name, ownership):
59
88
  self.work.run_command(f"chown -h {ownership} {file_name}", sudo=True)
@@ -64,16 +93,16 @@ class CopyComponent(Process):
64
93
  self.work.work_name = self.comp
65
94
  # 复制组件文件到rootfs中
66
95
  # rc.sysinit脚本只允许使用rootfs_user仓库的
67
- rc_sysinit_path = os.path.join(self.comp, "etc/rc.d/rc.sysinit")
96
+ rc_sysinit_path = os.path.join(self.package_folder, "etc/rc.d/rc.sysinit")
68
97
  if self.comp != "rootfs_user" and os.path.isfile(rc_sysinit_path):
69
98
  shutil.rmtree(rc_sysinit_path, ignore_errors=True)
70
99
  # 由于复制时有很多同名路径,cp命令有概率失败,10次复制尝试
71
100
  self._copy_files()
72
101
 
73
102
  # 执行权限控制逻辑
74
- per_cfg = os.path.join(self.comp, "permissions.ini")
103
+ per_cfg = os.path.join(self.package_folder, "permissions.ini")
75
104
  if not os.path.isfile(per_cfg):
76
- self.work.warning("权限文件 {} 不存在, 所以无法设置组件 {} 的权限".format(per_cfg, self.comp))
105
+ self.work.warning("权限文件 {} 不存在, 所以无法设置组件 {} 的权限".format(per_cfg, self.package_folder))
77
106
  return
78
107
  self.work.info("依据配置文件: {}, 修改权限".format(per_cfg))
79
108
  # NOTE 这里的配置时间, 偶发复制未完成, 就开始修改权限
@@ -120,7 +149,7 @@ class CopyComponent(Process):
120
149
 
121
150
  def _copy_files(self):
122
151
  # 由于复制时有很多同名路径,cp命令有概率失败,10次复制尝试
123
- cmd = "sudo cp -dfr {}/. {}".format(self.comp, self.config.rootfs_path)
152
+ cmd = "sudo cp -dfr {}/. {}".format(self.package_folder, self.config.rootfs_path)
124
153
  for _ in range(0, 10):
125
154
  try:
126
155
  # 尝试复制到目标目录,成功则break退出循环
@@ -155,22 +184,33 @@ class ConanLockParse:
155
184
  self._process_list = []
156
185
 
157
186
  def conan_install(self, cmd, com):
158
- real_cmd = shlex.split(cmd)
159
- pipe = subprocess.Popen(real_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
160
- # 超时时长1800s
161
- out, _ = pipe.communicate(timeout=1800)
162
- if pipe.returncode != 0:
163
- self.work.error(f"================== {com} 构建失败日志起始位置 ==================")
164
- self.work.info(out.decode())
165
- self.work.error(f"================== {com} 构建失败日志结束位置 ==================")
166
- self._queue.put(self.error_str)
167
- self._queue.put(self.error_str)
168
- self._queue.put(self.error_str)
169
- return
170
- self._queue.put(com)
171
- self._queue.put(com)
172
- self._queue.put(com)
173
- self.work.info(f"命令 {cmd} " + Fore.GREEN + "执行完成" + Style.RESET_ALL)
187
+ com_name = com[:com.index("/")]
188
+ com_install_log = os.path.join(self.work.config.temp_path, f"log/conan_install/{com_name}.log")
189
+ with os.fdopen(os.open(com_install_log, os.O_RDWR | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), "a+") as f:
190
+ self.work.info(f"命令 {cmd} " + Fore.GREEN + "开始" + Style.RESET_ALL)
191
+ real_cmd = shlex.split(cmd)
192
+ pipe = subprocess.Popen(real_cmd, stdout=f, stderr=f)
193
+ start_time = time.time()
194
+ while True:
195
+ if pipe.poll() is not None:
196
+ break
197
+ # 1800秒超时
198
+ if time.time() - start_time > 1800:
199
+ pipe.kill()
200
+ self.work.error(f"================== {com} 构建超时 ==================")
201
+ self._queue.put(self.error_str)
202
+ return
203
+ time.sleep(1)
204
+ if pipe.returncode != 0:
205
+ self.work.error(f"================== {com} 构建失败日志起始位置 ==================")
206
+ f.seek(0)
207
+ conan_log = f.read()
208
+ self.work.info(conan_log)
209
+ self.work.error(f"================== {com} 构建失败日志结束位置 ==================")
210
+ self._queue.put(self.error_str)
211
+ return
212
+ self._queue.put(com)
213
+ self.work.info(f"命令 {cmd} " + Fore.GREEN + "执行完成" + Style.RESET_ALL)
174
214
 
175
215
  def wait_task_finished(self):
176
216
  # 所有入度为0任务拉起后等待完成消息
@@ -200,6 +240,9 @@ class ConanLockParse:
200
240
  self._degree = {x: len(self._bundle[x].get('requires')) if self._bundle[x].get('requires') is not None else 0
201
241
  for x in self._bundle.keys()}
202
242
 
243
+ conan_log_path = os.path.join(self.work.config.temp_path, "log/conan_install")
244
+ os.makedirs(conan_log_path, exist_ok=True)
245
+
203
246
  self.work.success(">>>>>>>>>>>>>>>>> 开始构建所有组件 <<<<<<<<<<<<<<<<")
204
247
  self.work.info("这些进程将要花费几分钟(2-3分钟), 请耐心等待")
205
248
  while self._degree:
@@ -216,7 +259,6 @@ class ConanLockParse:
216
259
  t = Process(target=self.conan_install, args=(cmd_tmp, com))
217
260
  t.start()
218
261
  self._process_list.append(t)
219
- self.work.info(f"命令 {cmd_tmp} " + Fore.GREEN + "开始" + Style.RESET_ALL)
220
262
  # 入度为0的任务构建拉起后弹出
221
263
  self._degree.pop(com)
222
264
  else:
@@ -233,7 +275,11 @@ class TaskClass(Task):
233
275
 
234
276
  def __init__(self, config: Config, work_name=""):
235
277
  super(TaskClass, self).__init__(config, work_name)
278
+ self.skip_install_comp = False
279
+ self.only_manifest_yml = False
236
280
  self.update_path()
281
+ # 记录每个组件的GraphNode信息
282
+ self.package_info = os.path.join(self.config.rootfs_path, "package_info")
237
283
 
238
284
  @property
239
285
  def subsys_dir(self):
@@ -244,6 +290,20 @@ class TaskClass(Task):
244
290
  subsys = os.path.join(subsys, misc.StageEnum.STAGE_RC.value)
245
291
  return subsys
246
292
 
293
+ @staticmethod
294
+ def match_persist_type(usage):
295
+ for item in usage:
296
+ if item in PERSIST_TYPES:
297
+ return PERSIST_TYPES[item]
298
+ return None
299
+
300
+ @staticmethod
301
+ def match_persist_type(usage):
302
+ for item in usage:
303
+ if item in PERSIST_TYPES:
304
+ return PERSIST_TYPES[item]
305
+ return None
306
+
247
307
  def find_conan_package_and_write(self, search, comps, file_handler, stage):
248
308
  if search.find("/") < 0:
249
309
  search += "/"
@@ -256,18 +316,19 @@ class TaskClass(Task):
256
316
  if stage != misc.StageEnum.STAGE_DEV.value and comp.endswith(f"/{misc.StageEnum.STAGE_DEV.value}"):
257
317
  err_msg = f"组件包 {comp} user/channel 配置错误, 必须以 \"{stage}\" 作为结尾, 终止构建"
258
318
  raise errors.BmcGoException(err_msg)
319
+ if misc.conan_v2():
320
+ comp = comp.lower()
259
321
  self.add_new_dependencies(comp, file_handler)
260
322
  return
261
323
  raise errors.ConfigException(f"未知组件: {search}, 请检查配置 !!!")
262
324
 
263
325
  def update_path(self):
264
326
  # conan source folder
265
- self.conan_source = os.path.join(self.config.temp_path,
266
- f"conan_source_{self.config.build_type}_{self.config.stage}/openubmc")
327
+ self.conan_source = os.path.join(self.config.build_path, misc.community_name())
267
328
  # conan install folder
268
329
  self.conan_install = os.path.join(self.config.build_path, "conan_install")
269
330
  # openubmc install folder
270
- self.openubmc_ins_dir = os.path.join(self.conan_install, "openubmc")
331
+ self.openubmc_ins_dir = os.path.join(self.conan_install, misc.community_name())
271
332
  # openubmc top rootfs folder
272
333
  self.top_rootfs_dir = os.path.join(self.conan_install, "rootfs")
273
334
  self.board_option = ""
@@ -303,6 +364,18 @@ class TaskClass(Task):
303
364
  else:
304
365
  self.skip_package = False
305
366
 
367
+ def set_skip_install_comp(self, value):
368
+ if value:
369
+ self.skip_install_comp = True
370
+ else:
371
+ self.skip_install_comp = False
372
+
373
+ def set_only_manifest_yml(self, value):
374
+ if value:
375
+ self.only_manifest_yml = True
376
+ else:
377
+ self.only_manifest_yml = False
378
+
306
379
  def package_dependencies_parse(self, default_component: list):
307
380
  """解析manufacture或者tosupporte配置,不同的包设置不同组件的不同编译选项
308
381
  参数:
@@ -329,6 +402,58 @@ class TaskClass(Task):
329
402
  self.success(f"获取到依赖: {conan}")
330
403
  self.depdencies.append(conan)
331
404
 
405
+ def merge_manifest_v2(self):
406
+ comps = []
407
+ for f in os.listdir(self.subsys_dir):
408
+ with open(os.path.join(self.subsys_dir, f)) as fp:
409
+ yml = yaml.safe_load(fp)
410
+ deps = yml.get('dependencies')
411
+ for dep in deps:
412
+ conan = dep.get(misc.CONAN)
413
+ if conan:
414
+ comps.append(conan)
415
+ self.debug("依赖列表: {}".format(comps))
416
+ # 重建新的依赖关系
417
+ # 从单板目录manifest.yml,与subsys/<stage>目录下的组件合并
418
+ new_fd = os.fdopen(os.open("manifest.yml", os.O_WRONLY | os.O_CREAT,
419
+ stat.S_IWUSR | stat.S_IRUSR), 'w')
420
+ new_fd.write("base:\n")
421
+ ver = self.get_manufacture_config("base/version")
422
+
423
+ new_fd.write(f" version: \"{ver}\"\n")
424
+ self.info(f"包版本号: {ver}")
425
+
426
+ new_fd.write("dependencies:\n")
427
+ deps = self._get_dependencies_pkg()
428
+
429
+ # 如果是打包,依赖信息调整
430
+ deps = self.package_dependencies_parse(deps)
431
+ # 由于manifest.yml当中有对于此的新的配置,此处将配置读出,并重新分配
432
+ skynet_with_enable_luajit = False
433
+ for dep in deps:
434
+ conan = dep.get(misc.CONAN)
435
+ if not conan:
436
+ continue
437
+ if conan.find("@") > 0:
438
+ self.add_new_dependencies(conan.lower(), new_fd)
439
+ else:
440
+ self.find_conan_package_and_write(conan, comps, new_fd, self.config.stage)
441
+ options = dep.get("options", {})
442
+ name = conan.split("/")[0]
443
+ for key, val in options.items():
444
+ # skynet特殊处理:manifest.yml指定enable_luajit特性时需要覆盖用户输入
445
+ if name == "skynet" and key == "enable_luajit":
446
+ self.warning(f"根据manifest.yml配置,当前产品的enable_luajit配置为{val},忽略命令行指定的-jit参数")
447
+ skynet_with_enable_luajit = True
448
+ self.config.set_enable_luajit(val)
449
+ self.board_option = self.board_option + " -o */*:{}={}".format(key, val)
450
+ else:
451
+ self.board_option = self.board_option + " -o {}/*:{}={}".format(name, key, val)
452
+ # 当使能Luajit又未向skynet传递enable_luajit配置项时需要添加使能参数
453
+ if not skynet_with_enable_luajit and self.config.enable_luajit:
454
+ self.board_option += f" -o */*:enable_luajit={self.config.enable_luajit}"
455
+ new_fd.close()
456
+
332
457
  def merge_manifest(self):
333
458
  comps = []
334
459
  for f in os.listdir(self.subsys_dir):
@@ -347,7 +472,7 @@ class TaskClass(Task):
347
472
  new_fd.write("base:\n")
348
473
  ver = self.get_manufacture_config("base/version")
349
474
 
350
- new_fd.write(f" version: \"{ver}@{tools.conan_user}/{misc.StageEnum.STAGE_RC.value}\"\n")
475
+ new_fd.write(f" version: \"{ver}@{misc.conan_user()}/{misc.StageEnum.STAGE_RC.value}\"\n")
351
476
  self.info(f"包版本号: {ver}")
352
477
 
353
478
  new_fd.write("dependencies:\n")
@@ -367,7 +492,7 @@ class TaskClass(Task):
367
492
  stage = self.config.stage
368
493
  if stage != misc.StageEnum.STAGE_STABLE.value:
369
494
  stage = misc.StageEnum.STAGE_RC.value
370
- conan += f"@{tools.conan_user}/{stage}"
495
+ conan += f"@{misc.conan_user()}/{stage}"
371
496
  self.add_new_dependencies(conan, new_fd)
372
497
  else:
373
498
  self.find_conan_package_and_write(conan, comps, new_fd, self.config.stage)
@@ -383,17 +508,26 @@ class TaskClass(Task):
383
508
  # 当使能Luajit又未向skynet传递enable_luajit配置项时需要添加使能参数
384
509
  if not skynet_with_enable_luajit and self.config.enable_luajit:
385
510
  self.board_option += f" -o skynet:enable_luajit={self.config.enable_luajit}"
386
- self.merge_0502_default_options()
387
511
  sha256 = Tools.sha256sum(SDK_ROOT_MODULE_SYMVERS)
388
512
  self.board_option += " -o *:module_symvers={}".format(sha256)
389
513
  new_fd.close()
390
514
 
515
+ def merge_dep_options(self):
516
+ self.merge_0502_default_options()
517
+
391
518
  def merge_0502_default_options(self):
519
+ if misc.conan_v1():
520
+ prefix = "*"
521
+ else:
522
+ prefix = "*/*"
392
523
  if self.config.manufacture_code:
393
524
  default_options_path = f"manufacture/{self.config.manufacture_code}/default_options"
394
525
  default_options = self.get_manufacture_config(default_options_path, {})
395
526
  for key, val in default_options.items():
396
- self.board_option += f" -o *:{key}={val}"
527
+ self.board_option += f" -o {prefix}:{key}={val}"
528
+ module_symvers_path = os.path.join(SDK_PATH, MODULE_SYMVERS)
529
+ option, value = self.tools.get_hi171x_module_symver_option(module_symvers_path)
530
+ self.board_option += f" -o {prefix}:{option}={value}"
397
531
 
398
532
  def package_info_gen(self, bundle_file, dst_file):
399
533
  with open(bundle_file, "r") as fp:
@@ -402,8 +536,8 @@ class TaskClass(Task):
402
536
  list.sort(require_list)
403
537
  package_info_list = deepcopy(require_list)
404
538
  self.run_command(f"rm -rf {dst_file}", sudo=True)
405
- temp_dst_file = f"{self.config.rootfs_path}/{os.path.basename(dst_file)}"
406
- with os.fdopen(os.open(temp_dst_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, stat.S_IWUSR | stat.S_IRUSR |
539
+ self.package_info = f"{self.config.rootfs_path}/{os.path.basename(dst_file)}"
540
+ with os.fdopen(os.open(self.package_info, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, stat.S_IWUSR | stat.S_IRUSR |
407
541
  stat.S_IWGRP | stat.S_IRGRP | stat.S_IWOTH | stat.S_IROTH), 'w') as fp:
408
542
  for package in require_list:
409
543
  if "openubmc/" not in package:
@@ -411,8 +545,7 @@ class TaskClass(Task):
411
545
  else:
412
546
  package_info_list.remove(package)
413
547
  fp.close()
414
- self.run_command(f"cp -df {temp_dst_file} {dst_file}", sudo=True)
415
- self.run_command(f"rm -rf {temp_dst_file}", sudo=True)
548
+ self.run_command(f"cp -df {self.package_info} {dst_file}", sudo=True)
416
549
 
417
550
  def sensitive_data_conf_gen(self, comps, dst_file):
418
551
  self.run_command(f"rm -rf {dst_file}", sudo=True)
@@ -421,7 +554,8 @@ class TaskClass(Task):
421
554
  "TemporaryPer": {},
422
555
  "ResetPer": {},
423
556
  "PoweroffPer": {},
424
- "PermanentPer": {}
557
+ "PermanentPer": {},
558
+ "PrimaryKeys": {}
425
559
  }
426
560
  for comp in comps:
427
561
  try:
@@ -435,17 +569,20 @@ class TaskClass(Task):
435
569
  self.run_command(f"cp -df {temp_dst_file} {dst_file}", sudo=True)
436
570
  self.run_command(f"rm -rf {temp_dst_file}", sudo=True)
437
571
 
438
- def proc_sensitive(self, per_type, props, table_name, output):
439
- for prop_name, prop_data in props.items():
440
- for item in prop_data.get("usage", []):
441
- if item in output:
442
- per_type = item
443
- break
444
- is_sensitive = prop_data.get("sensitive", False)
445
- if per_type not in output or not is_sensitive:
572
+ def proc_sensitive(self, table_type, props, table_name, output):
573
+ for prop_key, prop_data in props.items():
574
+ prop_name = prop_key
575
+ if "alias" in prop_data:
576
+ prop_name = prop_data.get("alias")
577
+ if prop_data.get("primaryKey"):
578
+ output[PRIMARY_KEYS][table_name] = output[PRIMARY_KEYS].get(table_name, {})
579
+ output[PRIMARY_KEYS][table_name][prop_name] = prop_data.get("sensitive", False)
580
+ continue
581
+ per_type = self.match_persist_type(prop_data.get("usage", [])) or table_type
582
+ if not per_type:
446
583
  continue
447
584
  output[per_type][table_name] = output[per_type].get(table_name, {})
448
- output[per_type][table_name][prop_name] = True
585
+ output[per_type][table_name][prop_name] = prop_data.get("sensitive", False)
449
586
  return output
450
587
 
451
588
  def proc_comp(self, comp, output):
@@ -455,15 +592,15 @@ class TaskClass(Task):
455
592
  with open(model_file, "r") as fp:
456
593
  content = json.load(fp)
457
594
  for class_data in content.values():
458
- per_type = class_data.get("tableType", "")
595
+ table_type = PERSIST_TYPES.get(class_data.get("tableType", ""))
459
596
  table_name = class_data.get("tableName", "")
460
- if not table_name:
597
+ if not table_name or class_data.get("tableLocation") == "Local":
461
598
  continue
462
599
  class_props = [class_data.get("properties", {})]
463
600
  for intf_data in class_data.get("interfaces", {}).values():
464
601
  class_props.append(intf_data.get("properties", {}))
465
602
  for props in class_props:
466
- output = self.proc_sensitive(per_type, props, table_name, output)
603
+ output = self.proc_sensitive(table_type, props, table_name, output)
467
604
  return output
468
605
 
469
606
  def calc_options(self):
@@ -479,6 +616,79 @@ class TaskClass(Task):
479
616
  options += " -o *:gcov=True"
480
617
  return options
481
618
 
619
+ def proc_sensitive_v2(self, table_type, props, table_name, output):
620
+ for prop_name, prop_data in props.items():
621
+ if prop_data.get("primaryKey"):
622
+ output[PRIMARY_KEYS][table_name] = output[PRIMARY_KEYS].get(table_name, {})
623
+ output[PRIMARY_KEYS][table_name][prop_name] = prop_data.get("sensitive", False)
624
+ continue
625
+ per_type = self.match_persist_type(prop_data.get("usage", [])) or table_type
626
+ if not per_type:
627
+ continue
628
+ output[per_type][table_name] = output[per_type].get(table_name, {})
629
+ output[per_type][table_name][prop_name] = prop_data.get("sensitive", False)
630
+ return output
631
+
632
+ def proc_comp_v2(self, package_folder, output):
633
+ model_file = os.path.join(package_folder, "include", "mds", "model.json")
634
+ if not os.path.isfile(model_file):
635
+ return output
636
+ with open(model_file, "r") as fp:
637
+ content = json.load(fp)
638
+ for class_data in content.values():
639
+ table_type = PERSIST_TYPES.get(class_data.get("tableType", ""))
640
+ table_name = class_data.get("tableName", "")
641
+ if not table_name or class_data.get("tableLocation") == "Local":
642
+ continue
643
+ class_props = [class_data.get("properties", {})]
644
+ for intf_data in class_data.get("interfaces", {}).values():
645
+ class_props.append(intf_data.get("properties", {}))
646
+ for props in class_props:
647
+ output = self.proc_sensitive_v2(table_type, props, table_name, output)
648
+ return output
649
+
650
+ def calc_options_v2(self):
651
+ if self.config.build_type == "debug":
652
+ options = "-s build_type=Debug"
653
+ else:
654
+ options = "-s build_type=Release"
655
+
656
+ if self.config.enable_arm_gcov:
657
+ options += " -o */*:gcov=True"
658
+ return options
659
+
660
+ def install_openubmc_v2(self):
661
+ profile_file = os.path.join(self.tools.conan_profiles_dir, self.config.profile)
662
+ if not os.path.isfile(profile_file):
663
+ raise BmcGoException(f"{profile_file} 文件不存在")
664
+
665
+ options = self.calc_options_v2()
666
+ append_cmd = f"-r {self.config.remote}" if self.config.remote else ""
667
+ # 依据选项生成openubmc.lock和openubmc.bundle文件
668
+ base_cmd = f"-pr={self.config.profile} "
669
+ base_cmd += f"-pr:b profile.dt.ini {append_cmd} {options} {self.board_option}"
670
+ lockfile = os.path.join(self.config.build_path, "conan.lock")
671
+ graphfile = os.path.join(self.config.build_path, "graph.info")
672
+ lock_cmd = f"conan lock create . {base_cmd} --lockfile-out={lockfile}"
673
+ self.run_command(lock_cmd, capture_output=False)
674
+ graph_cmd = f"conan graph info . {base_cmd} -f json --lockfile={lockfile} --out-file={graphfile}"
675
+ self.run_command(graph_cmd)
676
+ self.success(f"start build dependency packages of {self.config.board_name}")
677
+ bcp = BuildConans(graphfile, lockfile, base_cmd, self.config.from_source, self.config.log_path)
678
+ bcp.build()
679
+
680
+ cmd = f"conan create . {base_cmd} --build=missing"
681
+ self.info(f"start build {self.config.board_name}: {cmd}")
682
+ self.run_command(cmd, capture_output=False)
683
+
684
+ self.graphfile = graphfile
685
+
686
+ shutil.copyfile(lockfile, f"{self.conan_install}/conan.lock")
687
+ # 检查使用到的组件是否都在单板目录 manifest.yml 中配置了
688
+ component_check = ComponentVersionCheck("manifest.yml", lockfile, "openubmc")
689
+ component_check.run()
690
+ self.clean_luac_out()
691
+
482
692
  def install_ibmc(self):
483
693
  profile_file = os.path.join(self.tools.conan_profiles_dir, self.config.profile)
484
694
  if not os.path.isfile(profile_file):
@@ -518,21 +728,30 @@ class TaskClass(Task):
518
728
  self.run_command(install_cmd, command_echo=True)
519
729
  shutil.copyfile(IBMC_LOCK_FILE, f"{self.conan_install}/conan.lock")
520
730
  # 检查使用到的组件是否都在单板目录 manifest.yml 中配置了
521
- component_check = ComponentVersionCheck(manifest_yml="manifest.yml", ibmc_lock=IBMC_LOCK_FILE)
731
+ component_check = ComponentVersionCheck("manifest.yml", IBMC_LOCK_FILE, "openubmc")
522
732
  component_check.run()
523
733
  self.clean_luac_out()
524
734
 
525
735
  def deploy(self):
526
- src = os.path.join(self.config.code_path, "conan_index/openubmc")
527
- shutil.copytree(src, self.conan_source, dirs_exist_ok=True)
736
+ conanfile_dir = os.path.join(cwd_dir, "conan")
528
737
  openubmc_dir = os.path.join(self.conan_source, "all")
738
+ os.makedirs(openubmc_dir, exist_ok=True)
739
+ shutil.copytree(conanfile_dir, openubmc_dir, dirs_exist_ok=True)
529
740
  self.chdir(openubmc_dir)
530
741
  # 复制manifest.yml文件
531
- self.merge_manifest()
742
+ if misc.conan_v2():
743
+ self.merge_manifest_v2()
744
+ else:
745
+ self.merge_manifest()
746
+ if self.only_manifest_yml:
747
+ return
748
+ if not self.skip_install_comp:
749
+ self.merge_dep_options()
532
750
  # 下载组件构建脚本
533
751
  ComponentHelper.download_recipes(self.depdencies, self.tools, self.config.remote_list)
534
752
  # 下载skynet
535
- self.install_luac_or_luajit()
753
+ if misc.conan_v1():
754
+ self.install_luac_or_luajit()
536
755
 
537
756
  # 复制全局定制rootfs到conan install目录
538
757
  top_rootfs = os.path.join(self.config.code_path, "rootfs")
@@ -557,7 +776,10 @@ class TaskClass(Task):
557
776
  if os.path.isfile(per_file):
558
777
  shutil.copy(per_file, self.openubmc_ins_dir)
559
778
 
560
- self.install_ibmc()
779
+ if misc.conan_v2():
780
+ self.install_openubmc_v2()
781
+ else:
782
+ self.install_ibmc()
561
783
 
562
784
  def clean_luac_out(self):
563
785
  # 清理冗余文件luac.out
@@ -579,15 +801,15 @@ class TaskClass(Task):
579
801
  self.run_command(f'cp -f conan.lock {os.path.join(inner_path, f"package_{self.config.board_name}.lock")}')
580
802
  self.run_command(f'cp -f conan.lock {os.path.join(self.config.output_path, f"package.lock")}')
581
803
 
582
- def copy_components(self, comps: list, profile: Profile):
804
+ def copy_components(self, comps: list, profile):
583
805
  # 优先处理rootfs
584
- p = CopyComponent(self, "rootfs", profile)
806
+ p = CopyComponent(self, "rootfs", os.path.join(self.conan_install, "rootfs"), profile)
585
807
  p.run()
586
808
  # 打印组件清单
587
809
  self.info(f"组件列表: {comps}")
588
810
  pools = []
589
811
  for comp in comps:
590
- p = CopyComponent(self, comp, profile)
812
+ p = CopyComponent(self, comp, os.path.join(self.conan_install, comp), profile)
591
813
  p.start()
592
814
  pools.append(p)
593
815
 
@@ -600,11 +822,161 @@ class TaskClass(Task):
600
822
  raise errors.BmcGoException(f"复制组件 ({p.comp}) 失败, 退出码: {p.exitcode}")
601
823
  pools.remove(p)
602
824
  # 最后处理openubmc
603
- p = CopyComponent(self, "openubmc", profile)
825
+ p = CopyComponent(self, misc.community_name(), os.path.join(self.conan_install, misc.community_name()), profile)
826
+ p.run()
827
+
828
+ def package_info_gen_v2(self, bmc_lock, dst_file):
829
+ with open(bmc_lock, "r") as bmc_lock_fp:
830
+ lock_info = json.load(bmc_lock_fp)
831
+ require_list = lock_info.get("requires")
832
+ self.run_command(f"rm -rf {dst_file}", sudo=True)
833
+ self.package_info = f"{self.config.rootfs_path}/{os.path.basename(dst_file)}"
834
+ with os.fdopen(os.open(self.package_info, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, stat.S_IWUSR | stat.S_IRUSR |
835
+ stat.S_IWGRP | stat.S_IRGRP | stat.S_IWOTH | stat.S_IROTH), 'w') as fp:
836
+ for package in require_list:
837
+ fp.write(f"{package}\n")
838
+ fp.close()
839
+ self.run_command(f"cp -df {self.package_info} {dst_file}", sudo=True)
840
+
841
+ def sensitive_data_conf_gen_v2(self, dst_file):
842
+ self.run_command(f"rm -rf {dst_file}", sudo=True)
843
+ temp_dst_file = f"{self.config.rootfs_path}/{os.path.basename(dst_file)}"
844
+ output = {
845
+ "TemporaryPer": {},
846
+ "ResetPer": {},
847
+ "PoweroffPer": {},
848
+ "PermanentPer": {},
849
+ "PrimaryKeys": {}
850
+ }
851
+ for name, node in self.graph_nodes.items():
852
+ try:
853
+ output = self.proc_comp_v2(node.package_folder, output)
854
+ except Exception as e:
855
+ raise errors.BmcGoException(f"分析组件 {name} 的 model.json 失败, 失败信息:{e}")
856
+
857
+ with os.fdopen(os.open(temp_dst_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC,
858
+ stat.S_IWUSR | stat.S_IRUSR), 'w') as fp:
859
+ json.dump(output, fp, sort_keys=True)
860
+ self.run_command(f"mkdir -p {os.path.dirname(dst_file)}", sudo=True)
861
+ self.run_command(f"cp -df {temp_dst_file} {dst_file}", sudo=True)
862
+ self.run_command(f"rm -rf {temp_dst_file}", sudo=True)
863
+
864
+ def get_all_subdirectories(self, dirpath):
865
+ exclude_files = ["permissions.ini", "conaninfo.txt", "conanmanifest.txt"]
866
+ out = []
867
+ for root, _, files in os.walk(dirpath):
868
+ for file in files:
869
+ file = os.path.join(root, file)
870
+ relative_path = os.path.relpath(file, dirpath)
871
+ if not relative_path.startswith("include") and relative_path not in exclude_files:
872
+ out.append(relative_path)
873
+ return out
874
+
875
+ def print_dup_files(self, comps, paths):
876
+ dup_paths = {}
877
+ for comp, files in comps.items():
878
+ for path in paths:
879
+ if path in files:
880
+ dup_paths.setdefault(path, []).append(comp)
881
+
882
+ for path, comp in dup_paths.items():
883
+ self.error(f"{comp} 存在重复文件 {path}")
884
+
885
+ def copy_components_v2(self, profile):
886
+ # 优先处理rootfs
887
+ p = CopyComponent(self, "rootfs", os.path.join(self.conan_install, "rootfs"), profile)
888
+ p.run()
889
+ # 打印组件清单
890
+ self.info(f"组件列表: {self.graph_nodes.keys()}")
891
+ pools = []
892
+ # 记录全部的文件信息
893
+ all_path = []
894
+ # 记录组件有哪些文件
895
+ comp_path = {}
896
+ for name, node in self.graph_nodes.items():
897
+ if name in ["rootfs", misc.community_name()]:
898
+ continue
899
+ file_paths = self.get_all_subdirectories(node.package_folder)
900
+ all_path.extend(file_paths)
901
+ comp_path[name] = file_paths
902
+ p = CopyComponent(self, node.name, node.package_folder, profile)
903
+ p.start()
904
+ pools.append(p)
905
+
906
+ while pools:
907
+ time.sleep(0.01)
908
+ for p in pools:
909
+ if p.is_alive():
910
+ continue
911
+ if p.exitcode is not None and p.exitcode != 0:
912
+ raise errors.BmcGoException(f"复制组件 ({p.name}) 失败, 退出码: {p.exitcode}")
913
+ pools.remove(p)
914
+ counter = Counter(all_path)
915
+ duplicates = {path for path, count in counter.items() if count > 1}
916
+ if len(duplicates):
917
+ self.print_dup_files(comp_path, duplicates)
918
+ raise errors.BmcGoException("请检查组件的打包逻辑,检测到重复文件: " + ".".join(duplicates))
919
+ # 最后处理openubmc
920
+ p = CopyComponent(self, misc.community_name(), os.path.join(self.conan_install, misc.community_name()), profile)
604
921
  p.run()
922
+ self.run_command(f"sudo rm -f {self.config.rootfs_path}/conaninfo.txt")
923
+ self.run_command(f"sudo rm -f {self.config.rootfs_path}/conanmanifest.txt")
924
+
925
+ def update_graph_nodes(self):
926
+ rootfs = GraphNode(None)
927
+ rootfs.name = "rootfs"
928
+ rootfs.package_folder = os.path.join(self.conan_install, "rootfs")
929
+ self.graph_nodes["rootfs"] = rootfs
930
+
931
+ with open(self.graphfile, "r") as fp:
932
+ graph = json.load(fp)
933
+ nodes = graph.get("graph", {}).get("nodes", {})
934
+ for _, info in nodes.items():
935
+ node = GraphNode(info)
936
+ context = info.get("context")
937
+ if context != "host":
938
+ continue
939
+ cmd = f"conan cache path {node.ref}:{node.package_id}"
940
+ node.package_folder = tools.run_command(cmd, capture_output=True).stdout.strip()
941
+ self.graph_nodes[node.name] = node
942
+
943
+ rootfs = GraphNode(None)
944
+ rootfs.name = misc.community_name()
945
+ rootfs.package_folder = os.path.join(self.conan_install, misc.community_name())
946
+ self.graph_nodes[misc.community_name()] = rootfs
947
+
948
+ def make_customization_symlinks(self):
949
+ if os.path.isdir(self.customization_dir):
950
+ self.run_command(f"rm -rf {self.customization_dir}")
951
+ self.run_command(f"mkdir -p {self.customization_dir}")
952
+ for name, node in self.graph_nodes.items():
953
+ cust = os.path.join(node.package_folder, "include", "customization.py")
954
+ if not os.path.isfile(cust):
955
+ continue
956
+ os.symlink(node.package_folder, os.path.join(self.customization_dir, name))
957
+
958
+ def package_v2(self):
959
+ self.update_graph_nodes()
960
+ self.make_customization_symlinks()
961
+ self.run_command(f"sudo rm -rf {self.config.rootfs_path}")
962
+ os.makedirs(self.config.rootfs_path)
963
+ profile, _ = self.get_profile_config()
964
+ self.copy_components_v2(profile)
965
+
966
+ self.chdir(self.config.rootfs_path)
967
+ self._component_cust_action("post_image")
968
+
969
+ self.chdir(self.config.rootfs_path)
970
+ self.package_info_gen_v2(f"{self.conan_install}/conan.lock", "etc/package_info")
971
+ self.sensitive_data_conf_gen_v2("opt/bmc/trust/sensitive_data.json")
972
+ if os.path.isfile("permissions.ini"):
973
+ os.unlink("permissions.ini")
605
974
 
606
975
  def package(self):
607
976
  self.chdir(self.conan_install)
977
+ if misc.conan_v2():
978
+ self.package_v2()
979
+ return
608
980
  self.run_command(f"sudo rm -rf {self.config.rootfs_path}")
609
981
  os.makedirs(self.config.rootfs_path)
610
982
  profile, _ = self.get_profile_config()
@@ -617,14 +989,8 @@ class TaskClass(Task):
617
989
 
618
990
  self.copy_components(comps, profile)
619
991
 
620
- # 运行组件定制化文件
621
- for comp in comps:
622
- cust = os.path.join(self.conan_install, comp, "include", "customization.py")
623
- if not os.path.isfile(cust):
624
- continue
625
- self.info(f"开始执行 {comp}/include/customization.py")
626
- post = ComponentPost(self.config, os.path.join(self.conan_install, comp), profile)
627
- post.post_work(self.config.rootfs_path, "post_image")
992
+ self.chdir(self.config.rootfs_path)
993
+ self._component_cust_action("post_image")
628
994
 
629
995
  self.chdir(self.config.rootfs_path)
630
996
  self.package_info_gen(f"{self.conan_source}/all/openubmc.bundle", "etc/package_info")
@@ -700,6 +1066,8 @@ class TaskClass(Task):
700
1066
  self.mkdir_work_path()
701
1067
  self.tools.clean_locks()
702
1068
  self.deploy()
1069
+ if self.only_manifest_yml:
1070
+ return
703
1071
  self.package_lock()
704
1072
  if not self.skip_package:
705
1073
  self.package()