lbkit 0.8.4__tar.gz → 0.8.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. lbkit-0.8.5/MANIFEST.in +3 -0
  2. {lbkit-0.8.4/lbkit.egg-info → lbkit-0.8.5}/PKG-INFO +1 -1
  3. lbkit-0.8.5/lbkit/__init__.py +2 -0
  4. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/cli.py +11 -26
  5. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/codegen/template/server.c.mako +1 -1
  6. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/log.py +1 -0
  7. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/misc.py +1 -0
  8. {lbkit-0.8.4/lbkit/integration → lbkit-0.8.5/lbkit/tasks}/config.py +40 -3
  9. lbkit-0.8.5/lbkit/tasks/executor.py +256 -0
  10. {lbkit-0.8.4/lbkit/integration → lbkit-0.8.5/lbkit/tasks}/task.py +18 -6
  11. lbkit-0.8.4/lbkit/integration/build_image.py → lbkit-0.8.5/lbkit/tasks/task_build_image.py +4 -6
  12. lbkit-0.8.4/lbkit/integration/build_manifest.py → lbkit-0.8.5/lbkit/tasks/task_build_manifest.py +4 -4
  13. lbkit-0.8.4/lbkit/integration/build_prepare.py → lbkit-0.8.5/lbkit/tasks/task_build_prepare.py +4 -4
  14. lbkit-0.8.4/lbkit/integration/build_rootfs.py → lbkit-0.8.5/lbkit/tasks/task_build_rootfs.py +4 -5
  15. lbkit-0.8.5/lbkit/tasks/task_test.py +29 -0
  16. lbkit-0.8.5/lbkit/tasks/test.py +40 -0
  17. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/tools.py +0 -1
  18. lbkit-0.8.5/lbkit/utils/env_detector.py +38 -0
  19. {lbkit-0.8.4 → lbkit-0.8.5/lbkit.egg-info}/PKG-INFO +1 -1
  20. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit.egg-info/SOURCES.txt +13 -9
  21. lbkit-0.8.4/MANIFEST.in +0 -3
  22. lbkit-0.8.4/lbkit/__init__.py +0 -2
  23. {lbkit-0.8.4 → lbkit-0.8.5}/AUTHORS +0 -0
  24. {lbkit-0.8.4 → lbkit-0.8.5}/LICENSE +0 -0
  25. {lbkit-0.8.4 → lbkit-0.8.5}/README.md +0 -0
  26. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/build_conan_parallel.py +0 -0
  27. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/ci_robot/__init__.py +0 -0
  28. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/ci_robot/gitee.py +0 -0
  29. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/codegen/__init__.py +0 -0
  30. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/codegen/codegen.py +0 -0
  31. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/codegen/ctype_defination.py +0 -0
  32. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/codegen/idf_interface.py +0 -0
  33. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/codegen/renderer.py +0 -0
  34. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/codegen/template/client.c.mako +0 -0
  35. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/codegen/template/client.h.mako +0 -0
  36. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/codegen/template/interface.c.mako +0 -0
  37. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/codegen/template/interface.introspect.xml.mako +0 -0
  38. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/codegen/template/public.c.mako +0 -0
  39. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/codegen/template/public.h.mako +0 -0
  40. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/codegen/template/server.h.mako +0 -0
  41. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/component/__init__.py +0 -0
  42. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/component/arg_parser.py +0 -0
  43. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/component/build.py +0 -0
  44. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/component/template/conanbase.mako +0 -0
  45. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/component/template/deploy.mako +0 -0
  46. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/component/test.py +0 -0
  47. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/errors.py +0 -0
  48. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/helper.py +0 -0
  49. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/lbkit.py +0 -0
  50. {lbkit-0.8.4/lbkit/integration → lbkit-0.8.5/lbkit/tasks}/__init__.py +0 -0
  51. {lbkit-0.8.4/lbkit/integration → lbkit-0.8.5/lbkit/tasks}/template/conanfile.py.mako +0 -0
  52. {lbkit-0.8.4/lbkit/integration → lbkit-0.8.5/lbkit/tasks}/template/rootfs.py.mako +0 -0
  53. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/utils/__init__.py +0 -0
  54. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/utils/images/__init__.py +0 -0
  55. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/utils/images/emmc.py +0 -0
  56. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit.egg-info/dependency_links.txt +0 -0
  57. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit.egg-info/entry_points.txt +0 -0
  58. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit.egg-info/requires.txt +0 -0
  59. {lbkit-0.8.4 → lbkit-0.8.5}/lbkit.egg-info/top_level.txt +0 -0
  60. {lbkit-0.8.4 → lbkit-0.8.5}/setup.cfg +0 -0
  61. {lbkit-0.8.4 → lbkit-0.8.5}/setup.py +0 -0
  62. {lbkit-0.8.4 → lbkit-0.8.5}/test/__init__.py +0 -0
  63. {lbkit-0.8.4 → lbkit-0.8.5}/test/test_codegen.py +0 -0
  64. {lbkit-0.8.4 → lbkit-0.8.5}/test/test_helper.py +0 -0
@@ -0,0 +1,3 @@
1
+ recursive-include lbkit/codegen/template *.mako
2
+ recursive-include lbkit/tasks/template *.mako
3
+ recursive-include lbkit/component/template *.mako
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: lbkit
3
- Version: 0.8.4
3
+ Version: 0.8.5
4
4
  Summary: Tools provided by litebmc.com
5
5
  Home-page: https://www.litebmc.com
6
6
  Author: xuhj@litebmc.com
@@ -0,0 +1,2 @@
1
+
2
+ __version__ = '0.8.5'
@@ -10,16 +10,13 @@ from lbkit import __version__ as client_version
10
10
  from lbkit.codegen.codegen import CodeGen
11
11
  from lbkit.component.build import BuildComponent
12
12
  from lbkit.component.test import TestComponent
13
- from lbkit.integration.build_manifest import BuildManifest
14
- from lbkit.integration.build_rootfs import BuildRootfs
15
- from lbkit.integration.build_prepare import BuildPrepare
16
- from lbkit.integration.build_image import BuildImage
13
+ from lbkit.tasks.executor import Executor
17
14
  from lbkit.component.arg_parser import ArgParser
18
- from lbkit.integration.config import Config
19
15
  from lbkit.ci_robot.gitee import Gitee
20
16
  from lbkit.log import Logger
21
17
  from lbkit import misc
22
18
  from lbkit import errors
19
+ from lbkit.utils.env_detector import EnvDetector
23
20
 
24
21
  log = Logger("cli")
25
22
 
@@ -80,26 +77,14 @@ class Command(object):
80
77
 
81
78
  组件需要支持多种跨平台构建场景,典型的包括DT(X86-64)、交叉编译(arm64)
82
79
  """
83
- arg_parser = ArgParser.new()
84
- build = BuildComponent(arg_parser, sys.argv[2:])
85
- build.run()
86
-
87
- # create product package
88
- def create(self, *args):
89
- """
90
- 构建组件.
91
-
92
- 组件需要支持多种跨平台构建场景,典型的包括DT(X86-64)、交叉编译(arm64)
93
- """
94
- cfg = Config(sys.argv[2:])
95
- build = BuildPrepare(cfg, "create_pre")
96
- build.run()
97
- build = BuildManifest(cfg, "create_run")
98
- build.run()
99
- build = BuildRootfs(cfg, "create_rootfs")
100
- build.run()
101
- build = BuildImage(cfg, "create_image")
102
- build.run()
80
+ env = EnvDetector()
81
+ if env.component:
82
+ arg_parser = ArgParser.new()
83
+ build = BuildComponent(arg_parser, sys.argv[2:])
84
+ build.run()
85
+ elif env.manifest:
86
+ exe = Executor(env)
87
+ exe.run()
103
88
 
104
89
  def gitee(self, *args):
105
90
  """
@@ -126,7 +111,7 @@ class Command(object):
126
111
  """
127
112
  grps = [("Code Generate commands", ["gen"]),
128
113
  ("Build Component commands", ["new", "build", "test"]),
129
- ("Build Product commands", ["create"]),
114
+ ("Build Product commands", ["build"]),
130
115
  ("Misc commands", ["help"]),
131
116
  ("CI Robot commands", ["gitee"])
132
117
  ]
@@ -86,7 +86,7 @@ static LBInterface _${class_name}_interface = {
86
86
  % if intf.plugin.install_dir:
87
87
  .plugin_dir = "${intf.plugin.install_dir}",
88
88
  % else:
89
- .plugin_dir = NULL,
89
+ .plugin_dir = "/opt/litebmc/plugins/${class_name}",
90
90
  % endif
91
91
  #endif
92
92
  };
@@ -12,6 +12,7 @@ class Logger(logging.getLoggerClass()):
12
12
  self.logenv = os.environ.get("LOG")
13
13
  if self.logenv is None:
14
14
  formatter = logging.Formatter('%(message)s')
15
+ self.setLevel(logging.INFO)
15
16
  else:
16
17
  formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
17
18
  if self.logenv == "info":
@@ -11,6 +11,7 @@ from jsonschema import validate, ValidationError
11
11
  from lbkit.errors import PackageConfigException, HttpRequestException
12
12
 
13
13
  LOG_DIR = "/tmp/lbkit/log"
14
+ TARGETS_DIR = "/usr/share/litebmc/targets"
14
15
 
15
16
 
16
17
  class Color(object):
@@ -2,7 +2,7 @@
2
2
  import argparse
3
3
  import os, sys
4
4
  from lbkit.log import Logger
5
- from lbkit.misc import load_yml_with_json_schema_validate
5
+ from lbkit.misc import load_yml_with_json_schema_validate, TARGETS_DIR
6
6
 
7
7
  log = Logger("build_config")
8
8
 
@@ -38,12 +38,14 @@ class Config(object):
38
38
  # conan.lock options
39
39
  self.using_lockfile = args.lockfile
40
40
  self.update_lockfile = args.update_lockfile
41
+ self.target = args.target
41
42
 
42
43
  # 设置并创建构建所需目录
43
44
  log.info("Work dir: %s", self.work_dir)
44
45
  self.code_path = os.getcwd()
45
46
  self.temp_path = os.path.join(self.code_path, ".temp")
46
47
  self.output_path = os.path.join(self.temp_path, "output")
48
+ self.download_path = os.path.join(self.temp_path, "download")
47
49
  self.tool_path = os.path.join(self.temp_path, "tools")
48
50
  # conan组件打包目录
49
51
  self.conan_install = []
@@ -56,21 +58,38 @@ class Config(object):
56
58
  os.makedirs(self.temp_path, exist_ok=True)
57
59
  os.makedirs(self.tool_path, exist_ok=True)
58
60
  os.makedirs(self.output_path, exist_ok=True)
61
+ os.makedirs(self.download_path, exist_ok=True)
59
62
  # 制作rootfs时需要strip镜像,所以需要单独指定stip路径
60
63
  self.strip = "strip"
61
64
 
65
+ @staticmethod
66
+ def target_list():
67
+ targets = {}
68
+ dirname = os.path.join(TARGETS_DIR)
69
+ for file in os.listdir(dirname):
70
+ if not file.endswith(".yml"):
71
+ continue
72
+ tgt_file = os.path.join(TARGETS_DIR, file)
73
+ targets[file] = tgt_file
74
+ return targets
75
+
62
76
  @staticmethod
63
77
  def arg_parser():
64
78
  """返回配置项支持的参数"""
65
- parser = argparse.ArgumentParser(description="Build LiteBMC")
79
+ parser = argparse.ArgumentParser(description="Build LiteBMC", formatter_class=argparse.RawTextHelpFormatter)
66
80
  parser.add_argument("-m", "--manifest", help="Specify the manifest.yml, ignored when -l is specified.", default="./manifest.yml")
67
81
  parser.add_argument("-s", "--from_source", help="Build from source", action="store_true")
68
82
  parser.add_argument("-pr", "--profile", help="Apply the specified profile to the host machine", default="litebmc.ini")
69
83
  parser.add_argument("-pr:b", "--profile_build", help="Apply the specified profile to the build machine", default="default")
70
- parser.add_argument("-t", "--build_type", type=str, choices=['debug', 'release', 'minsize'], help="Set the build type", default="debug")
84
+ parser.add_argument("-bt", "--build_type", type=str, choices=['debug', 'release', 'minsize'], help="Set the build type", default="debug")
71
85
  parser.add_argument("-r", "--remote", help="specified conan server", default="litebmc")
72
86
  parser.add_argument("-l", "--lockfile", help="using conan.lock", action="store_true")
73
87
  parser.add_argument("-ul", "--update_lockfile", help="update conan.lock", action="store_true")
88
+ targets = Config.target_list()
89
+ target_help = "build target:"
90
+ for tgt, _ in targets.items():
91
+ target_help += "\n* " + tgt[:-4]
92
+ parser.add_argument("-t", "--target", help=target_help, default="default")
74
93
  return parser
75
94
 
76
95
  def get_manifest_config(self, key: str, default=None):
@@ -88,4 +107,22 @@ class Config(object):
88
107
  template = {}
89
108
  template["code_path"] = self.code_path
90
109
  template["temp_path"] = self.temp_path
110
+ template["download_path"] = os.path.join(self.download_path)
91
111
  return load_yml_with_json_schema_validate(self.manifest, "/usr/share/litebmc/schema/pdf.v1.json", **template)
112
+
113
+ def set_build_type(self, value):
114
+ self.build_type = value
115
+
116
+ def deal_conf(self, config_dict):
117
+ """
118
+ 处理Target级别的配置"target_config"
119
+ 当Config类有set_xxx类方法时,则可以在target文件中配置xxx
120
+ """
121
+ if not config_dict or not isinstance(config_dict, dict):
122
+ return
123
+ for key, conf in config_dict.items():
124
+ try:
125
+ method = getattr(self, f"set_{key}")
126
+ method(conf)
127
+ except Exception as e:
128
+ raise Exception(f"目标 config 无效配置: {key}") from e
@@ -0,0 +1,256 @@
1
+ #! /usr/bin/env python3
2
+
3
+ import importlib
4
+ import os
5
+ import time
6
+ import sys
7
+
8
+ from multiprocessing import Process
9
+ from multiprocessing import Manager
10
+ from lbkit.errors import LiteBmcException
11
+ from lbkit.tools import Tools
12
+ from lbkit.tasks.config import Config
13
+ from lbkit.utils.env_detector import EnvDetector
14
+ import lbkit.misc as misc
15
+ from lbkit.misc import load_yml_with_json_schema_validate
16
+
17
+ # 任务失败状态
18
+ TASK_STATUS_FAILED = "Failed"
19
+ TASK_STATUS_SUCCED = "succed"
20
+ TASK_STATUS_EXCEPT = "Except"
21
+ TASK_STATUS_RUNNING = "Runing"
22
+ tool = Tools("executor")
23
+ log = tool.log
24
+ manager = Manager()
25
+ status_dict = manager.dict()
26
+ status_lock = manager.Lock()
27
+
28
+
29
+ def wait_finish(target_name, wait_list):
30
+ """
31
+ 等待任务结束
32
+ """
33
+ if not wait_list:
34
+ return True
35
+ start_time = time.time()
36
+ cnt = 0
37
+ while True:
38
+ finish = True
39
+ time.sleep(0.1)
40
+ for work_name in wait_list:
41
+ cur_time = time.time()
42
+ key = target_name + "/" + work_name
43
+ status = status_dict.get(key)
44
+ if status is None:
45
+ log.warn(f"等待不存在的任务{key}。如果要等待一个任务,这个任务必须在当前任务之前运行,否则触发异常")
46
+ return False
47
+ if status == TASK_STATUS_SUCCED:
48
+ continue
49
+ if status == TASK_STATUS_FAILED or status == TASK_STATUS_EXCEPT:
50
+ return False
51
+ finish = False
52
+ # 每等待60s打印一次日志
53
+ if int(cur_time - start_time) >= 60:
54
+ start_time = time.time()
55
+ cnt += 60
56
+ log.info("目标 {} 正在等待任务: {}, 当前已等待 {} 秒".format(target_name, work_name, cnt))
57
+ break
58
+ if finish:
59
+ return True
60
+
61
+
62
+ class TaskExecutor():
63
+ '''
64
+ '''
65
+ def __init__(self, target_name, work, config: Config):
66
+ super().__init__()
67
+ self.work = work
68
+ self.config: Config = config
69
+ self.target_name = target_name
70
+ self.work_name = self.work.get("task", "")
71
+ self.status_key = target_name + "/" + self.work_name
72
+ chunks = self.work_name.split(".", -1)
73
+ if len(chunks) == 1:
74
+ self.task_path = ""
75
+ self.work_name = chunks[1]
76
+ else:
77
+ self.task_path = "lbkit.tasks." + chunks[0]
78
+ self.work_name = chunks[1]
79
+ self.exception = None
80
+
81
+ def load_class(self):
82
+ if not self.task_path:
83
+ return None
84
+ log.debug("工作路径: {}".format(self.task_path))
85
+ work_py_file = importlib.import_module(self.task_path)
86
+ return getattr(work_py_file, "TaskClass")
87
+
88
+ def run(self):
89
+ '''
90
+ 功能描述:执行任务
91
+ '''
92
+ work_name = self.work_name
93
+ log.debug(f"任务{self.status_key}已就绪")
94
+ ret = wait_finish(self.target_name, self.work.get("wait"))
95
+ if not ret:
96
+ log.debug(f"任务{self.status_key}等待的其它任务发生错误")
97
+ return -1
98
+ work_class = self.load_class()
99
+ # 如果未指定类时,不需要执行
100
+ if work_class is not None:
101
+ work_x = work_class(self.config, work_name)
102
+ # work配置项和target配置项
103
+ work_config = self.work.get("config")
104
+ work_x.deal_conf(work_config)
105
+ with status_lock:
106
+ status = status_dict.get(self.status_key)
107
+ if status is None:
108
+ status_dict[self.status_key] = TASK_STATUS_RUNNING
109
+ if status is None:
110
+ # 创建进程并且等待完成或超时
111
+ ret = work_x.run()
112
+ if ret is not None and ret != 0:
113
+ return -1
114
+ else:
115
+ # 不需要创建进程,等待任务执行完成即可
116
+ wait_list = []
117
+ wait_list.append(work_name)
118
+ ret = wait_finish(self.target_name, wait_list)
119
+ if not ret:
120
+ log.debug(f"任务{self.status_key}等待的其它任务发生错误")
121
+ return -1
122
+
123
+ log.debug(f"任务 {work_name} 开始安装步骤")
124
+
125
+ # 创建子任务
126
+ ret = exec_works(self.work.get("subtasks", []), self.config, os.cpu_count())
127
+ ret = ret and exec_works(self.work.get("seqtasks", []), self.config, 1)
128
+ if not ret:
129
+ status_dict[self.status_key] = TASK_STATUS_FAILED
130
+ return -1
131
+
132
+ log.success(f"任务 {work_name} 完成")
133
+ with status_lock:
134
+ status_dict[self.status_key] = TASK_STATUS_SUCCED
135
+ return 0
136
+
137
+ def task_handler(te:TaskExecutor):
138
+ try:
139
+ ret = te.run()
140
+ except Exception as e:
141
+ log.error(f"Task {te.status_key} exit with exceiption: {str(e)}")
142
+ with status_lock:
143
+ status_dict[te.status_key] = TASK_STATUS_EXCEPT
144
+ return -1
145
+ if ret != 0:
146
+ with status_lock:
147
+ status_dict[te.status_key] = TASK_STATUS_FAILED
148
+ return -1
149
+ else:
150
+ with status_lock:
151
+ status_dict[te.status_key] = TASK_STATUS_SUCCED
152
+ return 0
153
+
154
+
155
+ class TaskInfo():
156
+ def __init__(self, te: TaskExecutor, proc: Process):
157
+ self.te = te
158
+ self.proc = proc
159
+
160
+
161
+ def wait_tasks(tasks: dict[str, TaskInfo], alow_processes_alive=0):
162
+ while True:
163
+ new_results = {}
164
+ cnt = 0
165
+ killall = False
166
+ for key, ti in tasks.items():
167
+ if ti.proc.is_alive():
168
+ new_results[key] = ti
169
+ cnt += 1
170
+ continue
171
+ with status_lock:
172
+ status = status_dict.get(ti.te.status_key)
173
+ if status == TASK_STATUS_EXCEPT:
174
+ killall = True
175
+ break
176
+ if killall:
177
+ for _, ti in tasks.items():
178
+ if ti.proc.is_alive():
179
+ ti.proc.terminate()
180
+ return None, False
181
+ tasks = new_results
182
+ if cnt > 0 and cnt >= alow_processes_alive:
183
+ time.sleep(0.1)
184
+ else:
185
+ return new_results, True
186
+
187
+ def exec_works(work_list, config, processes):
188
+ if not work_list:
189
+ return True
190
+ # 创建任务并等待完成
191
+ results: dict[str, TaskInfo] = {}
192
+ for work in work_list:
193
+ te = TaskExecutor(config.target, work, config)
194
+ result = Process(target=task_handler, args=(te, ))
195
+ results[te.status_key] = TaskInfo(te, result)
196
+ result.start()
197
+ results, ok = wait_tasks(results, processes)
198
+ if not ok:
199
+ return False
200
+ _, ok = wait_tasks(results, 0)
201
+ return ok
202
+
203
+
204
+ def target_executor(config):
205
+ log.info(f"创建新目标 {config.target} 构建计划表")
206
+ manifest_target = f"{config.code_path}/targets/{config.target}.yml"
207
+ lbkit_target = os.path.join(misc.TARGETS_DIR, config.target + ".yml")
208
+ if os.path.isfile(manifest_target):
209
+ target_file = manifest_target
210
+ elif os.path.isfile(lbkit_target):
211
+ target_file = lbkit_target
212
+ else:
213
+ raise Exception(f"构建目标文件 [target_]{config.target}.yml 不存在")
214
+
215
+ # 读取配置
216
+ work_list = load_yml_with_json_schema_validate(target_file, os.path.join(misc.TARGETS_DIR, "tdf.v1.json"))
217
+ target_cfg = work_list.get("config", {})
218
+ config.deal_conf(target_cfg)
219
+ environments = work_list.get("env", {})
220
+ for key, value in environments.items():
221
+ log.success(f"配置环境变量 {key}: {value}")
222
+ os.environ[key] = value
223
+ # 打印任务清单
224
+ log.debug(f"任务列表:{work_list}")
225
+ # 创建任务调度器
226
+ ret = exec_works(work_list.get("subtasks", []), config, os.cpu_count())
227
+ ret = ret and exec_works(work_list.get("seqtasks", []), config, 1)
228
+ return ret
229
+
230
+
231
+ class Executor(object):
232
+ def __init__(self, env: EnvDetector):
233
+ if not env.manifest:
234
+ raise LiteBmcException("未找到manifest.yml配置文件,当前目录不是一个合法的产品配置仓")
235
+ os.chdir(env.manifest.folder)
236
+
237
+ def run(self):
238
+ target = ""
239
+ succ = False
240
+ try:
241
+ config = Config(sys.argv[2:])
242
+ target = config.target
243
+ succ = target_executor(config)
244
+ except Exception as e:
245
+ log.error(str(e))
246
+ raise Exception(f"任务 {target} 执行失败")
247
+ if succ:
248
+ log.success(f"任务 {target} 执行成功")
249
+ return 0
250
+ else:
251
+ raise Exception(f"任务 {target} 执行失败")
252
+
253
+ if __name__ == "__main__":
254
+ env = EnvDetector()
255
+ exec = Executor(env)
256
+ exec.run()
@@ -1,16 +1,12 @@
1
1
  """任务基础类"""
2
- import shutil
3
2
  import importlib
4
- import os
5
- import hashlib
6
- import requests
7
- import yaml
3
+ import traceback
8
4
  from multiprocessing import Process
9
5
  from lbkit.log import Logger
10
6
  from lbkit.tools import Tools
11
7
  from lbkit.misc import load_yml_with_json_schema_validate
12
8
 
13
- from lbkit.integration.config import Config
9
+ from lbkit.tasks.config import Config
14
10
 
15
11
  class ManifestValidateError(OSError):
16
12
  """Raised when validation manifest.yml failed."""
@@ -22,6 +18,7 @@ class Task(Process):
22
18
  self.log: Logger = Logger("task")
23
19
  self.tools: Tools = Tools(name)
24
20
  self.config: Config = config
21
+ self.name = name
25
22
 
26
23
  def install(self):
27
24
  """安装任务"""
@@ -56,3 +53,18 @@ class Task(Process):
56
53
  def load_manifest(self):
57
54
  """加载manifest.yml并验证schema文件"""
58
55
  return self.config.load_manifest()
56
+
57
+ def deal_conf(self, config_dict):
58
+ """
59
+ 处理每个Task的私有配置"work_config"
60
+ 当work类有set_xxx类方法时,则可以在target文件中配置xxx
61
+ """
62
+ if not config_dict:
63
+ return
64
+ for conf in config_dict:
65
+ try:
66
+ exist = getattr(self, f"set_{conf}")
67
+ val = config_dict.get(conf)
68
+ exist(val)
69
+ except Exception as e:
70
+ raise Exception(f"无效配置: {conf}, {e}") from e
@@ -1,17 +1,15 @@
1
1
  """环境准备"""
2
2
  import os
3
- import tarfile
4
- from lbkit.integration.config import Config
5
- from lbkit.integration.task import Task
3
+ from lbkit.tasks.config import Config
4
+ from lbkit.tasks.task import Task
6
5
  from lbkit.log import Logger
7
- from lbkit import errors
8
6
  from lbkit.utils.images.emmc import MakeImage as MekeEmmcImage
9
7
 
10
8
  log = Logger("build_image")
11
9
 
12
10
  src_cwd = os.path.split(os.path.realpath(__file__))[0]
13
11
 
14
- class BuildImage(Task):
12
+ class TaskClass(Task):
15
13
  @staticmethod
16
14
  def _trans_to_blk_1m(cfg: str):
17
15
  if cfg.endswith("K"):
@@ -43,5 +41,5 @@ class BuildImage(Task):
43
41
 
44
42
  if __name__ == "__main__":
45
43
  config = Config()
46
- build = BuildImage(config)
44
+ build = TaskClass(config)
47
45
  build.run()
@@ -3,8 +3,8 @@ import os
3
3
  import shutil
4
4
  import json
5
5
  from mako.lookup import TemplateLookup
6
- from lbkit.integration.config import Config
7
- from lbkit.integration.task import Task
6
+ from lbkit.tasks.config import Config
7
+ from lbkit.tasks.task import Task
8
8
  from lbkit.log import Logger
9
9
  from lbkit.build_conan_parallel import BuildConanParallel
10
10
  from concurrent.futures import ThreadPoolExecutor
@@ -18,7 +18,7 @@ class ManifestValidateError(OSError):
18
18
 
19
19
  src_cwd = os.path.split(os.path.realpath(__file__))[0]
20
20
 
21
- class BuildManifest(Task):
21
+ class TaskClass(Task):
22
22
  """根据产品配置构建所有app,记录待安装应用路径到self.config.conan_install路径"""
23
23
  def __init__(self, cfg: Config, name: str):
24
24
  super().__init__(cfg, name)
@@ -159,5 +159,5 @@ class BuildManifest(Task):
159
159
 
160
160
  if __name__ == "__main__":
161
161
  config = Config()
162
- build = BuildManifest(config)
162
+ build = TaskClass(config)
163
163
  build.run()
@@ -3,8 +3,8 @@ import os
3
3
  import shutil
4
4
  import jinja2
5
5
  import configparser
6
- from lbkit.integration.config import Config
7
- from lbkit.integration.task import Task
6
+ from lbkit.tasks.config import Config
7
+ from lbkit.tasks.task import Task
8
8
  from lbkit.log import Logger
9
9
 
10
10
  log = Logger("product_prepare")
@@ -15,7 +15,7 @@ class ManifestValidateError(OSError):
15
15
 
16
16
  src_cwd = os.path.split(os.path.realpath(__file__))[0]
17
17
 
18
- class BuildPrepare(Task):
18
+ class TaskClass(Task):
19
19
  def load_conan_profile(self):
20
20
  profile = self.get_manifest_config("metadata/profile")
21
21
  if not os.path.isfile(profile):
@@ -55,5 +55,5 @@ class BuildPrepare(Task):
55
55
 
56
56
  if __name__ == "__main__":
57
57
  config = Config()
58
- build = BuildPrepare(config)
58
+ build = TaskClass(config)
59
59
  build.run()
@@ -1,9 +1,8 @@
1
1
  """完成rootfs镜像打包."""
2
2
  import os
3
3
  import shutil
4
- import tarfile
5
- from lbkit.integration.config import Config
6
- from lbkit.integration.task import Task
4
+ from lbkit.tasks.config import Config
5
+ from lbkit.tasks.task import Task
7
6
  from lbkit.log import Logger
8
7
  from lbkit import errors
9
8
 
@@ -12,7 +11,7 @@ log = Logger("build_rootfs")
12
11
  src_cwd = os.path.split(os.path.realpath(__file__))[0]
13
12
  IMG_FILE = "rootfs.img"
14
13
 
15
- class BuildRootfs(Task):
14
+ class TaskClass(Task):
16
15
  """构建rootfs镜像"""
17
16
  def do_permission(self, per_file: str):
18
17
  """完成组件制品赋权"""
@@ -158,5 +157,5 @@ class BuildRootfs(Task):
158
157
 
159
158
  if __name__ == "__main__":
160
159
  config = Config()
161
- build = BuildRootfs(config)
160
+ build = TaskClass(config)
162
161
  build.run()
@@ -0,0 +1,29 @@
1
+ """测试任务"""
2
+ import time
3
+
4
+ from lbkit.tasks.config import Config
5
+ from lbkit.tasks.task import Task
6
+
7
+ class TaskClass(Task):
8
+ def run(self):
9
+ print(f"task {self.name} start....")
10
+ time.sleep(2)
11
+ print(f"task {self.name} stop")
12
+
13
+ # def set_key1(self, val):
14
+ # print(f"Object {self.name} set key1: {val}")
15
+ # def set_key2(self, val):
16
+ # print(f"Object {self.name} set key2: {val}")
17
+ # def set_key3(self, val):
18
+ # print(f"Object {self.name} set key3: {val}")
19
+ # def set_key4(self, val):
20
+ # print(f"Object {self.name} set key4: {val}")
21
+ # def set_key5(self, val):
22
+ # print(f"Object {self.name} set key5: {val}")
23
+ # def set_key6(self, val):
24
+ # print(f"Object {self.name} set key6: {val}")
25
+
26
+ if __name__ == "__main__":
27
+ config = Config()
28
+ build = TaskClass(config)
29
+ build.run()
@@ -0,0 +1,40 @@
1
+ from multiprocessing import Pool
2
+ from multiprocessing.pool import ApplyResult
3
+ import time
4
+ from functools import partial
5
+
6
+ def square(number):
7
+ """计算一个数的平方"""
8
+ print(f"计算 {number} 的平方")
9
+ time.sleep(1) # 模拟耗时操作
10
+ if number % 10 == 9:
11
+ raise Exception("sssssssssss")
12
+ # return number * number
13
+
14
+ def task_error(pool, number, error):
15
+ print(f"Work {number} exist with error: {error}")
16
+ pool.terminate()
17
+
18
+ if __name__ == "__main__":
19
+ # 创建任务并等待完成
20
+ results: dict[str, ApplyResult] = {}
21
+ pool = Pool(3)
22
+ for work in range(100):
23
+ error_cb = partial(task_error, pool, work)
24
+ result = pool.apply_async(square, args=(work, ), error_callback=error_cb)
25
+ results[work] = result
26
+ # 检查任务结果
27
+ pool.close()
28
+ pool.join()
29
+ # print("xxxxxxxxxxxxxxxxxx")
30
+ # time.sleep(100)
31
+ for number, result in results.items():
32
+ try:
33
+ if not result.ready():
34
+ succ = False
35
+ else:
36
+ result.get()
37
+ except Exception as e:
38
+ print(f"任务{number}执行异常: {str(e)}")
39
+ succ = False
40
+
@@ -11,7 +11,6 @@ import hashlib
11
11
  import re
12
12
  from lbkit.log import Logger
13
13
  from lbkit import errors
14
- from lbkit import misc
15
14
  from lbkit.misc import Color
16
15
 
17
16
 
@@ -0,0 +1,38 @@
1
+ #!/usr/bin/env python3
2
+
3
+ import os
4
+ import configparser
5
+ from lbkit.tools import Tools
6
+
7
+ tools = Tools("EnvDetector")
8
+ log = tools.log
9
+
10
+
11
+ class LbkitComponent(object):
12
+ def __init__(self, folder, config: configparser.ConfigParser):
13
+ self.folder = os.path.realpath(folder)
14
+ self.config = config
15
+
16
+
17
+ class LbkitManifest(object):
18
+ def __init__(self, folder, config: configparser.ConfigParser):
19
+ self.folder = os.path.realpath(folder)
20
+ self.config = config
21
+
22
+
23
+ class EnvDetector(object):
24
+ def __init__(self):
25
+ """初始化"""
26
+ self.component: LbkitComponent = None
27
+ self.manifest: LbkitManifest = None
28
+ self.cwd = os.getcwd()
29
+ """探测环境"""
30
+ cwd = self.cwd
31
+ while cwd != "/":
32
+ if os.path.isfile(os.path.join(cwd, "manifest.yml")):
33
+ self.manifest = LbkitManifest(cwd, None)
34
+ return
35
+ if os.path.isfile(os.path.join(cwd, "mds/package.yml")):
36
+ self.component = LbkitComponent(cwd, None)
37
+ return
38
+ cwd = os.path.dirname(cwd)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: lbkit
3
- Version: 0.8.4
3
+ Version: 0.8.5
4
4
  Summary: Tools provided by litebmc.com
5
5
  Home-page: https://www.litebmc.com
6
6
  Author: xuhj@litebmc.com
@@ -39,16 +39,20 @@ lbkit/component/build.py
39
39
  lbkit/component/test.py
40
40
  lbkit/component/template/conanbase.mako
41
41
  lbkit/component/template/deploy.mako
42
- lbkit/integration/__init__.py
43
- lbkit/integration/build_image.py
44
- lbkit/integration/build_manifest.py
45
- lbkit/integration/build_prepare.py
46
- lbkit/integration/build_rootfs.py
47
- lbkit/integration/config.py
48
- lbkit/integration/task.py
49
- lbkit/integration/template/conanfile.py.mako
50
- lbkit/integration/template/rootfs.py.mako
42
+ lbkit/tasks/__init__.py
43
+ lbkit/tasks/config.py
44
+ lbkit/tasks/executor.py
45
+ lbkit/tasks/task.py
46
+ lbkit/tasks/task_build_image.py
47
+ lbkit/tasks/task_build_manifest.py
48
+ lbkit/tasks/task_build_prepare.py
49
+ lbkit/tasks/task_build_rootfs.py
50
+ lbkit/tasks/task_test.py
51
+ lbkit/tasks/test.py
52
+ lbkit/tasks/template/conanfile.py.mako
53
+ lbkit/tasks/template/rootfs.py.mako
51
54
  lbkit/utils/__init__.py
55
+ lbkit/utils/env_detector.py
52
56
  lbkit/utils/images/__init__.py
53
57
  lbkit/utils/images/emmc.py
54
58
  test/__init__.py
lbkit-0.8.4/MANIFEST.in DELETED
@@ -1,3 +0,0 @@
1
- recursive-include lbkit/codegen/template *.mako
2
- recursive-include lbkit/integration/template *.mako
3
- recursive-include lbkit/component/template *.mako
@@ -1,2 +0,0 @@
1
-
2
- __version__ = '0.8.4'
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes