lbkit 0.8.4__tar.gz → 0.8.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lbkit-0.8.5/MANIFEST.in +3 -0
- {lbkit-0.8.4/lbkit.egg-info → lbkit-0.8.5}/PKG-INFO +1 -1
- lbkit-0.8.5/lbkit/__init__.py +2 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/cli.py +11 -26
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/codegen/template/server.c.mako +1 -1
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/log.py +1 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/misc.py +1 -0
- {lbkit-0.8.4/lbkit/integration → lbkit-0.8.5/lbkit/tasks}/config.py +40 -3
- lbkit-0.8.5/lbkit/tasks/executor.py +256 -0
- {lbkit-0.8.4/lbkit/integration → lbkit-0.8.5/lbkit/tasks}/task.py +18 -6
- lbkit-0.8.4/lbkit/integration/build_image.py → lbkit-0.8.5/lbkit/tasks/task_build_image.py +4 -6
- lbkit-0.8.4/lbkit/integration/build_manifest.py → lbkit-0.8.5/lbkit/tasks/task_build_manifest.py +4 -4
- lbkit-0.8.4/lbkit/integration/build_prepare.py → lbkit-0.8.5/lbkit/tasks/task_build_prepare.py +4 -4
- lbkit-0.8.4/lbkit/integration/build_rootfs.py → lbkit-0.8.5/lbkit/tasks/task_build_rootfs.py +4 -5
- lbkit-0.8.5/lbkit/tasks/task_test.py +29 -0
- lbkit-0.8.5/lbkit/tasks/test.py +40 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/tools.py +0 -1
- lbkit-0.8.5/lbkit/utils/env_detector.py +38 -0
- {lbkit-0.8.4 → lbkit-0.8.5/lbkit.egg-info}/PKG-INFO +1 -1
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit.egg-info/SOURCES.txt +13 -9
- lbkit-0.8.4/MANIFEST.in +0 -3
- lbkit-0.8.4/lbkit/__init__.py +0 -2
- {lbkit-0.8.4 → lbkit-0.8.5}/AUTHORS +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/LICENSE +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/README.md +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/build_conan_parallel.py +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/ci_robot/__init__.py +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/ci_robot/gitee.py +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/codegen/__init__.py +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/codegen/codegen.py +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/codegen/ctype_defination.py +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/codegen/idf_interface.py +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/codegen/renderer.py +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/codegen/template/client.c.mako +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/codegen/template/client.h.mako +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/codegen/template/interface.c.mako +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/codegen/template/interface.introspect.xml.mako +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/codegen/template/public.c.mako +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/codegen/template/public.h.mako +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/codegen/template/server.h.mako +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/component/__init__.py +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/component/arg_parser.py +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/component/build.py +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/component/template/conanbase.mako +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/component/template/deploy.mako +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/component/test.py +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/errors.py +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/helper.py +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/lbkit.py +0 -0
- {lbkit-0.8.4/lbkit/integration → lbkit-0.8.5/lbkit/tasks}/__init__.py +0 -0
- {lbkit-0.8.4/lbkit/integration → lbkit-0.8.5/lbkit/tasks}/template/conanfile.py.mako +0 -0
- {lbkit-0.8.4/lbkit/integration → lbkit-0.8.5/lbkit/tasks}/template/rootfs.py.mako +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/utils/__init__.py +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/utils/images/__init__.py +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit/utils/images/emmc.py +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit.egg-info/dependency_links.txt +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit.egg-info/entry_points.txt +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit.egg-info/requires.txt +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/lbkit.egg-info/top_level.txt +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/setup.cfg +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/setup.py +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/test/__init__.py +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/test/test_codegen.py +0 -0
- {lbkit-0.8.4 → lbkit-0.8.5}/test/test_helper.py +0 -0
lbkit-0.8.5/MANIFEST.in
ADDED
|
@@ -10,16 +10,13 @@ from lbkit import __version__ as client_version
|
|
|
10
10
|
from lbkit.codegen.codegen import CodeGen
|
|
11
11
|
from lbkit.component.build import BuildComponent
|
|
12
12
|
from lbkit.component.test import TestComponent
|
|
13
|
-
from lbkit.
|
|
14
|
-
from lbkit.integration.build_rootfs import BuildRootfs
|
|
15
|
-
from lbkit.integration.build_prepare import BuildPrepare
|
|
16
|
-
from lbkit.integration.build_image import BuildImage
|
|
13
|
+
from lbkit.tasks.executor import Executor
|
|
17
14
|
from lbkit.component.arg_parser import ArgParser
|
|
18
|
-
from lbkit.integration.config import Config
|
|
19
15
|
from lbkit.ci_robot.gitee import Gitee
|
|
20
16
|
from lbkit.log import Logger
|
|
21
17
|
from lbkit import misc
|
|
22
18
|
from lbkit import errors
|
|
19
|
+
from lbkit.utils.env_detector import EnvDetector
|
|
23
20
|
|
|
24
21
|
log = Logger("cli")
|
|
25
22
|
|
|
@@ -80,26 +77,14 @@ class Command(object):
|
|
|
80
77
|
|
|
81
78
|
组件需要支持多种跨平台构建场景,典型的包括DT(X86-64)、交叉编译(arm64)
|
|
82
79
|
"""
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
组件需要支持多种跨平台构建场景,典型的包括DT(X86-64)、交叉编译(arm64)
|
|
93
|
-
"""
|
|
94
|
-
cfg = Config(sys.argv[2:])
|
|
95
|
-
build = BuildPrepare(cfg, "create_pre")
|
|
96
|
-
build.run()
|
|
97
|
-
build = BuildManifest(cfg, "create_run")
|
|
98
|
-
build.run()
|
|
99
|
-
build = BuildRootfs(cfg, "create_rootfs")
|
|
100
|
-
build.run()
|
|
101
|
-
build = BuildImage(cfg, "create_image")
|
|
102
|
-
build.run()
|
|
80
|
+
env = EnvDetector()
|
|
81
|
+
if env.component:
|
|
82
|
+
arg_parser = ArgParser.new()
|
|
83
|
+
build = BuildComponent(arg_parser, sys.argv[2:])
|
|
84
|
+
build.run()
|
|
85
|
+
elif env.manifest:
|
|
86
|
+
exe = Executor(env)
|
|
87
|
+
exe.run()
|
|
103
88
|
|
|
104
89
|
def gitee(self, *args):
|
|
105
90
|
"""
|
|
@@ -126,7 +111,7 @@ class Command(object):
|
|
|
126
111
|
"""
|
|
127
112
|
grps = [("Code Generate commands", ["gen"]),
|
|
128
113
|
("Build Component commands", ["new", "build", "test"]),
|
|
129
|
-
("Build Product commands", ["
|
|
114
|
+
("Build Product commands", ["build"]),
|
|
130
115
|
("Misc commands", ["help"]),
|
|
131
116
|
("CI Robot commands", ["gitee"])
|
|
132
117
|
]
|
|
@@ -12,6 +12,7 @@ class Logger(logging.getLoggerClass()):
|
|
|
12
12
|
self.logenv = os.environ.get("LOG")
|
|
13
13
|
if self.logenv is None:
|
|
14
14
|
formatter = logging.Formatter('%(message)s')
|
|
15
|
+
self.setLevel(logging.INFO)
|
|
15
16
|
else:
|
|
16
17
|
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
|
|
17
18
|
if self.logenv == "info":
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
import argparse
|
|
3
3
|
import os, sys
|
|
4
4
|
from lbkit.log import Logger
|
|
5
|
-
from lbkit.misc import load_yml_with_json_schema_validate
|
|
5
|
+
from lbkit.misc import load_yml_with_json_schema_validate, TARGETS_DIR
|
|
6
6
|
|
|
7
7
|
log = Logger("build_config")
|
|
8
8
|
|
|
@@ -38,12 +38,14 @@ class Config(object):
|
|
|
38
38
|
# conan.lock options
|
|
39
39
|
self.using_lockfile = args.lockfile
|
|
40
40
|
self.update_lockfile = args.update_lockfile
|
|
41
|
+
self.target = args.target
|
|
41
42
|
|
|
42
43
|
# 设置并创建构建所需目录
|
|
43
44
|
log.info("Work dir: %s", self.work_dir)
|
|
44
45
|
self.code_path = os.getcwd()
|
|
45
46
|
self.temp_path = os.path.join(self.code_path, ".temp")
|
|
46
47
|
self.output_path = os.path.join(self.temp_path, "output")
|
|
48
|
+
self.download_path = os.path.join(self.temp_path, "download")
|
|
47
49
|
self.tool_path = os.path.join(self.temp_path, "tools")
|
|
48
50
|
# conan组件打包目录
|
|
49
51
|
self.conan_install = []
|
|
@@ -56,21 +58,38 @@ class Config(object):
|
|
|
56
58
|
os.makedirs(self.temp_path, exist_ok=True)
|
|
57
59
|
os.makedirs(self.tool_path, exist_ok=True)
|
|
58
60
|
os.makedirs(self.output_path, exist_ok=True)
|
|
61
|
+
os.makedirs(self.download_path, exist_ok=True)
|
|
59
62
|
# 制作rootfs时需要strip镜像,所以需要单独指定stip路径
|
|
60
63
|
self.strip = "strip"
|
|
61
64
|
|
|
65
|
+
@staticmethod
|
|
66
|
+
def target_list():
|
|
67
|
+
targets = {}
|
|
68
|
+
dirname = os.path.join(TARGETS_DIR)
|
|
69
|
+
for file in os.listdir(dirname):
|
|
70
|
+
if not file.endswith(".yml"):
|
|
71
|
+
continue
|
|
72
|
+
tgt_file = os.path.join(TARGETS_DIR, file)
|
|
73
|
+
targets[file] = tgt_file
|
|
74
|
+
return targets
|
|
75
|
+
|
|
62
76
|
@staticmethod
|
|
63
77
|
def arg_parser():
|
|
64
78
|
"""返回配置项支持的参数"""
|
|
65
|
-
parser = argparse.ArgumentParser(description="Build LiteBMC")
|
|
79
|
+
parser = argparse.ArgumentParser(description="Build LiteBMC", formatter_class=argparse.RawTextHelpFormatter)
|
|
66
80
|
parser.add_argument("-m", "--manifest", help="Specify the manifest.yml, ignored when -l is specified.", default="./manifest.yml")
|
|
67
81
|
parser.add_argument("-s", "--from_source", help="Build from source", action="store_true")
|
|
68
82
|
parser.add_argument("-pr", "--profile", help="Apply the specified profile to the host machine", default="litebmc.ini")
|
|
69
83
|
parser.add_argument("-pr:b", "--profile_build", help="Apply the specified profile to the build machine", default="default")
|
|
70
|
-
parser.add_argument("-
|
|
84
|
+
parser.add_argument("-bt", "--build_type", type=str, choices=['debug', 'release', 'minsize'], help="Set the build type", default="debug")
|
|
71
85
|
parser.add_argument("-r", "--remote", help="specified conan server", default="litebmc")
|
|
72
86
|
parser.add_argument("-l", "--lockfile", help="using conan.lock", action="store_true")
|
|
73
87
|
parser.add_argument("-ul", "--update_lockfile", help="update conan.lock", action="store_true")
|
|
88
|
+
targets = Config.target_list()
|
|
89
|
+
target_help = "build target:"
|
|
90
|
+
for tgt, _ in targets.items():
|
|
91
|
+
target_help += "\n* " + tgt[:-4]
|
|
92
|
+
parser.add_argument("-t", "--target", help=target_help, default="default")
|
|
74
93
|
return parser
|
|
75
94
|
|
|
76
95
|
def get_manifest_config(self, key: str, default=None):
|
|
@@ -88,4 +107,22 @@ class Config(object):
|
|
|
88
107
|
template = {}
|
|
89
108
|
template["code_path"] = self.code_path
|
|
90
109
|
template["temp_path"] = self.temp_path
|
|
110
|
+
template["download_path"] = os.path.join(self.download_path)
|
|
91
111
|
return load_yml_with_json_schema_validate(self.manifest, "/usr/share/litebmc/schema/pdf.v1.json", **template)
|
|
112
|
+
|
|
113
|
+
def set_build_type(self, value):
|
|
114
|
+
self.build_type = value
|
|
115
|
+
|
|
116
|
+
def deal_conf(self, config_dict):
|
|
117
|
+
"""
|
|
118
|
+
处理Target级别的配置"target_config"
|
|
119
|
+
当Config类有set_xxx类方法时,则可以在target文件中配置xxx
|
|
120
|
+
"""
|
|
121
|
+
if not config_dict or not isinstance(config_dict, dict):
|
|
122
|
+
return
|
|
123
|
+
for key, conf in config_dict.items():
|
|
124
|
+
try:
|
|
125
|
+
method = getattr(self, f"set_{key}")
|
|
126
|
+
method(conf)
|
|
127
|
+
except Exception as e:
|
|
128
|
+
raise Exception(f"目标 config 无效配置: {key}") from e
|
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
#! /usr/bin/env python3
|
|
2
|
+
|
|
3
|
+
import importlib
|
|
4
|
+
import os
|
|
5
|
+
import time
|
|
6
|
+
import sys
|
|
7
|
+
|
|
8
|
+
from multiprocessing import Process
|
|
9
|
+
from multiprocessing import Manager
|
|
10
|
+
from lbkit.errors import LiteBmcException
|
|
11
|
+
from lbkit.tools import Tools
|
|
12
|
+
from lbkit.tasks.config import Config
|
|
13
|
+
from lbkit.utils.env_detector import EnvDetector
|
|
14
|
+
import lbkit.misc as misc
|
|
15
|
+
from lbkit.misc import load_yml_with_json_schema_validate
|
|
16
|
+
|
|
17
|
+
# 任务失败状态
|
|
18
|
+
TASK_STATUS_FAILED = "Failed"
|
|
19
|
+
TASK_STATUS_SUCCED = "succed"
|
|
20
|
+
TASK_STATUS_EXCEPT = "Except"
|
|
21
|
+
TASK_STATUS_RUNNING = "Runing"
|
|
22
|
+
tool = Tools("executor")
|
|
23
|
+
log = tool.log
|
|
24
|
+
manager = Manager()
|
|
25
|
+
status_dict = manager.dict()
|
|
26
|
+
status_lock = manager.Lock()
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def wait_finish(target_name, wait_list):
|
|
30
|
+
"""
|
|
31
|
+
等待任务结束
|
|
32
|
+
"""
|
|
33
|
+
if not wait_list:
|
|
34
|
+
return True
|
|
35
|
+
start_time = time.time()
|
|
36
|
+
cnt = 0
|
|
37
|
+
while True:
|
|
38
|
+
finish = True
|
|
39
|
+
time.sleep(0.1)
|
|
40
|
+
for work_name in wait_list:
|
|
41
|
+
cur_time = time.time()
|
|
42
|
+
key = target_name + "/" + work_name
|
|
43
|
+
status = status_dict.get(key)
|
|
44
|
+
if status is None:
|
|
45
|
+
log.warn(f"等待不存在的任务{key}。如果要等待一个任务,这个任务必须在当前任务之前运行,否则触发异常")
|
|
46
|
+
return False
|
|
47
|
+
if status == TASK_STATUS_SUCCED:
|
|
48
|
+
continue
|
|
49
|
+
if status == TASK_STATUS_FAILED or status == TASK_STATUS_EXCEPT:
|
|
50
|
+
return False
|
|
51
|
+
finish = False
|
|
52
|
+
# 每等待60s打印一次日志
|
|
53
|
+
if int(cur_time - start_time) >= 60:
|
|
54
|
+
start_time = time.time()
|
|
55
|
+
cnt += 60
|
|
56
|
+
log.info("目标 {} 正在等待任务: {}, 当前已等待 {} 秒".format(target_name, work_name, cnt))
|
|
57
|
+
break
|
|
58
|
+
if finish:
|
|
59
|
+
return True
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class TaskExecutor():
|
|
63
|
+
'''
|
|
64
|
+
'''
|
|
65
|
+
def __init__(self, target_name, work, config: Config):
|
|
66
|
+
super().__init__()
|
|
67
|
+
self.work = work
|
|
68
|
+
self.config: Config = config
|
|
69
|
+
self.target_name = target_name
|
|
70
|
+
self.work_name = self.work.get("task", "")
|
|
71
|
+
self.status_key = target_name + "/" + self.work_name
|
|
72
|
+
chunks = self.work_name.split(".", -1)
|
|
73
|
+
if len(chunks) == 1:
|
|
74
|
+
self.task_path = ""
|
|
75
|
+
self.work_name = chunks[1]
|
|
76
|
+
else:
|
|
77
|
+
self.task_path = "lbkit.tasks." + chunks[0]
|
|
78
|
+
self.work_name = chunks[1]
|
|
79
|
+
self.exception = None
|
|
80
|
+
|
|
81
|
+
def load_class(self):
|
|
82
|
+
if not self.task_path:
|
|
83
|
+
return None
|
|
84
|
+
log.debug("工作路径: {}".format(self.task_path))
|
|
85
|
+
work_py_file = importlib.import_module(self.task_path)
|
|
86
|
+
return getattr(work_py_file, "TaskClass")
|
|
87
|
+
|
|
88
|
+
def run(self):
|
|
89
|
+
'''
|
|
90
|
+
功能描述:执行任务
|
|
91
|
+
'''
|
|
92
|
+
work_name = self.work_name
|
|
93
|
+
log.debug(f"任务{self.status_key}已就绪")
|
|
94
|
+
ret = wait_finish(self.target_name, self.work.get("wait"))
|
|
95
|
+
if not ret:
|
|
96
|
+
log.debug(f"任务{self.status_key}等待的其它任务发生错误")
|
|
97
|
+
return -1
|
|
98
|
+
work_class = self.load_class()
|
|
99
|
+
# 如果未指定类时,不需要执行
|
|
100
|
+
if work_class is not None:
|
|
101
|
+
work_x = work_class(self.config, work_name)
|
|
102
|
+
# work配置项和target配置项
|
|
103
|
+
work_config = self.work.get("config")
|
|
104
|
+
work_x.deal_conf(work_config)
|
|
105
|
+
with status_lock:
|
|
106
|
+
status = status_dict.get(self.status_key)
|
|
107
|
+
if status is None:
|
|
108
|
+
status_dict[self.status_key] = TASK_STATUS_RUNNING
|
|
109
|
+
if status is None:
|
|
110
|
+
# 创建进程并且等待完成或超时
|
|
111
|
+
ret = work_x.run()
|
|
112
|
+
if ret is not None and ret != 0:
|
|
113
|
+
return -1
|
|
114
|
+
else:
|
|
115
|
+
# 不需要创建进程,等待任务执行完成即可
|
|
116
|
+
wait_list = []
|
|
117
|
+
wait_list.append(work_name)
|
|
118
|
+
ret = wait_finish(self.target_name, wait_list)
|
|
119
|
+
if not ret:
|
|
120
|
+
log.debug(f"任务{self.status_key}等待的其它任务发生错误")
|
|
121
|
+
return -1
|
|
122
|
+
|
|
123
|
+
log.debug(f"任务 {work_name} 开始安装步骤")
|
|
124
|
+
|
|
125
|
+
# 创建子任务
|
|
126
|
+
ret = exec_works(self.work.get("subtasks", []), self.config, os.cpu_count())
|
|
127
|
+
ret = ret and exec_works(self.work.get("seqtasks", []), self.config, 1)
|
|
128
|
+
if not ret:
|
|
129
|
+
status_dict[self.status_key] = TASK_STATUS_FAILED
|
|
130
|
+
return -1
|
|
131
|
+
|
|
132
|
+
log.success(f"任务 {work_name} 完成")
|
|
133
|
+
with status_lock:
|
|
134
|
+
status_dict[self.status_key] = TASK_STATUS_SUCCED
|
|
135
|
+
return 0
|
|
136
|
+
|
|
137
|
+
def task_handler(te:TaskExecutor):
|
|
138
|
+
try:
|
|
139
|
+
ret = te.run()
|
|
140
|
+
except Exception as e:
|
|
141
|
+
log.error(f"Task {te.status_key} exit with exceiption: {str(e)}")
|
|
142
|
+
with status_lock:
|
|
143
|
+
status_dict[te.status_key] = TASK_STATUS_EXCEPT
|
|
144
|
+
return -1
|
|
145
|
+
if ret != 0:
|
|
146
|
+
with status_lock:
|
|
147
|
+
status_dict[te.status_key] = TASK_STATUS_FAILED
|
|
148
|
+
return -1
|
|
149
|
+
else:
|
|
150
|
+
with status_lock:
|
|
151
|
+
status_dict[te.status_key] = TASK_STATUS_SUCCED
|
|
152
|
+
return 0
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
class TaskInfo():
|
|
156
|
+
def __init__(self, te: TaskExecutor, proc: Process):
|
|
157
|
+
self.te = te
|
|
158
|
+
self.proc = proc
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def wait_tasks(tasks: dict[str, TaskInfo], alow_processes_alive=0):
|
|
162
|
+
while True:
|
|
163
|
+
new_results = {}
|
|
164
|
+
cnt = 0
|
|
165
|
+
killall = False
|
|
166
|
+
for key, ti in tasks.items():
|
|
167
|
+
if ti.proc.is_alive():
|
|
168
|
+
new_results[key] = ti
|
|
169
|
+
cnt += 1
|
|
170
|
+
continue
|
|
171
|
+
with status_lock:
|
|
172
|
+
status = status_dict.get(ti.te.status_key)
|
|
173
|
+
if status == TASK_STATUS_EXCEPT:
|
|
174
|
+
killall = True
|
|
175
|
+
break
|
|
176
|
+
if killall:
|
|
177
|
+
for _, ti in tasks.items():
|
|
178
|
+
if ti.proc.is_alive():
|
|
179
|
+
ti.proc.terminate()
|
|
180
|
+
return None, False
|
|
181
|
+
tasks = new_results
|
|
182
|
+
if cnt > 0 and cnt >= alow_processes_alive:
|
|
183
|
+
time.sleep(0.1)
|
|
184
|
+
else:
|
|
185
|
+
return new_results, True
|
|
186
|
+
|
|
187
|
+
def exec_works(work_list, config, processes):
|
|
188
|
+
if not work_list:
|
|
189
|
+
return True
|
|
190
|
+
# 创建任务并等待完成
|
|
191
|
+
results: dict[str, TaskInfo] = {}
|
|
192
|
+
for work in work_list:
|
|
193
|
+
te = TaskExecutor(config.target, work, config)
|
|
194
|
+
result = Process(target=task_handler, args=(te, ))
|
|
195
|
+
results[te.status_key] = TaskInfo(te, result)
|
|
196
|
+
result.start()
|
|
197
|
+
results, ok = wait_tasks(results, processes)
|
|
198
|
+
if not ok:
|
|
199
|
+
return False
|
|
200
|
+
_, ok = wait_tasks(results, 0)
|
|
201
|
+
return ok
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def target_executor(config):
|
|
205
|
+
log.info(f"创建新目标 {config.target} 构建计划表")
|
|
206
|
+
manifest_target = f"{config.code_path}/targets/{config.target}.yml"
|
|
207
|
+
lbkit_target = os.path.join(misc.TARGETS_DIR, config.target + ".yml")
|
|
208
|
+
if os.path.isfile(manifest_target):
|
|
209
|
+
target_file = manifest_target
|
|
210
|
+
elif os.path.isfile(lbkit_target):
|
|
211
|
+
target_file = lbkit_target
|
|
212
|
+
else:
|
|
213
|
+
raise Exception(f"构建目标文件 [target_]{config.target}.yml 不存在")
|
|
214
|
+
|
|
215
|
+
# 读取配置
|
|
216
|
+
work_list = load_yml_with_json_schema_validate(target_file, os.path.join(misc.TARGETS_DIR, "tdf.v1.json"))
|
|
217
|
+
target_cfg = work_list.get("config", {})
|
|
218
|
+
config.deal_conf(target_cfg)
|
|
219
|
+
environments = work_list.get("env", {})
|
|
220
|
+
for key, value in environments.items():
|
|
221
|
+
log.success(f"配置环境变量 {key}: {value}")
|
|
222
|
+
os.environ[key] = value
|
|
223
|
+
# 打印任务清单
|
|
224
|
+
log.debug(f"任务列表:{work_list}")
|
|
225
|
+
# 创建任务调度器
|
|
226
|
+
ret = exec_works(work_list.get("subtasks", []), config, os.cpu_count())
|
|
227
|
+
ret = ret and exec_works(work_list.get("seqtasks", []), config, 1)
|
|
228
|
+
return ret
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
class Executor(object):
|
|
232
|
+
def __init__(self, env: EnvDetector):
|
|
233
|
+
if not env.manifest:
|
|
234
|
+
raise LiteBmcException("未找到manifest.yml配置文件,当前目录不是一个合法的产品配置仓")
|
|
235
|
+
os.chdir(env.manifest.folder)
|
|
236
|
+
|
|
237
|
+
def run(self):
|
|
238
|
+
target = ""
|
|
239
|
+
succ = False
|
|
240
|
+
try:
|
|
241
|
+
config = Config(sys.argv[2:])
|
|
242
|
+
target = config.target
|
|
243
|
+
succ = target_executor(config)
|
|
244
|
+
except Exception as e:
|
|
245
|
+
log.error(str(e))
|
|
246
|
+
raise Exception(f"任务 {target} 执行失败")
|
|
247
|
+
if succ:
|
|
248
|
+
log.success(f"任务 {target} 执行成功")
|
|
249
|
+
return 0
|
|
250
|
+
else:
|
|
251
|
+
raise Exception(f"任务 {target} 执行失败")
|
|
252
|
+
|
|
253
|
+
if __name__ == "__main__":
|
|
254
|
+
env = EnvDetector()
|
|
255
|
+
exec = Executor(env)
|
|
256
|
+
exec.run()
|
|
@@ -1,16 +1,12 @@
|
|
|
1
1
|
"""任务基础类"""
|
|
2
|
-
import shutil
|
|
3
2
|
import importlib
|
|
4
|
-
import
|
|
5
|
-
import hashlib
|
|
6
|
-
import requests
|
|
7
|
-
import yaml
|
|
3
|
+
import traceback
|
|
8
4
|
from multiprocessing import Process
|
|
9
5
|
from lbkit.log import Logger
|
|
10
6
|
from lbkit.tools import Tools
|
|
11
7
|
from lbkit.misc import load_yml_with_json_schema_validate
|
|
12
8
|
|
|
13
|
-
from lbkit.
|
|
9
|
+
from lbkit.tasks.config import Config
|
|
14
10
|
|
|
15
11
|
class ManifestValidateError(OSError):
|
|
16
12
|
"""Raised when validation manifest.yml failed."""
|
|
@@ -22,6 +18,7 @@ class Task(Process):
|
|
|
22
18
|
self.log: Logger = Logger("task")
|
|
23
19
|
self.tools: Tools = Tools(name)
|
|
24
20
|
self.config: Config = config
|
|
21
|
+
self.name = name
|
|
25
22
|
|
|
26
23
|
def install(self):
|
|
27
24
|
"""安装任务"""
|
|
@@ -56,3 +53,18 @@ class Task(Process):
|
|
|
56
53
|
def load_manifest(self):
|
|
57
54
|
"""加载manifest.yml并验证schema文件"""
|
|
58
55
|
return self.config.load_manifest()
|
|
56
|
+
|
|
57
|
+
def deal_conf(self, config_dict):
|
|
58
|
+
"""
|
|
59
|
+
处理每个Task的私有配置"work_config"
|
|
60
|
+
当work类有set_xxx类方法时,则可以在target文件中配置xxx
|
|
61
|
+
"""
|
|
62
|
+
if not config_dict:
|
|
63
|
+
return
|
|
64
|
+
for conf in config_dict:
|
|
65
|
+
try:
|
|
66
|
+
exist = getattr(self, f"set_{conf}")
|
|
67
|
+
val = config_dict.get(conf)
|
|
68
|
+
exist(val)
|
|
69
|
+
except Exception as e:
|
|
70
|
+
raise Exception(f"无效配置: {conf}, {e}") from e
|
|
@@ -1,17 +1,15 @@
|
|
|
1
1
|
"""环境准备"""
|
|
2
2
|
import os
|
|
3
|
-
import
|
|
4
|
-
from lbkit.
|
|
5
|
-
from lbkit.integration.task import Task
|
|
3
|
+
from lbkit.tasks.config import Config
|
|
4
|
+
from lbkit.tasks.task import Task
|
|
6
5
|
from lbkit.log import Logger
|
|
7
|
-
from lbkit import errors
|
|
8
6
|
from lbkit.utils.images.emmc import MakeImage as MekeEmmcImage
|
|
9
7
|
|
|
10
8
|
log = Logger("build_image")
|
|
11
9
|
|
|
12
10
|
src_cwd = os.path.split(os.path.realpath(__file__))[0]
|
|
13
11
|
|
|
14
|
-
class
|
|
12
|
+
class TaskClass(Task):
|
|
15
13
|
@staticmethod
|
|
16
14
|
def _trans_to_blk_1m(cfg: str):
|
|
17
15
|
if cfg.endswith("K"):
|
|
@@ -43,5 +41,5 @@ class BuildImage(Task):
|
|
|
43
41
|
|
|
44
42
|
if __name__ == "__main__":
|
|
45
43
|
config = Config()
|
|
46
|
-
build =
|
|
44
|
+
build = TaskClass(config)
|
|
47
45
|
build.run()
|
lbkit-0.8.4/lbkit/integration/build_manifest.py → lbkit-0.8.5/lbkit/tasks/task_build_manifest.py
RENAMED
|
@@ -3,8 +3,8 @@ import os
|
|
|
3
3
|
import shutil
|
|
4
4
|
import json
|
|
5
5
|
from mako.lookup import TemplateLookup
|
|
6
|
-
from lbkit.
|
|
7
|
-
from lbkit.
|
|
6
|
+
from lbkit.tasks.config import Config
|
|
7
|
+
from lbkit.tasks.task import Task
|
|
8
8
|
from lbkit.log import Logger
|
|
9
9
|
from lbkit.build_conan_parallel import BuildConanParallel
|
|
10
10
|
from concurrent.futures import ThreadPoolExecutor
|
|
@@ -18,7 +18,7 @@ class ManifestValidateError(OSError):
|
|
|
18
18
|
|
|
19
19
|
src_cwd = os.path.split(os.path.realpath(__file__))[0]
|
|
20
20
|
|
|
21
|
-
class
|
|
21
|
+
class TaskClass(Task):
|
|
22
22
|
"""根据产品配置构建所有app,记录待安装应用路径到self.config.conan_install路径"""
|
|
23
23
|
def __init__(self, cfg: Config, name: str):
|
|
24
24
|
super().__init__(cfg, name)
|
|
@@ -159,5 +159,5 @@ class BuildManifest(Task):
|
|
|
159
159
|
|
|
160
160
|
if __name__ == "__main__":
|
|
161
161
|
config = Config()
|
|
162
|
-
build =
|
|
162
|
+
build = TaskClass(config)
|
|
163
163
|
build.run()
|
lbkit-0.8.4/lbkit/integration/build_prepare.py → lbkit-0.8.5/lbkit/tasks/task_build_prepare.py
RENAMED
|
@@ -3,8 +3,8 @@ import os
|
|
|
3
3
|
import shutil
|
|
4
4
|
import jinja2
|
|
5
5
|
import configparser
|
|
6
|
-
from lbkit.
|
|
7
|
-
from lbkit.
|
|
6
|
+
from lbkit.tasks.config import Config
|
|
7
|
+
from lbkit.tasks.task import Task
|
|
8
8
|
from lbkit.log import Logger
|
|
9
9
|
|
|
10
10
|
log = Logger("product_prepare")
|
|
@@ -15,7 +15,7 @@ class ManifestValidateError(OSError):
|
|
|
15
15
|
|
|
16
16
|
src_cwd = os.path.split(os.path.realpath(__file__))[0]
|
|
17
17
|
|
|
18
|
-
class
|
|
18
|
+
class TaskClass(Task):
|
|
19
19
|
def load_conan_profile(self):
|
|
20
20
|
profile = self.get_manifest_config("metadata/profile")
|
|
21
21
|
if not os.path.isfile(profile):
|
|
@@ -55,5 +55,5 @@ class BuildPrepare(Task):
|
|
|
55
55
|
|
|
56
56
|
if __name__ == "__main__":
|
|
57
57
|
config = Config()
|
|
58
|
-
build =
|
|
58
|
+
build = TaskClass(config)
|
|
59
59
|
build.run()
|
lbkit-0.8.4/lbkit/integration/build_rootfs.py → lbkit-0.8.5/lbkit/tasks/task_build_rootfs.py
RENAMED
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
"""完成rootfs镜像打包."""
|
|
2
2
|
import os
|
|
3
3
|
import shutil
|
|
4
|
-
import
|
|
5
|
-
from lbkit.
|
|
6
|
-
from lbkit.integration.task import Task
|
|
4
|
+
from lbkit.tasks.config import Config
|
|
5
|
+
from lbkit.tasks.task import Task
|
|
7
6
|
from lbkit.log import Logger
|
|
8
7
|
from lbkit import errors
|
|
9
8
|
|
|
@@ -12,7 +11,7 @@ log = Logger("build_rootfs")
|
|
|
12
11
|
src_cwd = os.path.split(os.path.realpath(__file__))[0]
|
|
13
12
|
IMG_FILE = "rootfs.img"
|
|
14
13
|
|
|
15
|
-
class
|
|
14
|
+
class TaskClass(Task):
|
|
16
15
|
"""构建rootfs镜像"""
|
|
17
16
|
def do_permission(self, per_file: str):
|
|
18
17
|
"""完成组件制品赋权"""
|
|
@@ -158,5 +157,5 @@ class BuildRootfs(Task):
|
|
|
158
157
|
|
|
159
158
|
if __name__ == "__main__":
|
|
160
159
|
config = Config()
|
|
161
|
-
build =
|
|
160
|
+
build = TaskClass(config)
|
|
162
161
|
build.run()
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""测试任务"""
|
|
2
|
+
import time
|
|
3
|
+
|
|
4
|
+
from lbkit.tasks.config import Config
|
|
5
|
+
from lbkit.tasks.task import Task
|
|
6
|
+
|
|
7
|
+
class TaskClass(Task):
|
|
8
|
+
def run(self):
|
|
9
|
+
print(f"task {self.name} start....")
|
|
10
|
+
time.sleep(2)
|
|
11
|
+
print(f"task {self.name} stop")
|
|
12
|
+
|
|
13
|
+
# def set_key1(self, val):
|
|
14
|
+
# print(f"Object {self.name} set key1: {val}")
|
|
15
|
+
# def set_key2(self, val):
|
|
16
|
+
# print(f"Object {self.name} set key2: {val}")
|
|
17
|
+
# def set_key3(self, val):
|
|
18
|
+
# print(f"Object {self.name} set key3: {val}")
|
|
19
|
+
# def set_key4(self, val):
|
|
20
|
+
# print(f"Object {self.name} set key4: {val}")
|
|
21
|
+
# def set_key5(self, val):
|
|
22
|
+
# print(f"Object {self.name} set key5: {val}")
|
|
23
|
+
# def set_key6(self, val):
|
|
24
|
+
# print(f"Object {self.name} set key6: {val}")
|
|
25
|
+
|
|
26
|
+
if __name__ == "__main__":
|
|
27
|
+
config = Config()
|
|
28
|
+
build = TaskClass(config)
|
|
29
|
+
build.run()
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
from multiprocessing import Pool
|
|
2
|
+
from multiprocessing.pool import ApplyResult
|
|
3
|
+
import time
|
|
4
|
+
from functools import partial
|
|
5
|
+
|
|
6
|
+
def square(number):
|
|
7
|
+
"""计算一个数的平方"""
|
|
8
|
+
print(f"计算 {number} 的平方")
|
|
9
|
+
time.sleep(1) # 模拟耗时操作
|
|
10
|
+
if number % 10 == 9:
|
|
11
|
+
raise Exception("sssssssssss")
|
|
12
|
+
# return number * number
|
|
13
|
+
|
|
14
|
+
def task_error(pool, number, error):
|
|
15
|
+
print(f"Work {number} exist with error: {error}")
|
|
16
|
+
pool.terminate()
|
|
17
|
+
|
|
18
|
+
if __name__ == "__main__":
|
|
19
|
+
# 创建任务并等待完成
|
|
20
|
+
results: dict[str, ApplyResult] = {}
|
|
21
|
+
pool = Pool(3)
|
|
22
|
+
for work in range(100):
|
|
23
|
+
error_cb = partial(task_error, pool, work)
|
|
24
|
+
result = pool.apply_async(square, args=(work, ), error_callback=error_cb)
|
|
25
|
+
results[work] = result
|
|
26
|
+
# 检查任务结果
|
|
27
|
+
pool.close()
|
|
28
|
+
pool.join()
|
|
29
|
+
# print("xxxxxxxxxxxxxxxxxx")
|
|
30
|
+
# time.sleep(100)
|
|
31
|
+
for number, result in results.items():
|
|
32
|
+
try:
|
|
33
|
+
if not result.ready():
|
|
34
|
+
succ = False
|
|
35
|
+
else:
|
|
36
|
+
result.get()
|
|
37
|
+
except Exception as e:
|
|
38
|
+
print(f"任务{number}执行异常: {str(e)}")
|
|
39
|
+
succ = False
|
|
40
|
+
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import configparser
|
|
5
|
+
from lbkit.tools import Tools
|
|
6
|
+
|
|
7
|
+
tools = Tools("EnvDetector")
|
|
8
|
+
log = tools.log
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class LbkitComponent(object):
|
|
12
|
+
def __init__(self, folder, config: configparser.ConfigParser):
|
|
13
|
+
self.folder = os.path.realpath(folder)
|
|
14
|
+
self.config = config
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class LbkitManifest(object):
|
|
18
|
+
def __init__(self, folder, config: configparser.ConfigParser):
|
|
19
|
+
self.folder = os.path.realpath(folder)
|
|
20
|
+
self.config = config
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class EnvDetector(object):
|
|
24
|
+
def __init__(self):
|
|
25
|
+
"""初始化"""
|
|
26
|
+
self.component: LbkitComponent = None
|
|
27
|
+
self.manifest: LbkitManifest = None
|
|
28
|
+
self.cwd = os.getcwd()
|
|
29
|
+
"""探测环境"""
|
|
30
|
+
cwd = self.cwd
|
|
31
|
+
while cwd != "/":
|
|
32
|
+
if os.path.isfile(os.path.join(cwd, "manifest.yml")):
|
|
33
|
+
self.manifest = LbkitManifest(cwd, None)
|
|
34
|
+
return
|
|
35
|
+
if os.path.isfile(os.path.join(cwd, "mds/package.yml")):
|
|
36
|
+
self.component = LbkitComponent(cwd, None)
|
|
37
|
+
return
|
|
38
|
+
cwd = os.path.dirname(cwd)
|
|
@@ -39,16 +39,20 @@ lbkit/component/build.py
|
|
|
39
39
|
lbkit/component/test.py
|
|
40
40
|
lbkit/component/template/conanbase.mako
|
|
41
41
|
lbkit/component/template/deploy.mako
|
|
42
|
-
lbkit/
|
|
43
|
-
lbkit/
|
|
44
|
-
lbkit/
|
|
45
|
-
lbkit/
|
|
46
|
-
lbkit/
|
|
47
|
-
lbkit/
|
|
48
|
-
lbkit/
|
|
49
|
-
lbkit/
|
|
50
|
-
lbkit/
|
|
42
|
+
lbkit/tasks/__init__.py
|
|
43
|
+
lbkit/tasks/config.py
|
|
44
|
+
lbkit/tasks/executor.py
|
|
45
|
+
lbkit/tasks/task.py
|
|
46
|
+
lbkit/tasks/task_build_image.py
|
|
47
|
+
lbkit/tasks/task_build_manifest.py
|
|
48
|
+
lbkit/tasks/task_build_prepare.py
|
|
49
|
+
lbkit/tasks/task_build_rootfs.py
|
|
50
|
+
lbkit/tasks/task_test.py
|
|
51
|
+
lbkit/tasks/test.py
|
|
52
|
+
lbkit/tasks/template/conanfile.py.mako
|
|
53
|
+
lbkit/tasks/template/rootfs.py.mako
|
|
51
54
|
lbkit/utils/__init__.py
|
|
55
|
+
lbkit/utils/env_detector.py
|
|
52
56
|
lbkit/utils/images/__init__.py
|
|
53
57
|
lbkit/utils/images/emmc.py
|
|
54
58
|
test/__init__.py
|
lbkit-0.8.4/MANIFEST.in
DELETED
lbkit-0.8.4/lbkit/__init__.py
DELETED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|