jolt 0.9.123__py3-none-any.whl → 0.9.435__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (196) hide show
  1. jolt/__init__.py +80 -7
  2. jolt/__main__.py +9 -1
  3. jolt/bin/fstree-darwin-x86_64 +0 -0
  4. jolt/bin/fstree-linux-x86_64 +0 -0
  5. jolt/cache.py +832 -362
  6. jolt/chroot.py +156 -0
  7. jolt/cli.py +281 -162
  8. jolt/common_pb2.py +63 -0
  9. jolt/common_pb2_grpc.py +4 -0
  10. jolt/config.py +98 -41
  11. jolt/error.py +19 -4
  12. jolt/filesystem.py +2 -6
  13. jolt/graph.py +705 -117
  14. jolt/hooks.py +43 -0
  15. jolt/influence.py +122 -3
  16. jolt/loader.py +369 -121
  17. jolt/log.py +225 -63
  18. jolt/manifest.py +28 -38
  19. jolt/options.py +35 -10
  20. jolt/pkgs/abseil.py +42 -0
  21. jolt/pkgs/asio.py +25 -0
  22. jolt/pkgs/autoconf.py +41 -0
  23. jolt/pkgs/automake.py +41 -0
  24. jolt/pkgs/b2.py +31 -0
  25. jolt/pkgs/boost.py +111 -0
  26. jolt/pkgs/boringssl.py +32 -0
  27. jolt/pkgs/busybox.py +39 -0
  28. jolt/pkgs/bzip2.py +43 -0
  29. jolt/pkgs/cares.py +29 -0
  30. jolt/pkgs/catch2.py +36 -0
  31. jolt/pkgs/cbindgen.py +17 -0
  32. jolt/pkgs/cista.py +19 -0
  33. jolt/pkgs/clang.py +44 -0
  34. jolt/pkgs/cli11.py +24 -0
  35. jolt/pkgs/cmake.py +48 -0
  36. jolt/pkgs/cpython.py +196 -0
  37. jolt/pkgs/crun.py +29 -0
  38. jolt/pkgs/curl.py +38 -0
  39. jolt/pkgs/dbus.py +18 -0
  40. jolt/pkgs/double_conversion.py +24 -0
  41. jolt/pkgs/fastfloat.py +21 -0
  42. jolt/pkgs/ffmpeg.py +28 -0
  43. jolt/pkgs/flatbuffers.py +29 -0
  44. jolt/pkgs/fmt.py +27 -0
  45. jolt/pkgs/fstree.py +20 -0
  46. jolt/pkgs/gflags.py +18 -0
  47. jolt/pkgs/glib.py +18 -0
  48. jolt/pkgs/glog.py +25 -0
  49. jolt/pkgs/glslang.py +21 -0
  50. jolt/pkgs/golang.py +16 -11
  51. jolt/pkgs/googlebenchmark.py +18 -0
  52. jolt/pkgs/googletest.py +46 -0
  53. jolt/pkgs/gperf.py +15 -0
  54. jolt/pkgs/grpc.py +73 -0
  55. jolt/pkgs/hdf5.py +19 -0
  56. jolt/pkgs/help2man.py +14 -0
  57. jolt/pkgs/inja.py +28 -0
  58. jolt/pkgs/jsoncpp.py +31 -0
  59. jolt/pkgs/libarchive.py +43 -0
  60. jolt/pkgs/libcap.py +44 -0
  61. jolt/pkgs/libdrm.py +44 -0
  62. jolt/pkgs/libedit.py +42 -0
  63. jolt/pkgs/libevent.py +31 -0
  64. jolt/pkgs/libexpat.py +27 -0
  65. jolt/pkgs/libfastjson.py +21 -0
  66. jolt/pkgs/libffi.py +16 -0
  67. jolt/pkgs/libglvnd.py +30 -0
  68. jolt/pkgs/libogg.py +28 -0
  69. jolt/pkgs/libpciaccess.py +18 -0
  70. jolt/pkgs/libseccomp.py +21 -0
  71. jolt/pkgs/libtirpc.py +24 -0
  72. jolt/pkgs/libtool.py +42 -0
  73. jolt/pkgs/libunwind.py +35 -0
  74. jolt/pkgs/libva.py +18 -0
  75. jolt/pkgs/libvorbis.py +33 -0
  76. jolt/pkgs/libxml2.py +35 -0
  77. jolt/pkgs/libxslt.py +17 -0
  78. jolt/pkgs/libyajl.py +16 -0
  79. jolt/pkgs/llvm.py +81 -0
  80. jolt/pkgs/lua.py +54 -0
  81. jolt/pkgs/lz4.py +26 -0
  82. jolt/pkgs/m4.py +14 -0
  83. jolt/pkgs/make.py +17 -0
  84. jolt/pkgs/mesa.py +81 -0
  85. jolt/pkgs/meson.py +17 -0
  86. jolt/pkgs/mstch.py +28 -0
  87. jolt/pkgs/mysql.py +60 -0
  88. jolt/pkgs/nasm.py +49 -0
  89. jolt/pkgs/ncurses.py +30 -0
  90. jolt/pkgs/ng_log.py +25 -0
  91. jolt/pkgs/ninja.py +45 -0
  92. jolt/pkgs/nlohmann_json.py +25 -0
  93. jolt/pkgs/nodejs.py +19 -11
  94. jolt/pkgs/opencv.py +24 -0
  95. jolt/pkgs/openjdk.py +26 -0
  96. jolt/pkgs/openssl.py +103 -0
  97. jolt/pkgs/paho.py +76 -0
  98. jolt/pkgs/patchelf.py +16 -0
  99. jolt/pkgs/perl.py +42 -0
  100. jolt/pkgs/pkgconfig.py +64 -0
  101. jolt/pkgs/poco.py +39 -0
  102. jolt/pkgs/protobuf.py +77 -0
  103. jolt/pkgs/pugixml.py +27 -0
  104. jolt/pkgs/python.py +19 -0
  105. jolt/pkgs/qt.py +35 -0
  106. jolt/pkgs/rapidjson.py +26 -0
  107. jolt/pkgs/rapidyaml.py +28 -0
  108. jolt/pkgs/re2.py +30 -0
  109. jolt/pkgs/re2c.py +17 -0
  110. jolt/pkgs/readline.py +15 -0
  111. jolt/pkgs/rust.py +41 -0
  112. jolt/pkgs/sdl.py +28 -0
  113. jolt/pkgs/simdjson.py +27 -0
  114. jolt/pkgs/soci.py +46 -0
  115. jolt/pkgs/spdlog.py +29 -0
  116. jolt/pkgs/spirv_llvm.py +21 -0
  117. jolt/pkgs/spirv_tools.py +24 -0
  118. jolt/pkgs/sqlite.py +83 -0
  119. jolt/pkgs/ssl.py +12 -0
  120. jolt/pkgs/texinfo.py +15 -0
  121. jolt/pkgs/tomlplusplus.py +22 -0
  122. jolt/pkgs/wayland.py +26 -0
  123. jolt/pkgs/x11.py +58 -0
  124. jolt/pkgs/xerces_c.py +20 -0
  125. jolt/pkgs/xorg.py +360 -0
  126. jolt/pkgs/xz.py +29 -0
  127. jolt/pkgs/yamlcpp.py +30 -0
  128. jolt/pkgs/zeromq.py +47 -0
  129. jolt/pkgs/zlib.py +87 -0
  130. jolt/pkgs/zstd.py +33 -0
  131. jolt/plugins/alias.py +3 -0
  132. jolt/plugins/allure.py +5 -2
  133. jolt/plugins/autotools.py +66 -0
  134. jolt/plugins/cache.py +133 -0
  135. jolt/plugins/cmake.py +74 -6
  136. jolt/plugins/conan.py +238 -0
  137. jolt/plugins/cxx.py +698 -0
  138. jolt/plugins/cxxinfo.py +7 -0
  139. jolt/plugins/dashboard.py +1 -1
  140. jolt/plugins/docker.py +80 -23
  141. jolt/plugins/email.py +2 -2
  142. jolt/plugins/email.xslt +144 -101
  143. jolt/plugins/environ.py +11 -0
  144. jolt/plugins/fetch.py +141 -0
  145. jolt/plugins/gdb.py +39 -19
  146. jolt/plugins/gerrit.py +1 -14
  147. jolt/plugins/git.py +283 -85
  148. jolt/plugins/googletest.py +2 -1
  149. jolt/plugins/http.py +36 -38
  150. jolt/plugins/libtool.py +63 -0
  151. jolt/plugins/linux.py +990 -0
  152. jolt/plugins/logstash.py +4 -4
  153. jolt/plugins/meson.py +61 -0
  154. jolt/plugins/ninja-compdb.py +99 -30
  155. jolt/plugins/ninja.py +468 -166
  156. jolt/plugins/paths.py +11 -1
  157. jolt/plugins/pkgconfig.py +219 -0
  158. jolt/plugins/podman.py +136 -92
  159. jolt/plugins/python.py +137 -0
  160. jolt/plugins/remote_execution/__init__.py +0 -0
  161. jolt/plugins/remote_execution/administration_pb2.py +46 -0
  162. jolt/plugins/remote_execution/administration_pb2_grpc.py +170 -0
  163. jolt/plugins/remote_execution/log_pb2.py +32 -0
  164. jolt/plugins/remote_execution/log_pb2_grpc.py +68 -0
  165. jolt/plugins/remote_execution/scheduler_pb2.py +41 -0
  166. jolt/plugins/remote_execution/scheduler_pb2_grpc.py +141 -0
  167. jolt/plugins/remote_execution/worker_pb2.py +38 -0
  168. jolt/plugins/remote_execution/worker_pb2_grpc.py +112 -0
  169. jolt/plugins/report.py +12 -2
  170. jolt/plugins/rust.py +25 -0
  171. jolt/plugins/scheduler.py +710 -0
  172. jolt/plugins/selfdeploy/setup.py +8 -4
  173. jolt/plugins/selfdeploy.py +138 -88
  174. jolt/plugins/strings.py +35 -22
  175. jolt/plugins/symlinks.py +26 -11
  176. jolt/plugins/telemetry.py +5 -2
  177. jolt/plugins/timeline.py +13 -3
  178. jolt/plugins/volume.py +46 -48
  179. jolt/scheduler.py +589 -192
  180. jolt/tasks.py +625 -121
  181. jolt/templates/timeline.html.template +44 -47
  182. jolt/timer.py +22 -0
  183. jolt/tools.py +638 -282
  184. jolt/utils.py +211 -7
  185. jolt/version.py +1 -1
  186. jolt/xmldom.py +12 -2
  187. {jolt-0.9.123.dist-info → jolt-0.9.435.dist-info}/METADATA +97 -38
  188. jolt-0.9.435.dist-info/RECORD +207 -0
  189. {jolt-0.9.123.dist-info → jolt-0.9.435.dist-info}/WHEEL +1 -1
  190. jolt/plugins/amqp.py +0 -834
  191. jolt/plugins/debian.py +0 -338
  192. jolt/plugins/ftp.py +0 -181
  193. jolt/plugins/repo.py +0 -253
  194. jolt-0.9.123.dist-info/RECORD +0 -77
  195. {jolt-0.9.123.dist-info → jolt-0.9.435.dist-info}/entry_points.txt +0 -0
  196. {jolt-0.9.123.dist-info → jolt-0.9.435.dist-info}/top_level.txt +0 -0
jolt/plugins/linux.py ADDED
@@ -0,0 +1,990 @@
1
+
2
+ from jolt import Task, Parameter, ListParameter
3
+ from jolt import attributes
4
+ from jolt import log
5
+ from jolt import utils
6
+ from jolt.error import raise_task_error_if
7
+ from jolt.plugins import podman
8
+
9
+
10
+ import os
11
+ import shutil
12
+
13
+
14
+ def linux_arch_to_container_platform(arch):
15
+ """
16
+ Convert Linux architecture to Podman platform.
17
+
18
+ Args:
19
+ arch: Linux architecture.
20
+
21
+ Returns:
22
+ - linux/amd64
23
+ - linux/arm
24
+ - linux/arm/v5
25
+ - linux/arm64
26
+ - linux/mips
27
+ - linux/ppc64
28
+ - linux/riscv64
29
+ - linux/s390x
30
+ """
31
+ platforms = {
32
+ "arm": "linux/arm",
33
+ "arm64": "linux/arm64",
34
+ "armv5": "linux/arm/v5",
35
+ "mips": "linux/mips64le",
36
+ "powerpc": "linux/ppc64le",
37
+ "riscv": "linux/riscv64",
38
+ "s390": "linux/s390x",
39
+ "x86": "linux/amd64",
40
+ }
41
+ arch = str(arch)
42
+ try:
43
+ return platforms[arch]
44
+ except KeyError:
45
+ raise ValueError(f"Unsupported architecture {arch}")
46
+
47
+
48
+ def linux_arch_to_debian_arch(arch):
49
+ """
50
+ Convert Linux architecture to Debian architecture.
51
+
52
+ Args:
53
+ arch: Linux architecture.
54
+
55
+ Returns:
56
+ - amd64
57
+ - armhf
58
+ - arm64
59
+ - mips
60
+ - ppc64el
61
+ - riscv64
62
+ - s390x
63
+ """
64
+ debian_arch = {
65
+ "amd64": "amd64",
66
+ "arm": "armhf",
67
+ "arm64": "arm64",
68
+ "armv5": "armel",
69
+ "mips": "mips",
70
+ "powerpc": "ppc64el",
71
+ "riscv": "riscv64",
72
+ "s390": "s390x",
73
+ "x86": "i386",
74
+ }
75
+ return debian_arch[str(arch)]
76
+
77
+
78
+ class ArchParameter(Parameter):
79
+ """
80
+ Linux target architecture parameter.
81
+
82
+ Supported values:
83
+
84
+ - amd64
85
+ - arm
86
+ - arm64
87
+ - armv5
88
+ - mips
89
+ - powerpc
90
+ - riscv
91
+ - s390
92
+ - x86
93
+
94
+ """
95
+
96
+ def __init__(self, *args, **kwargs):
97
+ kwargs["values"] = [
98
+ "amd64",
99
+ "arm",
100
+ "arm64",
101
+ "armv5",
102
+ "mips",
103
+ "powerpc",
104
+ "riscv",
105
+ "s390",
106
+ "x86",
107
+ ]
108
+ kwargs["help"] = "Target architecture."
109
+ super().__init__(*args, **kwargs)
110
+
111
+
112
+ class _ContainerImageBase(podman.ContainerImage):
113
+ """ Builds a container image from external source tree """
114
+
115
+ abstract = True
116
+ """ Must be subclassed """
117
+
118
+ arch = ArchParameter()
119
+ """ Target architecture [amd64, arm, arm64, armv5, mips, powerpc, riscv, s390, x86] """
120
+
121
+ @property
122
+ def target(self):
123
+ return linux_arch_to_container_platform(self.arch)
124
+
125
+
126
+ class _DebianSdkImage(podman.ContainerImage):
127
+ """ Provides a Debian SDK for building Linux kernel and U-Boot """
128
+
129
+ abstract = True
130
+ """ Must be subclassed """
131
+
132
+ arch = ArchParameter()
133
+ """ Target architecture """
134
+
135
+ dockerfile = """
136
+ FROM debian:{version}-slim
137
+ ARG DEBIAN_FRONTEND=noninteractive
138
+ RUN apt-get update && apt-get install -y crossbuild-essential-{debian_arch}
139
+ """
140
+
141
+ version = "stable"
142
+ """ Debian codename/version """
143
+
144
+ @property
145
+ def debian_arch(self):
146
+ return {
147
+ "amd64": "amd64",
148
+ "arm": "armhf",
149
+ "arm64": "arm64",
150
+ "mips": "mips",
151
+ "powerpc": "ppc64el",
152
+ "riscv": "riscv64",
153
+ "s390": "s390x",
154
+ "x86": "i386",
155
+ }[str(self.arch)]
156
+
157
+
158
+ class DebianHostSdk(Task):
159
+ """
160
+ Helper task that exports a Debian host's cross-compiler toolchains to consumers.
161
+
162
+ The task verifies that the cross-compiler toolchains are installed on the host.
163
+ If not, the task will raise an error with instructions.
164
+
165
+ Exported environment variables:
166
+
167
+ - ``CC``: Cross-compiler
168
+ - ``CPP``: Cross-preprocessor
169
+ - ``CXX``: Cross-C++ compiler
170
+ - ``CROSS_COMPILE``: Cross-compile prefix
171
+
172
+ """
173
+ abstract = True
174
+ """ Must be subclassed """
175
+
176
+ arch = ArchParameter()
177
+ """ Target architecture [amd64, arm, arm64, armv5, mips, powerpc, riscv, s390, x86] """
178
+
179
+ def publish(self, artifact, tools):
180
+ arch_to_cross_compile = {
181
+ "amd64": "x86_64-linux-gnu-",
182
+ "arm": "arm-linux-gnueabihf-",
183
+ "arm64": "aarch64-linux-gnu-",
184
+ "armv5": "arm-linux-gnueabi-",
185
+ "mips": "mips-linux-gnu-",
186
+ "powerpc": "powerpc64-linux-gnu-",
187
+ "riscv": "riscv64-linux-gnu-",
188
+ "s390": "s390x-linux-gnu-",
189
+ "x86": "i686-linux-gnu-",
190
+ }
191
+ debian_arch = linux_arch_to_debian_arch(self.arch)
192
+ debian_cross = arch_to_cross_compile[str(self.arch)]
193
+ raise_task_error_if(
194
+ not tools.which(arch_to_cross_compile[str(self.arch)] + "gcc"),
195
+ self, f"Cross compiler not found. Please install crossbuild-essential-{debian_arch} package.")
196
+ artifact.environ.CC = f"{debian_cross}gcc"
197
+ artifact.environ.CPP = f"{debian_cross}cpp"
198
+ artifact.environ.CXX = f"{debian_cross}g++"
199
+ artifact.environ.CROSS_COMPILE = debian_cross
200
+
201
+
202
+ class Initramfs(_ContainerImageBase):
203
+ """
204
+ Builds an initramfs image using Podman.
205
+
206
+ The task builds the initramfs image using the given Dockerfile and
207
+ publishes the resulting cpio archive. The task requires the following
208
+ attributes to be set:
209
+
210
+ - arch: Target architecture
211
+ - dockerfile: Path to Dockerfile, or Dockerfile content.
212
+
213
+ When building images for an architecture other than the host, the
214
+ binfmt-support package must be installed and configured to support the
215
+ target architecture. The package is available in most Linux distributions.
216
+
217
+ The location of the resulting cpio archive is stored in the
218
+ 'artifact.paths.cpio' attribute.
219
+
220
+ """
221
+ abstract = True
222
+ """ Must be subclassed """
223
+
224
+ output = ["cpio"]
225
+
226
+ def publish(self, artifact, tools):
227
+ super().publish(artifact, tools)
228
+ artifact.strings.arch = str(self.arch)
229
+ artifact.paths.cpio = "cpio/{_imagefile}.cpio"
230
+
231
+
232
+ class Squashfs(_ContainerImageBase):
233
+ """
234
+ Builds a squashfs image using Podman.
235
+
236
+ The task builds a container image using the given Dockerfile and converts
237
+ the resulting container filesystem to a squashfs image which is published.
238
+
239
+ When building images for an architecture other than the host, the
240
+ binfmt-support package must be installed and configured to support running
241
+ applications for the target architecture. The package is available in most
242
+ Linux distributions.
243
+
244
+ The location of the resulting squashfs image is stored in the
245
+ ``artifact.paths.squashfs`` artifact attribute.
246
+ """
247
+ abstract = True
248
+ """ Must be subclassed """
249
+
250
+ output = ["squashfs"]
251
+
252
+ size = None
253
+ """
254
+ Size of the squashfs image.
255
+
256
+ Typically used to align the image size to a supported SD card size (power of two).
257
+
258
+ Supported units are 'K', 'M', 'G', 'T'.
259
+ """
260
+
261
+ def run(self, deps, tools):
262
+ super().run(deps, tools)
263
+ if self.size:
264
+ with tools.cwd(tools.builddir("squashfs")):
265
+ tools.run("fallocate -l {size} image.squashfs")
266
+
267
+ def publish(self, artifact, tools):
268
+ super().publish(artifact, tools)
269
+ artifact.strings.arch = str(self.arch)
270
+ artifact.paths.squashfs = "squashfs/{_imagefile}.squashfs"
271
+
272
+
273
+ class Ext4(_ContainerImageBase):
274
+ """
275
+ Builds an ext4 image using Podman.
276
+
277
+ The task builds a container image using the given Dockerfile and converts
278
+ the resulting container filesystem to an ext4 image which is published.
279
+
280
+ When building images for an architecture other than the host, the
281
+ binfmt-support package must be installed and configured to support running
282
+ applications for the target architecture. The package is available in most
283
+ Linux distributions.
284
+
285
+ The location of the resulting squashfs image is stored in the
286
+ ``artifact.paths.squashfs`` artifact attribute.
287
+ """
288
+ abstract = True
289
+ """ Must be subclassed """
290
+
291
+ output = ["ext4"]
292
+
293
+ size = None
294
+ """
295
+ Size of the ext4 image.
296
+
297
+ Typically used to align the image size to a supported SD card size (power of two).
298
+
299
+ Supported units are 'K', 'M', 'G', 'T'.
300
+ """
301
+
302
+ def run(self, deps, tools):
303
+ super().run(deps, tools)
304
+ if self.size:
305
+ with tools.cwd(tools.builddir("ext4")):
306
+ tools.run("fallocate -l {size} image.ext4")
307
+
308
+ def publish(self, artifact, tools):
309
+ super().publish(artifact, tools)
310
+ artifact.strings.arch = str(self.arch)
311
+ artifact.paths.ext4 = "ext4/{_imagefile}.ext4"
312
+
313
+
314
+ class _KernelBase(Task):
315
+ abstract = True
316
+ """ Must be subclassed """
317
+
318
+ arch = ArchParameter()
319
+ """ Linux target architecture [amd64, arm, arm64, mips, powerpc, riscv, s390, x86] """
320
+
321
+ defconfig = Parameter("allnoconfig", help="Name of kernel defconfig")
322
+ """ Default configuration """
323
+
324
+ features = ListParameter([], help="List of feature configs to apply")
325
+ """
326
+ List of feature configuration snippets to apply.
327
+
328
+ If used, the task will search for feature configuration snippets in the
329
+ configured paths and merge them into the selected defconfig.
330
+ """
331
+
332
+ configpaths = [
333
+ "{srcdir}/arch/{arch}/configs",
334
+ ]
335
+ """
336
+ Paths to configuration file snippets.
337
+
338
+ When building the kernel with features, the task will search for
339
+ feature configuration snippets in these directories. The snippets
340
+ are applied and merged with the selected defconfig.
341
+ """
342
+
343
+ defconfigpath = "arch/{arch}/configs/{defconfig}_defconfig"
344
+ """ Path to defconfig file. """
345
+
346
+ srcdir = None
347
+ """ Location of Linux kernel source tree """
348
+
349
+ targets = ListParameter(help="Targets to build and publish")
350
+ """ Override and set accepted values """
351
+
352
+ def clean(self, tools):
353
+ self.moddir = tools.builddir("modules")
354
+ self.objdir = tools.builddir("objects", incremental=True)
355
+ tools.rmtree(self.moddir)
356
+ tools.rmtree(self.objdir)
357
+
358
+ def run(self, deps, tools):
359
+ raise_task_error_if(not tools.which("bison"), self, "bison is required to build the kernel")
360
+ raise_task_error_if(not tools.which("flex"), self, "flex is required to build the kernel")
361
+ raise_task_error_if(not tools.which("make"), self, "make is required to build the kernel")
362
+
363
+ self.objdir = tools.builddir("objects", incremental=True)
364
+
365
+ config = tools.expand_path("{objdir}/.config")
366
+ tools.unlink(config, ignore_errors=True)
367
+
368
+ arch = f"ARCH={self.arch}" if self.arch else ""
369
+ verbose = "V=1" if log.is_verbose() else ""
370
+ defconf_conf = self.run_find_defconfig_configs(deps, tools)
371
+ feature_confs = self.run_find_feature_configs(deps, tools)
372
+ defconfigpath = tools.expand(self.defconfigpath)
373
+ if self.defconfig in ["defconfig"]:
374
+ defconfigpath = os.path.join(os.path.dirname(defconfigpath), "defconfig")
375
+
376
+ with tools.cwd(self.srcdir):
377
+ if self.defconfig in ["allnoconfig"]:
378
+ self.info("Generating allnoconfig configuration")
379
+ tools.run("{} make allnoconfig O={objdir}", arch)
380
+ tools.run("{} scripts/kconfig/merge_config.sh -O {objdir} {objdir}/.config {} {}", arch, defconf_conf, " ".join(feature_confs))
381
+ else:
382
+ self.info("Generating {defconfig} configuration")
383
+ tools.run("{} scripts/kconfig/merge_config.sh -O {objdir} {} {} {}", arch, defconfigpath, defconf_conf, " ".join(feature_confs))
384
+
385
+ # Run build for each requested target
386
+ for target in self.targets:
387
+ fn = getattr(self, f"run_{target}", None)
388
+ assert fn, f"Dont know how to build {target}"
389
+ with tools.runprefix("{} make {} O={objdir} -j{}", arch, verbose, tools.thread_count()):
390
+ fn(deps, tools)
391
+
392
+ def run_find_defconfig_configs(self, deps, tools):
393
+ """ Finds local overrides for the chosen defconfig """
394
+ configpaths = [tools.expand_path(tools.expand(path)) for path in getattr(self, "configpaths", [])]
395
+ return shutil.which(tools.expand("{defconfig}.config"), os.F_OK, os.pathsep.join(configpaths)) or ""
396
+
397
+ def run_find_feature_configs(self, deps, tools):
398
+ """ Finds local overrides for chosen features """
399
+ configpaths = [tools.expand_path(tools.expand(path)) for path in getattr(self, "configpaths", [])]
400
+ return [shutil.which(feature + ".config", os.F_OK, os.pathsep.join(configpaths)) or "" for feature in self.features]
401
+
402
+ def publish(self, artifact, tools):
403
+ artifact.strings.arch = str(self.arch)
404
+
405
+ for target in self.targets:
406
+ fn = getattr(self, f"publish_{target}", None)
407
+ assert fn, f"Dont know how to publish {target}"
408
+ fn(artifact, tools)
409
+
410
+
411
+ class UBoot(_KernelBase):
412
+ """
413
+ Builds u-boot makefile target(s) and publishes the result.
414
+
415
+ An implementor must subclass this task and set the following attributes:
416
+
417
+ - srcdir: Path to U-boot source tree
418
+
419
+ It is recommended to use a task dependency to setup the
420
+ required tools by adding them to ``PATH`` and/or setting the ``CROSS_COMPILE``
421
+ environment variable. When building on a Debian host, the
422
+ :class:`jolt.plugins.linux.DebianHostSdk` helper task can be used to export
423
+ the cross-compiler tools.
424
+
425
+ """
426
+ abstract = True
427
+
428
+ defconfig = Parameter("allnoconfig", help="Name of u-boot defconfig")
429
+ """ Default configuration """
430
+
431
+ targets = ListParameter(
432
+ ["uboot"],
433
+ values=["tools", "uboot"],
434
+ help="Targets to build and publish",
435
+ )
436
+ """ Build targets [tools, uboot] """
437
+
438
+ def run_tools(self, deps, tools):
439
+ self.info("Building tools ...")
440
+ tools.run("tools")
441
+
442
+ def run_uboot(self, deps, tools):
443
+ self.info("Building u-boot ...")
444
+ tools.run("u-boot.bin")
445
+
446
+ def publish_tools(self, artifact, tools):
447
+ self.info("Publishing tools ...")
448
+ with tools.cwd(self.objdir):
449
+ artifact.collect("tools/mkimage")
450
+ artifact.environ.PATH.append("tools")
451
+
452
+ def publish_uboot(self, artifact, tools):
453
+ self.info("Publishing u-boot ...")
454
+ with tools.cwd(self.objdir):
455
+ artifact.collect("*.bin")
456
+
457
+
458
+ @utils.concat_attributes("dtbs", "dtbs_{defconfig}")
459
+ class Kernel(_KernelBase):
460
+ """
461
+ Builds kernel makefile target(s) and publishes the result.
462
+
463
+ Targets are selected by assigning the ``targets`` parameter. The following
464
+ values are supported:
465
+
466
+ - ``binary``: Build kernel binary
467
+ - ``dtbs``: Build device trees
468
+ - ``dtc``: Build device tree compiler
469
+ - ``gzimage``: Build gzipped kernel image
470
+ - ``image``: Build kernel image
471
+ - ``modules``: Build kernel modules
472
+ - ``uimage``: Build uImage
473
+ - ``vmlinux``: Build vmlinux
474
+ - ``zimage``: Build zImage
475
+
476
+ Mutliple targets can be selected at once, such as ``vmlinux+dtbs+modules``.
477
+ The resulting artifacts are published into different directories
478
+ based on the target name.
479
+
480
+ It is recommended to use a task dependency to setup the
481
+ required tools by adding them to ``PATH`` and/or setting the ``CROSS_COMPILE``
482
+ environment variable. When building on a Debian host, the
483
+ :class:`jolt.plugins.linux.DebianHostSdk` helper task can be used to export
484
+ the cross-compiler tools.
485
+
486
+ The following tools are required to be available in PATH:
487
+
488
+ - bison
489
+ - flex
490
+ - make
491
+ - gcc
492
+
493
+ Artifact path attributes are created to point to the published files.
494
+
495
+ """
496
+
497
+ abstract = True
498
+ """ Must be subclassed """
499
+
500
+ dtbs = []
501
+ """
502
+ List of device trees to build when the 'dtbs' target is requested.
503
+
504
+ If empty, all device trees associated with the target architecture are built.
505
+ """
506
+
507
+ loadaddr = 0
508
+ """
509
+ Kernel load address.
510
+ """
511
+
512
+ loadaddr_fdt = 0
513
+ """
514
+ Device-tree load address.
515
+ """
516
+
517
+ targets = ListParameter(
518
+ ["vmlinux"],
519
+ values=["binary", "dtbs", "dtc", "gzimage", "image", "modules", "uimage", "vmlinux", "zimage"],
520
+ help="Targets to build and publish",
521
+ )
522
+ """ Build targets [binary, dtbs, dtc, gzimage, image, modules, uimage, vmlinux, zimage] """
523
+
524
+ def run_binary(self, deps, tools):
525
+ self.info("Building binary kernel ...")
526
+ tools.run("LOADADDR={loadaddr} vmlinux")
527
+ objcopy = tools.getenv("OBJCOPY", tools.getenv("CROSS_COMPILE", "") + "objcopy")
528
+ tools._run_prefix = [] # Hack to cancel previous prefix
529
+ tools.run("{} -O binary -R .note -R .comment -S {objdir}/vmlinux {objdir}/vmlinux.bin", objcopy)
530
+
531
+ def run_dtbs(self, deps, tools):
532
+ self.info("Building device trees ...")
533
+ if not self._dtbs():
534
+ tools.run("dtbs")
535
+ else:
536
+ tools.run(" ".join(self._dtbs()))
537
+
538
+ def run_dtc(self, deps, tools):
539
+ self.info("Building device tree compiler ...")
540
+ tools.run("CONFIG_DTC=y scripts")
541
+
542
+ def run_modules(self, deps, tools):
543
+ self.info("Building modules ...")
544
+ self.moddir = tools.builddir("modules")
545
+ tools.run("INSTALL_MOD_PATH={moddir} modules")
546
+ tools.run("INSTALL_MOD_PATH={moddir} modules_install")
547
+
548
+ def run_image(self, deps, tools):
549
+ self.info("Building Image ...")
550
+ tools.run("LOADADDR={loadaddr} Image")
551
+
552
+ def run_gzimage(self, deps, tools):
553
+ self.info("Building Image.gz ...")
554
+ tools.run("LOADADDR={loadaddr} Image.gz")
555
+
556
+ def run_uimage(self, deps, tools):
557
+ self.info("Building uImage ...")
558
+ tools.run("LOADADDR={loadaddr} uImage")
559
+
560
+ def run_vmlinux(self, deps, tools):
561
+ self.info("Building vmlinux ...")
562
+ tools.run("vmlinux")
563
+
564
+ def run_zimage(self, deps, tools):
565
+ self.info("Building zImage ...")
566
+ tools.run("zImage")
567
+
568
+ def publish(self, artifact, tools):
569
+ super().publish(artifact, tools)
570
+
571
+ if self.loadaddr != 0:
572
+ artifact.strings.loadaddr = str(self.loadaddr)
573
+ if self.loadaddr_fdt != 0:
574
+ artifact.strings.loadaddr_fdt = str(self.loadaddr_fdt)
575
+
576
+ def publish_binary(self, artifact, tools):
577
+ self.info("Publishing binary ...")
578
+ with tools.cwd(self.objdir):
579
+ artifact.collect("vmlinux.bin", "binary/")
580
+ artifact.paths.binary = "binary/vmlinux.bin"
581
+
582
+ def publish_dtbs(self, artifact, tools):
583
+ self.info("Publishing device trees ...")
584
+ with tools.cwd(self.objdir, "arch/{arch}/boot/dts"):
585
+ if not self._dtbs():
586
+ artifact.collect("**/*.dtb", "dtbs/")
587
+ else:
588
+ for dtb in self._dtbs():
589
+ artifact.collect(dtb, "dtbs/")
590
+ artifact.paths.dtbs = "dtbs"
591
+
592
+ def publish_dtc(self, artifact, tools):
593
+ self.info("Publishing device tree compiler ...")
594
+ with tools.cwd(self.objdir, "scripts/dtc"):
595
+ artifact.collect("dtc", "bin/")
596
+ artifact.environ.PATH.append("bin")
597
+
598
+ def publish_modules(self, artifact, tools):
599
+ self.info("Publishing modules ...")
600
+ with tools.cwd(self.moddir):
601
+ artifact.collect(".", "modules/", symlinks=True)
602
+ artifact.paths.modules = "modules"
603
+
604
+ with tools.cwd(self.objdir):
605
+ if os.path.exists(tools.expand_path("Module.symvers")):
606
+ artifact.collect("Module.symvers")
607
+ artifact.paths.symvers = "Module.symvers"
608
+
609
+ def publish_image(self, artifact, tools):
610
+ self.info("Publishing Image ...")
611
+ with tools.cwd(self.objdir, "arch/{arch}/boot"):
612
+ artifact.collect("Image", "image/")
613
+ artifact.paths.image = "image/Image"
614
+
615
+ def publish_gzimage(self, artifact, tools):
616
+ self.info("Publishing Image.gz ...")
617
+ with tools.cwd(self.objdir, "arch/{arch}/boot"):
618
+ artifact.collect("Image.gz", "gzimage/")
619
+ artifact.paths.gzimage = "gzimage/Image.gz"
620
+
621
+ def publish_uimage(self, artifact, tools):
622
+ self.info("Publishing uImage ...")
623
+ with tools.cwd(self.objdir, "arch/{arch}/boot"):
624
+ artifact.collect("uImage", "uimage/")
625
+ artifact.paths.uimage = "uimage/uImage"
626
+
627
+ def publish_vmlinux(self, artifact, tools):
628
+ self.info("Publishing vmlinux ...")
629
+ with tools.cwd(self.objdir):
630
+ artifact.collect("vmlinux", "vmlinux/")
631
+ artifact.paths.vmlinux = "vmlinux/vmlinux"
632
+
633
+ def publish_zimage(self, artifact, tools):
634
+ self.info("Publishing zImage ...")
635
+ with tools.cwd(self.objdir, "arch/{arch}/boot"):
636
+ artifact.collect("zImage", "zimage/")
637
+ artifact.paths.zimage = "zimage/zImage"
638
+
639
+
640
+ class Module(Kernel):
641
+ """
642
+ Builds a kernel module from external source tree.
643
+
644
+ The task is based on the :class:`Kernel` task and builds a kernel module
645
+ from the given source tree specified by the ``srcdir_module`` attribute.
646
+ The ``targets`` attribute is set to ``modules`` by default.
647
+
648
+ See :class:`Kernel` for additional information on building kernel targets.
649
+ In particular, the ``srcdir`` attribute must be set to the kernel source tree
650
+ to build the modules against.
651
+ """
652
+
653
+ abstract = True
654
+ """ Must be subclassed """
655
+
656
+ srcdir_module = None
657
+ """ Path to kernel module source tree """
658
+
659
+ targets = ["modules"]
660
+ """ Build targets [modules] """
661
+
662
+ def run(self, deps, tools):
663
+ self.targets = ["modules"]
664
+ super().run(deps, tools)
665
+
666
+ def run_modules(self, deps, tools):
667
+ with tools.cwd(self.objdir):
668
+ for _, artifact in deps.items():
669
+ if str(artifact.paths.symvers):
670
+ self.info("Copying Module.symvers")
671
+ tools.copy(str(artifact.paths.symvers), ".")
672
+
673
+ self.info("Building modules ...")
674
+ self.moddir = tools.builddir("modules")
675
+ tools.run("INSTALL_MOD_PATH={moddir} modules_prepare")
676
+ tools.run("INSTALL_MOD_PATH={moddir} M={joltdir}/{srcdir_module} modules")
677
+ tools.run("INSTALL_MOD_PATH={moddir} M={joltdir}/{srcdir_module} modules_install")
678
+
679
+
680
+ @utils.concat_attributes("configs", "configs")
681
+ class FIT(Task):
682
+ """
683
+ Builds and publishes a FIT Image.
684
+
685
+ The task builds a FIT image using the given kernel, device tree and ramdisk
686
+ tasks and publishes the resulting image. Multiple FIT configurations can be
687
+ created by defining the ``configs`` attribute. The image is optinally signed
688
+ using the provided key.
689
+
690
+ The task requires the ``mkimage`` tool to be available in PATH.
691
+ """
692
+
693
+ abstract = True
694
+ """ Must be subclassed """
695
+
696
+ configs = {}
697
+ """
698
+ FIT configurations to create.
699
+
700
+ Dictionary of 2 or 3 element tuples according to this format:
701
+
702
+ .. code-block:: python
703
+
704
+ configs = {
705
+ "<name of config>": ("<name of kernel task>", "<name of dtb>"),
706
+ "<name of config with ramdisk>": ("<name of kernel task>", "<name of dtb>", "<name of ramdisk task>"),
707
+ }
708
+
709
+ Example:
710
+
711
+ .. code-block:: python
712
+
713
+ requires = [
714
+ "kernel=linux/kernel:arch=arm,targets=dtbs+binary_gz,defconfig=bcm2835",
715
+ "ramdisk=busybox/irfs:arch=armv7"
716
+ ]
717
+
718
+ configs = {
719
+ "conf-rpi0w": ("kernel", "bcm2835-rpi-zero-w.dtb"),
720
+ "conf-rpi0w-irfs": ("kernel", "bcm2835-rpi-zero-w.dtb", "ramdisk"),
721
+ }
722
+
723
+ Loaded from defs/{defconfig}.py
724
+ """
725
+
726
+ config_default = None
727
+ """ Default FIT configuration """
728
+
729
+ _template = """/dts-v1/;
730
+
731
+ / {
732
+ description = "U-Boot fitImage";
733
+
734
+ images {
735
+ {% for i, kernel in enumerate(_kernels) %}
736
+ kernel@{{kernel}} {
737
+ description = "Linux kernel";
738
+ {% if str(deps[kernel].paths.zimage) %}
739
+ data = /incbin/("{{deps[kernel].paths.zimage}}");
740
+ compression = "none";
741
+ {% elif str(deps[kernel].paths.gzimage) %}
742
+ data = /incbin/("{{deps[kernel].paths.gzimage}}");
743
+ compression = "gzip";
744
+ {% elif str(deps[kernel].paths.image) %}
745
+ data = /incbin/("{{deps[kernel].paths.image}}");
746
+ compression = "none";
747
+ {% else %}
748
+ #error "Kernel type not supported."
749
+ {% endif %}
750
+ type = "kernel";
751
+ arch = "{{deps[kernel].strings.arch}}";
752
+ os = "linux";
753
+ {% if deps[kernel].strings.loadaddr %}
754
+ load = <{{deps[kernel].strings.loadaddr}}>;
755
+ entry = <{{deps[kernel].strings.loadaddr}}>;
756
+ {% endif %}
757
+ hash {
758
+ algo = "sha256";
759
+ };
760
+ };
761
+
762
+ {% endfor %}
763
+ {% for kernel, dtb in _dtbs %}
764
+ fdt@{{path.basename(dtb)}} {
765
+ description = "Flattened Device Tree blob";
766
+ data = /incbin/("{{deps[kernel].paths.dtbs}}/{{dtb}}");
767
+ type = "flat_dt";
768
+ arch = "{{deps[kernel].strings.arch}}";
769
+ compression = "none";
770
+ os = "linux";
771
+ {% if deps[kernel].strings.loadaddr_fdt %}
772
+ load = <{{deps[kernel].strings.loadaddr_fdt}}>;
773
+ {% endif %}
774
+ hash {
775
+ algo = "sha256";
776
+ };
777
+ };
778
+
779
+ {% endfor %}
780
+ {% for ramdisk in _ramdisks %}
781
+ ramdisk@{{ramdisk}} {
782
+ description = "Ramdisk";
783
+ data = /incbin/("{{deps[ramdisk].paths.cpio}}");
784
+ type = "ramdisk";
785
+ arch = "{{deps[ramdisk].strings.arch}}";
786
+ os = "linux";
787
+ {% if deps[ramdisk].strings.compression %}
788
+ compression = "{{deps[ramdisk].strings.compression}}";
789
+ {% else %}
790
+ compression = "none";
791
+ {% endif %}
792
+ hash {
793
+ algo = "sha256";
794
+ };
795
+ };
796
+
797
+ {% endfor %}
798
+ };
799
+
800
+ configurations {
801
+ {% if config_default %}
802
+ default = "conf@{{config_default}}";
803
+ {% endif %}
804
+
805
+ {% for name, config in _configs().items() %}
806
+ conf@{{name}} {
807
+ {% if len(config) > 2 %}
808
+ description = "Linux kernel, FDT blob, Ramdisk";
809
+ {% else %}
810
+ description = "Linux kernel, FDT blob";
811
+ {% endif %}
812
+ kernel = "kernel@{{config[0]}}";
813
+ os = "linux";
814
+ fdt = "fdt@{{path.basename(config[1])}}";
815
+ {% if len(config) > 2 %}
816
+ ramdisk = "ramdisk@{{config[2]}}";
817
+ {% endif %}
818
+
819
+ hash {
820
+ algo = "sha256";
821
+ };
822
+ signature {
823
+ algo = "sha256,rsa2048";
824
+ {% if signature_key_name %}
825
+ key-name-hint = "{{signature_key_name}}";
826
+ {% endif %}
827
+ {% if len(config) > 2 %}
828
+ sign-images = "kernel", "fdt", "ramdisk";
829
+ {% else %}
830
+ sign-images = "kernel", "fdt";
831
+ {% endif %}
832
+ };
833
+ };
834
+
835
+ {% endfor %}
836
+ };
837
+ };
838
+ """
839
+
840
+ signature_key_name = None
841
+ """ Name of key used to sign image. If None, image won't be signed. """
842
+
843
+ signature_key_path = None
844
+ """ Directory path to signature key """
845
+
846
+ def run(self, deps, tools):
847
+ assert tools.which("mkimage"), "mkimage is required to build the FIT image"
848
+
849
+ kernels = []
850
+ dtbs = []
851
+ ramdisks = []
852
+
853
+ # Add kernels and dtbs to individual lists
854
+ for name, config in self._configs().items():
855
+ kernels.append(config[0])
856
+ dtbs.append((config[0], config[1]))
857
+ try:
858
+ ramdisks.append(config[2])
859
+ except IndexError:
860
+ pass
861
+ dtbs = list(set(dtbs))
862
+ kernels = list(set(kernels))
863
+
864
+ # Render image device tree source template
865
+ self.info("Rendering FIT image source file")
866
+ its = tools.render(
867
+ self._template,
868
+ enumerate=enumerate,
869
+ len=len,
870
+ path=os.path,
871
+ print=print,
872
+ str=str,
873
+ deps=deps,
874
+ _dtbs=list(set(dtbs)),
875
+ _kernels=list(set(kernels)),
876
+ _ramdisks=list(set(ramdisks)))
877
+
878
+ with tools.cwd(tools.builddir()):
879
+ tools.write_file("fitImage.its", its, expand=False)
880
+ print(its)
881
+
882
+ self.info("Building FIT image")
883
+ tools.run(["mkimage", "-V"], shell=False)
884
+ tools.run(["mkimage", "-D", "-I dts -O dtb -p 2000", "-f", "fitImage.its", "fitImage"], shell=False)
885
+
886
+ # Optionally sign the image
887
+ if self.signature_key_name:
888
+ self.info("Signing FIT image")
889
+ signature_key_path = tools.expand_path(self.signature_key_path)
890
+ tools.copy("fitImage", "fitImage.unsigned")
891
+ tools.run(["mkimage", "-D", "-I dts -O dtb -p 2000", "-F", "-k", signature_key_path, "-r", "fitImage"], shell=False)
892
+
893
+ def publish(self, artifact, tools):
894
+ with tools.cwd(tools.builddir()):
895
+ artifact.collect("fitImage")
896
+ artifact.collect("fitImage.unsigned")
897
+ artifact.collect("fitImage.its")
898
+ artifact.paths.fitimage = "fitImage"
899
+
900
+
901
+ @attributes.attribute("binary", "binary_{arch}")
902
+ class Qemu(Task):
903
+ """
904
+ Runs Qemu with the given kernel, initramfs and rootfs.
905
+
906
+ The task automatically selects the correct Qemu binary for the architecture
907
+ and builds the command line arguments based on the provided kernel, initramfs
908
+ and rootfs tasks.
909
+
910
+ Additional arguments can be passed to QEMU by setting the ``arguments``
911
+ attribute.
912
+
913
+ The selected Qemu binary must be available in PATH.
914
+ """
915
+
916
+ abstract = True
917
+ """ Must be subclassed """
918
+
919
+ arch = ArchParameter()
920
+ """ Target architecture [amd64, arm, arm64, mips, powerpc, riscv, s390, x86] """
921
+
922
+ arguments = []
923
+ """ Additional arguments to pass to QEMU """
924
+
925
+ binary = None
926
+ """ Name of QEMU binary """
927
+
928
+ binary_arm = "qemu-system-arm"
929
+ binary_arm64 = "qemu-system-aarch64"
930
+ binary_mips = "qemu-system-mips"
931
+ binary_powerpc = "qemu-system-ppc"
932
+ binary_riscv = "qemu-system-riscv64"
933
+ binary_s390 = "qemu-system-s390x"
934
+ binary_x86 = "qemu-system-x86_64"
935
+
936
+ cacheable = False
937
+ """ Task is not cacheable """
938
+
939
+ dtb = None
940
+ """ Path to device tree in kernel artifact """
941
+
942
+ initrd = None
943
+ """ Name of initrd/ramfs task, if any """
944
+
945
+ kernel = None
946
+ """ Name of kernel task """
947
+
948
+ machine = None
949
+ """ Target machine """
950
+
951
+ memory = None
952
+ """ Memory size. Example: 512M """
953
+
954
+ rootfs = None
955
+ """ Name of rootfs task, if any """
956
+
957
+ def requires(self):
958
+ r = []
959
+ if self.kernel:
960
+ r.append("kernel=" + self.kernel)
961
+ if self.initrd:
962
+ r.append("initrd=" + self.initrd)
963
+ if self.rootfs:
964
+ r.append("rootfs=" + self.rootfs)
965
+ return r
966
+
967
+ def run(self, deps, tools):
968
+ assert tools.which(self.binary), "{binary} is required to run QEMU"
969
+ assert self.machine, "Machine type is required to run QEMU"
970
+
971
+ self.deps = deps
972
+
973
+ # Get kernel and initrd paths
974
+ arguments = self.expand(self.arguments)
975
+ dtb = ["-dtb", os.path.join(str(deps[self.kernel].paths.dtbs), self.dtb)] if self.dtb else []
976
+ initrd = ["-initrd", str(deps[self.initrd].paths.cpio)] if self.initrd else []
977
+ kernel = ["-kernel", str(deps[self.kernel].paths.zimage)] if self.kernel else []
978
+ memory = ["-m", self.memory] if self.memory else []
979
+
980
+ # Run QEMU
981
+ binary = tools.which(self.binary)
982
+
983
+ cmdline = [
984
+ binary,
985
+ "-M", self.machine,
986
+ ] + kernel + dtb + initrd + memory + arguments
987
+
988
+ self.info("Running QEMU with command: {}", cmdline)
989
+
990
+ os.execv(binary, cmdline)