cibuildmp 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cibuildmp/__init__.py +19 -0
- cibuildmp/__main__.py +4 -0
- cibuildmp/build.py +300 -0
- cibuildmp/cli.py +367 -0
- cibuildmp/options.py +267 -0
- cibuildmp/resources/natmod.toml +167 -0
- cibuildmp/resources.py +22 -0
- cibuildmp/sources.py +275 -0
- cibuildmp/targets.py +259 -0
- cibuildmp/toolchains.py +302 -0
- cibuildmp-0.3.0.dist-info/METADATA +528 -0
- cibuildmp-0.3.0.dist-info/RECORD +16 -0
- cibuildmp-0.3.0.dist-info/WHEEL +5 -0
- cibuildmp-0.3.0.dist-info/entry_points.txt +2 -0
- cibuildmp-0.3.0.dist-info/licenses/LICENSE +21 -0
- cibuildmp-0.3.0.dist-info/top_level.txt +1 -0
cibuildmp/__init__.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""cibuildmp -- build MicroPython native C extensions on CI, and locally."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import sys
|
|
6
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
7
|
+
|
|
8
|
+
try:
|
|
9
|
+
__version__ = version("cibuildmp")
|
|
10
|
+
except PackageNotFoundError: # running from a source tree, not installed
|
|
11
|
+
__version__ = "0.0.0.dev0"
|
|
12
|
+
|
|
13
|
+
from .cli import main as _main
|
|
14
|
+
|
|
15
|
+
__all__ = ["__version__", "main"]
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def main() -> None:
|
|
19
|
+
sys.exit(_main())
|
cibuildmp/__main__.py
ADDED
cibuildmp/build.py
ADDED
|
@@ -0,0 +1,300 @@
|
|
|
1
|
+
"""Running the build itself: pre-build command, make, collect, verify,
|
|
2
|
+
package.
|
|
3
|
+
|
|
4
|
+
Fails fast, one target at a time -- matching cibuildwheel's own
|
|
5
|
+
build-in-container loop (platforms/linux.py), which lets a
|
|
6
|
+
`subprocess.CalledProcessError` from one identifier abort the whole
|
|
7
|
+
invocation rather than collecting per-target failures into a report.
|
|
8
|
+
`cli.build()` already handles that: this module raises, cli.main() catches
|
|
9
|
+
alongside SourceError/ToolchainError.
|
|
10
|
+
|
|
11
|
+
Also cibuildwheel-shaped: `collect_output()`/`verify_output()` mirror its
|
|
12
|
+
"exactly one artifact, or a named error" check (BuildProducedNoWheelError/
|
|
13
|
+
RepairStepProducedMultipleWheelsError), and the BuildResult accumulated per
|
|
14
|
+
target mirrors its BuildInfo summary line.
|
|
15
|
+
|
|
16
|
+
No separate `cibuildmp publish` step (see docs/BACKLOG.md D14): each
|
|
17
|
+
target's own directory under `output-dir` already holds everything mip
|
|
18
|
+
needs -- the `.mpy`, any `extra-files` companions, and a `package.json` --
|
|
19
|
+
the moment the build finishes. Assembling a ready-to-upload tree is as far
|
|
20
|
+
as this goes; creating a release or uploading it stays the caller's own CI
|
|
21
|
+
step, the same way cibuildwheel never runs `twine upload` itself.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import json
|
|
27
|
+
import subprocess
|
|
28
|
+
import sys
|
|
29
|
+
import time
|
|
30
|
+
from collections.abc import Sequence
|
|
31
|
+
from dataclasses import dataclass
|
|
32
|
+
from pathlib import Path
|
|
33
|
+
|
|
34
|
+
from .options import BuildOptions
|
|
35
|
+
from .targets import NATIVE_ARCH_CODE
|
|
36
|
+
from .toolchains import ResolvedToolchain
|
|
37
|
+
|
|
38
|
+
MPY_HEADER_MAGIC = ord("M")
|
|
39
|
+
MPY_ARCH_FLAGS_BIT = 0x40
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class BuildError(Exception):
|
|
43
|
+
pass
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass(frozen=True)
|
|
47
|
+
class BuildResult:
|
|
48
|
+
identifier: str
|
|
49
|
+
output: Path
|
|
50
|
+
duration: float
|
|
51
|
+
|
|
52
|
+
@property
|
|
53
|
+
def size(self) -> int:
|
|
54
|
+
return self.output.stat().st_size
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def make_command(
|
|
58
|
+
build_options: BuildOptions, mpy_dir: Path, module_root: Path
|
|
59
|
+
) -> list[str]:
|
|
60
|
+
"""The make invocation for one target.
|
|
61
|
+
|
|
62
|
+
PYTHON=<sys.executable> is what makes pyelftools/ar cibuildmp's own
|
|
63
|
+
dependencies rather than something installed at build time (D12):
|
|
64
|
+
py/dynruntime.mk assigns PYTHON with a plain `=`, never `override`, so
|
|
65
|
+
this wins over the `python3` it would otherwise default to -- the same
|
|
66
|
+
mechanism already used for CROSS= in toolchains.ResolvedToolchain.
|
|
67
|
+
"""
|
|
68
|
+
return [
|
|
69
|
+
"make",
|
|
70
|
+
"-C",
|
|
71
|
+
str(module_root),
|
|
72
|
+
f"ARCH={build_options.target.arch}",
|
|
73
|
+
f"MPY_DIR={mpy_dir}",
|
|
74
|
+
f"PYTHON={sys.executable}",
|
|
75
|
+
*build_options.extra_make_args,
|
|
76
|
+
build_options.make_target,
|
|
77
|
+
]
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def run_pre_build_command(module_root: Path, command: str, env: dict[str, str]) -> None:
|
|
81
|
+
if not command:
|
|
82
|
+
return
|
|
83
|
+
try:
|
|
84
|
+
subprocess.run(command, shell=True, cwd=module_root, env=env, check=True)
|
|
85
|
+
except subprocess.CalledProcessError as exc:
|
|
86
|
+
raise BuildError(f"pre-build-command failed: {command!r} ({exc})") from exc
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def run_make(
|
|
90
|
+
build_options: BuildOptions, mpy_dir: Path, module_root: Path, env: dict[str, str]
|
|
91
|
+
) -> None:
|
|
92
|
+
# No cwd= here: `-C module_root` in the command itself already makes
|
|
93
|
+
# make chdir there, and module_root can be relative (it usually is --
|
|
94
|
+
# options.package_dir is often "."), so also passing it as cwd would
|
|
95
|
+
# chdir twice and have make look for module_root nested inside itself.
|
|
96
|
+
command = make_command(build_options, mpy_dir, module_root)
|
|
97
|
+
try:
|
|
98
|
+
subprocess.run(command, env=env, check=True)
|
|
99
|
+
except subprocess.CalledProcessError as exc:
|
|
100
|
+
raise BuildError(
|
|
101
|
+
f"{build_options.identifier}: `{' '.join(command)}` failed with exit "
|
|
102
|
+
f"code {exc.returncode}"
|
|
103
|
+
) from exc
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def collect_output(build_options: BuildOptions, module_root: Path) -> Path:
|
|
107
|
+
"""Find the one .mpy the `dist` target produced.
|
|
108
|
+
|
|
109
|
+
Every natmod Makefile in the wild drops it under build/<arch>*/ -- the
|
|
110
|
+
same layout build-natmod's own artifact-upload step already assumes
|
|
111
|
+
(`path: natmod/build/${{ matrix.arch }}*/`), not something cibuildmp
|
|
112
|
+
invents here.
|
|
113
|
+
"""
|
|
114
|
+
arch = build_options.target.arch
|
|
115
|
+
candidates = sorted(module_root.glob(f"build/{arch}*/*.mpy"))
|
|
116
|
+
if not candidates:
|
|
117
|
+
raise BuildError(
|
|
118
|
+
f"{build_options.identifier}: `{build_options.make_target}` produced no "
|
|
119
|
+
f".mpy under {module_root}/build/{arch}*/"
|
|
120
|
+
)
|
|
121
|
+
if len(candidates) > 1:
|
|
122
|
+
names = ", ".join(p.name for p in candidates)
|
|
123
|
+
raise BuildError(
|
|
124
|
+
f"{build_options.identifier}: ambiguous output -- found "
|
|
125
|
+
f"{len(candidates)} .mpy files under {module_root}/build/{arch}*/: {names}"
|
|
126
|
+
)
|
|
127
|
+
return candidates[0]
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _read_varint(data: bytes, offset: int) -> tuple[int, int]:
|
|
131
|
+
"""MicroPython's own uint encoding: big-endian 7-bit groups, MSB=more.
|
|
132
|
+
|
|
133
|
+
Same format tools/mpy_ld.py's MPYOutput.write_uint() writes and
|
|
134
|
+
py/persistentcode.c's read_uint() reads. Returns (value, bytes consumed).
|
|
135
|
+
"""
|
|
136
|
+
value = 0
|
|
137
|
+
consumed = 0
|
|
138
|
+
while True:
|
|
139
|
+
if offset + consumed >= len(data):
|
|
140
|
+
raise BuildError("truncated .mpy header (arch-flags field cut off)")
|
|
141
|
+
byte = data[offset + consumed]
|
|
142
|
+
value = (value << 7) | (byte & 0x7F)
|
|
143
|
+
consumed += 1
|
|
144
|
+
if not byte & 0x80:
|
|
145
|
+
return value, consumed
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def read_mpy_header(mpy_path: Path) -> tuple[int, int, int]:
|
|
149
|
+
"""(MPY_VERSION, native-arch code, arch_flags) from a compiled .mpy.
|
|
150
|
+
|
|
151
|
+
Layout from tools/mpy_ld.py's build_mpy() / py/persistentcode.h: byte 0
|
|
152
|
+
is 'M', byte 1 is MPY_VERSION, byte 2 packs MPY_SUB_VERSION in bits 0-1,
|
|
153
|
+
the native-arch code in bits 2-6 (`MPY_FEATURE_DECODE_ARCH`, mask 0x2F
|
|
154
|
+
*after* the shift -- bit 6 is the arch-flags marker, not part of the
|
|
155
|
+
arch code, and must be excluded or a flagged file decodes as a bogus
|
|
156
|
+
arch), byte 3 is MP_SMALL_INT_BITS. A variable-length uint (arch_flags)
|
|
157
|
+
follows when that marker bit is set.
|
|
158
|
+
"""
|
|
159
|
+
data = mpy_path.read_bytes()
|
|
160
|
+
if len(data) < 4 or data[0] != MPY_HEADER_MAGIC:
|
|
161
|
+
raise BuildError(f"{mpy_path}: does not look like a compiled .mpy (bad header)")
|
|
162
|
+
feat = data[2]
|
|
163
|
+
arch_code = (feat >> 2) & 0x2F
|
|
164
|
+
arch_flags = 0
|
|
165
|
+
if feat & MPY_ARCH_FLAGS_BIT:
|
|
166
|
+
arch_flags, _consumed = _read_varint(data, 4)
|
|
167
|
+
return data[1], arch_code, arch_flags
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def read_native_arch(mpy_path: Path) -> int:
|
|
171
|
+
"""The MP_NATIVE_ARCH_* code baked into a native .mpy's own header."""
|
|
172
|
+
return read_mpy_header(mpy_path)[1]
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def verify_output(build_options: BuildOptions, mpy_path: Path) -> None:
|
|
176
|
+
"""cibuildmp's equivalent of auditwheel: the header the linker actually
|
|
177
|
+
wrote must name the arch (and, for rv32imc, the arch-flags) this target
|
|
178
|
+
was building for, not just live in the right build/<arch>*/ directory --
|
|
179
|
+
catches "built the wrong thing into the right directory" the way a wheel
|
|
180
|
+
tag/platform mismatch would.
|
|
181
|
+
|
|
182
|
+
Exact match on arch_flags, not the "required subset of available" rule
|
|
183
|
+
mip applies when installing (micropython/micropython#19479) -- that
|
|
184
|
+
rule is about whether a *device* can run this file; this check is about
|
|
185
|
+
whether the *linker* encoded what the config actually asked for.
|
|
186
|
+
"""
|
|
187
|
+
target = build_options.target
|
|
188
|
+
_version, actual_arch, actual_flags = read_mpy_header(mpy_path)
|
|
189
|
+
expected_arch = NATIVE_ARCH_CODE[target.arch]
|
|
190
|
+
if actual_arch != expected_arch:
|
|
191
|
+
raise BuildError(
|
|
192
|
+
f"{build_options.identifier}: {mpy_path.name}'s header encodes native "
|
|
193
|
+
f"arch code {actual_arch}, expected {expected_arch} ({target.arch})"
|
|
194
|
+
)
|
|
195
|
+
if actual_flags != target.arch_flags:
|
|
196
|
+
raise BuildError(
|
|
197
|
+
f"{build_options.identifier}: {mpy_path.name}'s header encodes "
|
|
198
|
+
f"arch_flags {actual_flags:#x}, expected {target.arch_flags:#x}"
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def output_name(build_options: BuildOptions, mpy_path: Path) -> str:
|
|
203
|
+
# Identifier-qualified even though the file already lives in its own
|
|
204
|
+
# identifier/ directory: package.json's own urls stay unambiguous even
|
|
205
|
+
# if a caller later flattens several identifiers' directories into one
|
|
206
|
+
# namespace (e.g. a GitHub Release's own asset list, which cannot nest
|
|
207
|
+
# directories -- see D14's "still open" deployment note).
|
|
208
|
+
return f"{mpy_path.stem}-{build_options.identifier}{mpy_path.suffix}"
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _write_package_json(path: Path, urls: list[tuple[str, str]], version: str) -> None:
|
|
212
|
+
manifest = {
|
|
213
|
+
"urls": [[target_path, url] for target_path, url in urls],
|
|
214
|
+
"version": version,
|
|
215
|
+
}
|
|
216
|
+
path.write_text(json.dumps(manifest, indent=2) + "\n")
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def package_target(
|
|
220
|
+
build_options: BuildOptions,
|
|
221
|
+
identifier_dir: Path,
|
|
222
|
+
install_name: str,
|
|
223
|
+
mpy_dest: Path,
|
|
224
|
+
extra_files: list[Path],
|
|
225
|
+
version: str,
|
|
226
|
+
) -> None:
|
|
227
|
+
"""Copy `extra-files` alongside the built `.mpy` and write this
|
|
228
|
+
identifier's own `package.json` (**D14**): today's plain, always-
|
|
229
|
+
supported two-element `urls` schema, not a unified multi-arch manifest
|
|
230
|
+
-- see build.__doc__ and docs/BACKLOG.md D14 for why.
|
|
231
|
+
|
|
232
|
+
`urls` entries are `[target_path, url]`, deliberately not the same
|
|
233
|
+
string twice: `target_path` is what `import <module>` needs on-device
|
|
234
|
+
-- the project's own original basename (e.g. `template.mpy`), not the
|
|
235
|
+
identifier-qualified one `mpy_dest` was stored under -- while `url` is
|
|
236
|
+
that qualified, collision-safe filename actually sitting next to this
|
|
237
|
+
`package.json`. Conflating the two would make mip install the file
|
|
238
|
+
under its long, arch-qualified name, and `import template` would not
|
|
239
|
+
find it.
|
|
240
|
+
|
|
241
|
+
A no-op when `version` is unset: an identifier directory with a `.mpy`
|
|
242
|
+
and no `package.json` is still useful (the file itself is what a
|
|
243
|
+
Makefile-driven consumer wants), it just is not mip-installable yet.
|
|
244
|
+
"""
|
|
245
|
+
if not version:
|
|
246
|
+
return
|
|
247
|
+
urls = [(install_name, mpy_dest.name)]
|
|
248
|
+
for extra in extra_files:
|
|
249
|
+
if not extra.is_file():
|
|
250
|
+
raise BuildError(
|
|
251
|
+
f"{build_options.identifier}: extra-files entry not found: {extra}"
|
|
252
|
+
)
|
|
253
|
+
dest = identifier_dir / extra.name
|
|
254
|
+
dest.write_bytes(extra.read_bytes())
|
|
255
|
+
urls.append((extra.name, extra.name))
|
|
256
|
+
_write_package_json(identifier_dir / "package.json", urls, version)
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def build_target(
|
|
260
|
+
build_options: BuildOptions,
|
|
261
|
+
chain: ResolvedToolchain,
|
|
262
|
+
mpy_dir: Path,
|
|
263
|
+
module_root: Path,
|
|
264
|
+
output_dir: Path,
|
|
265
|
+
*,
|
|
266
|
+
extra_files: Sequence[Path] = (),
|
|
267
|
+
version: str = "",
|
|
268
|
+
) -> BuildResult:
|
|
269
|
+
"""Run one target's build end to end: pre-build-command, make, collect,
|
|
270
|
+
verify, package.
|
|
271
|
+
|
|
272
|
+
Writes into `output_dir/<identifier>/`, not a flat `output_dir/` --
|
|
273
|
+
every identifier gets its own directory from the start (D14), so there
|
|
274
|
+
is no separate reorganising step between building and having something
|
|
275
|
+
mip can install. Two targets can never collide here: `Target` is keyed
|
|
276
|
+
on (abi, mode, arch, tag, arch_flags), and natmod_targets()/tag_groups()
|
|
277
|
+
both dedupe by construction, so distinct targets always get distinct
|
|
278
|
+
identifiers and therefore distinct directories.
|
|
279
|
+
"""
|
|
280
|
+
start = time.time()
|
|
281
|
+
env = chain.env()
|
|
282
|
+
|
|
283
|
+
run_pre_build_command(module_root, build_options.pre_build_command, env)
|
|
284
|
+
run_make(build_options, mpy_dir, module_root, env)
|
|
285
|
+
|
|
286
|
+
produced = collect_output(build_options, module_root)
|
|
287
|
+
verify_output(build_options, produced)
|
|
288
|
+
|
|
289
|
+
identifier_dir = output_dir / build_options.identifier
|
|
290
|
+
identifier_dir.mkdir(parents=True, exist_ok=True)
|
|
291
|
+
dest = identifier_dir / output_name(build_options, produced)
|
|
292
|
+
dest.write_bytes(produced.read_bytes())
|
|
293
|
+
|
|
294
|
+
package_target(
|
|
295
|
+
build_options, identifier_dir, produced.name, dest, list(extra_files), version
|
|
296
|
+
)
|
|
297
|
+
|
|
298
|
+
return BuildResult(
|
|
299
|
+
identifier=build_options.identifier, output=dest, duration=time.time() - start
|
|
300
|
+
)
|
cibuildmp/cli.py
ADDED
|
@@ -0,0 +1,367 @@
|
|
|
1
|
+
"""Command line interface."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import shutil
|
|
9
|
+
import sys
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
from . import __version__
|
|
13
|
+
from .build import BuildError, BuildResult, build_target
|
|
14
|
+
from .options import BuildOptions, ConfigError, Options
|
|
15
|
+
from .sources import (
|
|
16
|
+
SourceError,
|
|
17
|
+
build_mpy_cross,
|
|
18
|
+
cache_root,
|
|
19
|
+
fetch_micropython,
|
|
20
|
+
read_mpy_abi,
|
|
21
|
+
)
|
|
22
|
+
from .targets import (
|
|
23
|
+
LATEST_KNOWN_ABI,
|
|
24
|
+
NATMOD_ARCHS,
|
|
25
|
+
Target,
|
|
26
|
+
UnknownArchError,
|
|
27
|
+
is_abi_known,
|
|
28
|
+
)
|
|
29
|
+
from .toolchains import ResolvedToolchain, ToolchainError, resolve
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
33
|
+
parser = argparse.ArgumentParser(
|
|
34
|
+
prog="cibuildmp",
|
|
35
|
+
description="Build MicroPython native C extensions across every target "
|
|
36
|
+
"a module supports, from one declarative config.",
|
|
37
|
+
epilog="Most options are supplied via cibuildmp.toml or CIBMP_* environment "
|
|
38
|
+
"variables. See docs/BACKLOG.md for the design and what is implemented.",
|
|
39
|
+
)
|
|
40
|
+
parser.add_argument(
|
|
41
|
+
"package_dir",
|
|
42
|
+
nargs="?",
|
|
43
|
+
default=".",
|
|
44
|
+
type=Path,
|
|
45
|
+
help='Directory containing the module and its config (default: ".")',
|
|
46
|
+
)
|
|
47
|
+
parser.add_argument(
|
|
48
|
+
"--version",
|
|
49
|
+
action="version",
|
|
50
|
+
version=f"cibuildmp {__version__}",
|
|
51
|
+
)
|
|
52
|
+
parser.add_argument(
|
|
53
|
+
"--config-file",
|
|
54
|
+
type=Path,
|
|
55
|
+
default=None,
|
|
56
|
+
help="Config file to use instead of <package_dir>/cibuildmp.toml",
|
|
57
|
+
)
|
|
58
|
+
parser.add_argument(
|
|
59
|
+
"--output-dir",
|
|
60
|
+
type=Path,
|
|
61
|
+
default=None,
|
|
62
|
+
help="Where to collect built .mpy files (overrides the config)",
|
|
63
|
+
)
|
|
64
|
+
parser.add_argument(
|
|
65
|
+
"--only",
|
|
66
|
+
default=None,
|
|
67
|
+
metavar="IDENTIFIER",
|
|
68
|
+
help="Build exactly this one identifier, overriding the config's own "
|
|
69
|
+
"build/skip selectors. Opt into a job-per-target CI layout with this; "
|
|
70
|
+
"the default is one invocation building every selected target.",
|
|
71
|
+
)
|
|
72
|
+
parser.add_argument(
|
|
73
|
+
"--archs",
|
|
74
|
+
default=None,
|
|
75
|
+
help="Comma-separated list of architectures to build, or 'all'. Overrides "
|
|
76
|
+
"the config's own archs. There is no 'auto': every natmod arch is a "
|
|
77
|
+
"cross-compile, so none of them depends on what this machine is.",
|
|
78
|
+
)
|
|
79
|
+
parser.add_argument(
|
|
80
|
+
"--toolchain",
|
|
81
|
+
default="auto",
|
|
82
|
+
choices=["auto", "host", "download"],
|
|
83
|
+
help="How to obtain each target's cross toolchain. auto (default) uses "
|
|
84
|
+
"one already on PATH and downloads a pinned tarball otherwise; host "
|
|
85
|
+
"refuses to download; download ignores what is on PATH.",
|
|
86
|
+
)
|
|
87
|
+
parser.add_argument(
|
|
88
|
+
"--clean-cache",
|
|
89
|
+
action="store_true",
|
|
90
|
+
help="Delete cibuildmp's cache (MicroPython checkouts, mpy-cross builds "
|
|
91
|
+
"and downloaded toolchains) and exit",
|
|
92
|
+
)
|
|
93
|
+
parser.add_argument(
|
|
94
|
+
"--dry-run",
|
|
95
|
+
action="store_true",
|
|
96
|
+
help="Print the resolved build plan and exit without building",
|
|
97
|
+
)
|
|
98
|
+
parser.add_argument(
|
|
99
|
+
"--print-build-identifiers",
|
|
100
|
+
action="store_true",
|
|
101
|
+
help="Print the identifiers this config selects, one per line, then exit",
|
|
102
|
+
)
|
|
103
|
+
parser.add_argument(
|
|
104
|
+
"--print-build-matrix",
|
|
105
|
+
action="store_true",
|
|
106
|
+
help="Print a JSON array of {only, os} objects, then exit. Feed it to a "
|
|
107
|
+
"GitHub Actions `strategy.matrix.include` via fromJSON() to get one job "
|
|
108
|
+
"per target. Only worth it when targets need different runners.",
|
|
109
|
+
)
|
|
110
|
+
parser.add_argument(
|
|
111
|
+
"--json",
|
|
112
|
+
action="store_true",
|
|
113
|
+
help="With --print-build-identifiers, emit a JSON array instead of lines",
|
|
114
|
+
)
|
|
115
|
+
parser.add_argument(
|
|
116
|
+
"--allow-empty",
|
|
117
|
+
action="store_true",
|
|
118
|
+
help="Do not report an error code if no target is selected",
|
|
119
|
+
)
|
|
120
|
+
parser.add_argument(
|
|
121
|
+
"--debug-traceback",
|
|
122
|
+
action="store_true",
|
|
123
|
+
default=os.environ.get("CIBMP_DEBUG_TRACEBACK", "") not in {"", "0"},
|
|
124
|
+
help="Print a full traceback for all errors",
|
|
125
|
+
)
|
|
126
|
+
parser.add_argument(
|
|
127
|
+
"--platform",
|
|
128
|
+
default="natmod",
|
|
129
|
+
choices=["natmod"],
|
|
130
|
+
help="Build mode. Only natmod is implemented; usermod is planned "
|
|
131
|
+
"(see docs/BACKLOG.md).",
|
|
132
|
+
)
|
|
133
|
+
return parser
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _plan_line(
|
|
137
|
+
index: int,
|
|
138
|
+
total: int,
|
|
139
|
+
options: BuildOptions,
|
|
140
|
+
chain: ResolvedToolchain | None = None,
|
|
141
|
+
) -> str:
|
|
142
|
+
make = ["make", "-C", options.module_dir, f"ARCH={options.target.arch}"]
|
|
143
|
+
make += options.extra_make_args
|
|
144
|
+
make.append(options.make_target)
|
|
145
|
+
# The prefix actually in play, which is not always the one dynruntime.mk
|
|
146
|
+
# hardcodes -- showing target.cross here would contradict the CROSS=
|
|
147
|
+
# override sitting in the same line's make command.
|
|
148
|
+
prefix = chain.prefix if chain is not None else options.target.cross
|
|
149
|
+
# Right-align the counter so the columns after it stay put once the
|
|
150
|
+
# index gains a digit ([10/10] is wider than [9/10]).
|
|
151
|
+
counter = f"[{index:>{len(str(total))}}/{total}]"
|
|
152
|
+
return (
|
|
153
|
+
f"{counter} {options.target.identifier:<28} "
|
|
154
|
+
f"CROSS={prefix or '(host)':<22} {' '.join(make)}"
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def build(options: Options, targets: list[Target], *, toolchain: str = "auto") -> int:
|
|
159
|
+
"""Build every selected target in one invocation.
|
|
160
|
+
|
|
161
|
+
Sequential and in-process on purpose (D9), the same shape cibuildwheel
|
|
162
|
+
uses for the Python versions inside one runner. Fetching MicroPython and
|
|
163
|
+
building mpy-cross are identical for every natmod arch *sharing an ABI*,
|
|
164
|
+
so doing them once per ABI group here is strictly cheaper than paying
|
|
165
|
+
for them in each of ten matrix legs -- and unlike cibuildwheel, no
|
|
166
|
+
natmod target needs a runner any other target cannot use, so nothing
|
|
167
|
+
forces a fan-out. Callers who want one anyway (failure isolation,
|
|
168
|
+
wall-clock) opt in with --only.
|
|
169
|
+
|
|
170
|
+
Grouped by MicroPython tag (**D13**): almost always one group, since
|
|
171
|
+
that is the common case, but `tag_groups()` can hand back more than one
|
|
172
|
+
when `micropython` spans an ABI boundary, and each needs its own
|
|
173
|
+
checkout and its own mpy-cross.
|
|
174
|
+
"""
|
|
175
|
+
resolved = [options.build_options(t) for t in targets]
|
|
176
|
+
total = len(resolved)
|
|
177
|
+
|
|
178
|
+
# Toolchains are resolved before the plan is printed, not after: where a
|
|
179
|
+
# toolchain's prefix is not the one dynruntime.mk hardcodes, resolution
|
|
180
|
+
# adds a CROSS= override, and a plan printed first would show a make
|
|
181
|
+
# command that is not the one about to run.
|
|
182
|
+
print(f"cibuildmp: resolving toolchains for {total} target(s)")
|
|
183
|
+
chains = []
|
|
184
|
+
for build_options in resolved:
|
|
185
|
+
chain = resolve(build_options.target.arch, strategy=toolchain)
|
|
186
|
+
chains.append(chain)
|
|
187
|
+
build_options.extra_make_args = [
|
|
188
|
+
*chain.make_overrides,
|
|
189
|
+
*build_options.extra_make_args,
|
|
190
|
+
]
|
|
191
|
+
|
|
192
|
+
tags = ", ".join(tag for tag, _abi in options.tag_groups())
|
|
193
|
+
print(f"\ncibuildmp: {total} target(s) against MicroPython {tags}")
|
|
194
|
+
for index, (build_options, chain) in enumerate(
|
|
195
|
+
zip(resolved, chains, strict=True), 1
|
|
196
|
+
):
|
|
197
|
+
print(" " + _plan_line(index, total, build_options, chain))
|
|
198
|
+
print(f" {chain.describe()}")
|
|
199
|
+
|
|
200
|
+
# Resolved once, not per target: the same files and version apply to
|
|
201
|
+
# every identifier's own package.json (D14). Checked up front so a
|
|
202
|
+
# missing extra-files entry fails before any target builds, not after
|
|
203
|
+
# the first one succeeds.
|
|
204
|
+
extra_files = [options.package_dir / f for f in options.extra_files()]
|
|
205
|
+
for extra in extra_files:
|
|
206
|
+
if not extra.is_file():
|
|
207
|
+
raise BuildError(f"extra-files entry not found: {extra}")
|
|
208
|
+
|
|
209
|
+
results: list[BuildResult] = []
|
|
210
|
+
index = 0
|
|
211
|
+
# Preserves first-appearance order (options.targets() emits one ABI
|
|
212
|
+
# group at a time), not sorted -- a later tag never jumps ahead of an
|
|
213
|
+
# earlier one just because it sorts first.
|
|
214
|
+
build_tags = list(dict.fromkeys(bo.target.tag for bo in resolved))
|
|
215
|
+
for tag in build_tags:
|
|
216
|
+
group = [
|
|
217
|
+
(bo, chain)
|
|
218
|
+
for bo, chain in zip(resolved, chains, strict=True)
|
|
219
|
+
if bo.target.tag == tag
|
|
220
|
+
]
|
|
221
|
+
abi = group[0][0].target.abi # one ABI per tag group, by construction
|
|
222
|
+
|
|
223
|
+
# Shared setup, paid once per ABI group rather than once per target
|
|
224
|
+
# in it -- see build()'s own docstring and D9.
|
|
225
|
+
print(f"\ncibuildmp: preparing MicroPython {tag}")
|
|
226
|
+
mpy_dir = fetch_micropython(tag, submodules=options.micropython_submodules)
|
|
227
|
+
build_mpy_cross(mpy_dir)
|
|
228
|
+
|
|
229
|
+
# The checkout is authoritative about the ABI; targets.MPY_ABI's
|
|
230
|
+
# table is only a way to answer the question without one. A
|
|
231
|
+
# disagreement means the identifiers already printed are wrong, so
|
|
232
|
+
# it stops here rather than producing files labelled with an ABI
|
|
233
|
+
# they do not have.
|
|
234
|
+
actual_abi = read_mpy_abi(mpy_dir)
|
|
235
|
+
if actual_abi != abi:
|
|
236
|
+
raise SourceError(
|
|
237
|
+
f"MicroPython {tag} has .mpy ABI {actual_abi}, but the "
|
|
238
|
+
f"identifiers were built assuming {abi}. Set `mpy-abi = "
|
|
239
|
+
f'"{actual_abi}"` in the config, or report the stale entry in '
|
|
240
|
+
f"cibuildmp.targets.MPY_ABI."
|
|
241
|
+
)
|
|
242
|
+
|
|
243
|
+
print(f"\ncibuildmp: building {len(group)} target(s) for MicroPython {tag}")
|
|
244
|
+
for build_options, chain in group:
|
|
245
|
+
index += 1
|
|
246
|
+
print("\n " + _plan_line(index, total, build_options, chain))
|
|
247
|
+
module_root = options.package_dir / build_options.module_dir
|
|
248
|
+
output_dir = options.package_dir / build_options.output_dir
|
|
249
|
+
result = build_target(
|
|
250
|
+
build_options,
|
|
251
|
+
chain,
|
|
252
|
+
mpy_dir,
|
|
253
|
+
module_root,
|
|
254
|
+
output_dir,
|
|
255
|
+
extra_files=extra_files,
|
|
256
|
+
version=options.version,
|
|
257
|
+
)
|
|
258
|
+
results.append(result)
|
|
259
|
+
print(f" done in {result.duration:.1f}s -> {result.output}")
|
|
260
|
+
|
|
261
|
+
total_duration = sum(r.duration for r in results)
|
|
262
|
+
print(f"\ncibuildmp: {total} target(s) built in {total_duration:.1f}s")
|
|
263
|
+
for result in results:
|
|
264
|
+
print(f" {result.identifier}: {result.output.name} ({result.size} bytes)")
|
|
265
|
+
return 0
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def main(argv: list[str] | None = None) -> int:
|
|
269
|
+
parser = build_parser()
|
|
270
|
+
args = parser.parse_args(argv)
|
|
271
|
+
|
|
272
|
+
if args.clean_cache:
|
|
273
|
+
root = cache_root()
|
|
274
|
+
if not root.exists():
|
|
275
|
+
print(f"cibuildmp: nothing to clean ({root} does not exist)")
|
|
276
|
+
return 0
|
|
277
|
+
shutil.rmtree(root)
|
|
278
|
+
print(f"cibuildmp: removed {root}")
|
|
279
|
+
return 0
|
|
280
|
+
|
|
281
|
+
try:
|
|
282
|
+
options = Options.load(args.package_dir, args.config_file)
|
|
283
|
+
if args.output_dir is not None:
|
|
284
|
+
options.output_dir = args.output_dir
|
|
285
|
+
if args.archs is not None:
|
|
286
|
+
options.archs = (
|
|
287
|
+
list(NATMOD_ARCHS)
|
|
288
|
+
if args.archs.strip() == "all"
|
|
289
|
+
else [a.strip() for a in args.archs.split(",") if a.strip()]
|
|
290
|
+
)
|
|
291
|
+
targets = options.targets()
|
|
292
|
+
except (ConfigError, UnknownArchError, SourceError) as exc:
|
|
293
|
+
if args.debug_traceback:
|
|
294
|
+
raise
|
|
295
|
+
print(f"cibuildmp: error: {exc}", file=sys.stderr)
|
|
296
|
+
return 2
|
|
297
|
+
|
|
298
|
+
if args.only is not None:
|
|
299
|
+
# --only overrides build/skip, matching cibuildwheel's own semantics
|
|
300
|
+
# for the flag: the caller has already decided what this invocation
|
|
301
|
+
# is for, and a matrix leg that reached here was selected when the
|
|
302
|
+
# matrix was generated.
|
|
303
|
+
targets = [t for t in targets if t.identifier == args.only]
|
|
304
|
+
if not targets:
|
|
305
|
+
print(
|
|
306
|
+
f"cibuildmp: error: --only {args.only!r} matches no target this "
|
|
307
|
+
f"config can produce",
|
|
308
|
+
file=sys.stderr,
|
|
309
|
+
)
|
|
310
|
+
return 2
|
|
311
|
+
|
|
312
|
+
if args.print_build_matrix:
|
|
313
|
+
print(
|
|
314
|
+
json.dumps(
|
|
315
|
+
[
|
|
316
|
+
{"only": bo.target.identifier, "os": bo.runs_on}
|
|
317
|
+
for bo in (options.build_options(t) for t in targets)
|
|
318
|
+
]
|
|
319
|
+
)
|
|
320
|
+
)
|
|
321
|
+
return 0
|
|
322
|
+
|
|
323
|
+
if args.print_build_identifiers:
|
|
324
|
+
identifiers = [t.identifier for t in targets]
|
|
325
|
+
if args.json:
|
|
326
|
+
print(json.dumps(identifiers))
|
|
327
|
+
else:
|
|
328
|
+
for identifier in identifiers:
|
|
329
|
+
print(identifier)
|
|
330
|
+
return 0
|
|
331
|
+
|
|
332
|
+
if not targets:
|
|
333
|
+
if args.allow_empty:
|
|
334
|
+
print("cibuildmp: no targets selected")
|
|
335
|
+
return 0
|
|
336
|
+
print(
|
|
337
|
+
"cibuildmp: error: no targets selected. Pass --allow-empty if that "
|
|
338
|
+
"is expected.",
|
|
339
|
+
file=sys.stderr,
|
|
340
|
+
)
|
|
341
|
+
return 2
|
|
342
|
+
|
|
343
|
+
unknown_tags = [tag for tag in options.micropython if not is_abi_known(tag)]
|
|
344
|
+
if unknown_tags:
|
|
345
|
+
print(
|
|
346
|
+
f"cibuildmp: warning: no recorded .mpy ABI for MicroPython "
|
|
347
|
+
f"{', '.join(unknown_tags)}; assuming {LATEST_KNOWN_ABI}. The ABI "
|
|
348
|
+
f"actually encoded in each built .mpy is verified against its "
|
|
349
|
+
f"identifier, so a wrong guess fails the build rather than shipping.",
|
|
350
|
+
file=sys.stderr,
|
|
351
|
+
)
|
|
352
|
+
|
|
353
|
+
if args.dry_run:
|
|
354
|
+
total = len(targets)
|
|
355
|
+
tags = ", ".join(tag for tag, _abi in options.tag_groups())
|
|
356
|
+
print(f"cibuildmp: {total} target(s) against MicroPython {tags}")
|
|
357
|
+
for index, target in enumerate(targets, 1):
|
|
358
|
+
print(" " + _plan_line(index, total, options.build_options(target)))
|
|
359
|
+
return 0
|
|
360
|
+
|
|
361
|
+
try:
|
|
362
|
+
return build(options, targets, toolchain=args.toolchain)
|
|
363
|
+
except (SourceError, ToolchainError, BuildError) as exc:
|
|
364
|
+
if args.debug_traceback:
|
|
365
|
+
raise
|
|
366
|
+
print(f"cibuildmp: error: {exc}", file=sys.stderr)
|
|
367
|
+
return 2
|