cibuildmp 0.3.0a1__py3-none-any.whl

cibuildmp/__init__.py

@@ -0,0 +1,19 @@
+"""cibuildmp -- build MicroPython native C extensions on CI, and locally."""
+
+from __future__ import annotations
+
+import sys
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("cibuildmp")
+except PackageNotFoundError:  # running from a source tree, not installed
+    __version__ = "0.0.0.dev0"
+
+from .cli import main as _main
+
+__all__ = ["__version__", "main"]
+
+
+def main() -> None:
+    sys.exit(_main())

cibuildmp/__main__.py

@@ -0,0 +1,4 @@
+from cibuildmp import main
+
+if __name__ == "__main__":
+    main()

cibuildmp/cli.py

@@ -0,0 +1,322 @@
+"""Command line interface."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import shutil
+import sys
+from pathlib import Path
+
+from . import __version__
+from .options import BuildOptions, ConfigError, Options
+from .sources import (
+    SourceError,
+    build_mpy_cross,
+    cache_root,
+    fetch_micropython,
+    read_mpy_abi,
+)
+from .targets import NATMOD_ARCHS, Target, UnknownArchError, is_abi_known
+from .toolchains import ResolvedToolchain, ToolchainError, resolve
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="cibuildmp",
+        description="Build MicroPython native C extensions across every target "
+        "a module supports, from one declarative config.",
+        epilog="Most options are supplied via cibuildmp.toml or CIBMP_* environment "
+        "variables. See docs/BACKLOG.md for the design and what is implemented.",
+    )
+    parser.add_argument(
+        "package_dir",
+        nargs="?",
+        default=".",
+        type=Path,
+        help='Directory containing the module and its config (default: ".")',
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=f"cibuildmp {__version__}",
+    )
+    parser.add_argument(
+        "--config-file",
+        type=Path,
+        default=None,
+        help="Config file to use instead of <package_dir>/cibuildmp.toml",
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=Path,
+        default=None,
+        help="Where to collect built .mpy files (overrides the config)",
+    )
+    parser.add_argument(
+        "--only",
+        default=None,
+        metavar="IDENTIFIER",
+        help="Build exactly this one identifier, overriding the config's own "
+        "build/skip selectors. Opt into a job-per-target CI layout with this; "
+        "the default is one invocation building every selected target.",
+    )
+    parser.add_argument(
+        "--archs",
+        default=None,
+        help="Comma-separated list of architectures to build, or 'all'. Overrides "
+        "the config's own archs. There is no 'auto': every natmod arch is a "
+        "cross-compile, so none of them depends on what this machine is.",
+    )
+    parser.add_argument(
+        "--toolchain",
+        default="auto",
+        choices=["auto", "host", "download"],
+        help="How to obtain each target's cross toolchain. auto (default) uses "
+        "one already on PATH and downloads a pinned tarball otherwise; host "
+        "refuses to download; download ignores what is on PATH.",
+    )
+    parser.add_argument(
+        "--clean-cache",
+        action="store_true",
+        help="Delete cibuildmp's cache (MicroPython checkouts, mpy-cross builds "
+        "and downloaded toolchains) and exit",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Print the resolved build plan and exit without building",
+    )
+    parser.add_argument(
+        "--print-build-identifiers",
+        action="store_true",
+        help="Print the identifiers this config selects, one per line, then exit",
+    )
+    parser.add_argument(
+        "--print-build-matrix",
+        action="store_true",
+        help="Print a JSON array of {only, os} objects, then exit. Feed it to a "
+        "GitHub Actions `strategy.matrix.include` via fromJSON() to get one job "
+        "per target. Only worth it when targets need different runners.",
+    )
+    parser.add_argument(
+        "--json",
+        action="store_true",
+        help="With --print-build-identifiers, emit a JSON array instead of lines",
+    )
+    parser.add_argument(
+        "--allow-empty",
+        action="store_true",
+        help="Do not report an error code if no target is selected",
+    )
+    parser.add_argument(
+        "--debug-traceback",
+        action="store_true",
+        default=os.environ.get("CIBMP_DEBUG_TRACEBACK", "") not in {"", "0"},
+        help="Print a full traceback for all errors",
+    )
+    parser.add_argument(
+        "--platform",
+        default="natmod",
+        choices=["natmod"],
+        help="Build mode. Only natmod is implemented; usermod is planned "
+        "(see docs/BACKLOG.md).",
+    )
+    return parser
+
+
+def _plan_line(
+    index: int,
+    total: int,
+    options: BuildOptions,
+    chain: ResolvedToolchain | None = None,
+) -> str:
+    make = ["make", "-C", options.module_dir, f"ARCH={options.target.arch}"]
+    make += options.extra_make_args
+    make.append(options.make_target)
+    # The prefix actually in play, which is not always the one dynruntime.mk
+    # hardcodes -- showing target.cross here would contradict the CROSS=
+    # override sitting in the same line's make command.
+    prefix = chain.prefix if chain is not None else options.target.cross
+    # Right-align the counter so the columns after it stay put once the
+    # index gains a digit ([10/10] is wider than [9/10]).
+    counter = f"[{index:>{len(str(total))}}/{total}]"
+    return (
+        f"{counter} {options.target.identifier:<28} "
+        f"CROSS={prefix or '(host)':<22} {' '.join(make)}"
+    )
+
+
+def build(options: Options, targets: list[Target], *, toolchain: str = "auto") -> int:
+    """Build every selected target in one invocation.
+
+    Sequential and in-process on purpose (D9), the same shape cibuildwheel
+    uses for the Python versions inside one runner. Fetching MicroPython and
+    building mpy-cross are identical for every natmod arch, so doing them
+    once here is strictly cheaper than paying for them in each of ten matrix
+    legs -- and unlike cibuildwheel, no natmod target needs a runner any
+    other target cannot use, so nothing forces a fan-out. Callers who want
+    one anyway (failure isolation, wall-clock) opt in with --only.
+    """
+    resolved = [options.build_options(t) for t in targets]
+    total = len(resolved)
+
+    # Toolchains are resolved before the plan is printed, not after: where a
+    # toolchain's prefix is not the one dynruntime.mk hardcodes, resolution
+    # adds a CROSS= override, and a plan printed first would show a make
+    # command that is not the one about to run.
+    print(f"cibuildmp: resolving toolchains for {total} target(s)")
+    chains = []
+    for build_options in resolved:
+        chain = resolve(build_options.target.arch, strategy=toolchain)
+        chains.append(chain)
+        build_options.extra_make_args = [
+            *chain.make_overrides,
+            *build_options.extra_make_args,
+        ]
+
+    print(
+        f"\ncibuildmp: {total} target(s) against MicroPython {options.micropython} "
+        f"(.mpy ABI {options.abi})"
+    )
+    for index, (build_options, chain) in enumerate(
+        zip(resolved, chains, strict=True), 1
+    ):
+        print("  " + _plan_line(index, total, build_options, chain))
+        print(f"        {chain.describe()}")
+
+    # Shared setup, paid once for the whole invocation rather than once per
+    # matrix leg -- see build()'s own docstring and D9.
+    print("\ncibuildmp: preparing MicroPython")
+    mpy_dir = fetch_micropython(
+        options.micropython, submodules=options.micropython_submodules
+    )
+    build_mpy_cross(mpy_dir)
+
+    # The checkout is authoritative about the ABI; targets.MPY_ABI's table
+    # is only a way to answer the question without one. A disagreement means
+    # the identifiers already printed are wrong, so it stops here rather
+    # than producing files labelled with an ABI they do not have.
+    actual_abi = read_mpy_abi(mpy_dir)
+    if actual_abi != options.abi:
+        raise SourceError(
+            f"MicroPython {options.micropython} has .mpy ABI {actual_abi}, but the "
+            f"identifiers were built assuming {options.abi}. Set `mpy-abi = "
+            f'"{actual_abi}"` in the config, or report the stale entry in '
+            f"cibuildmp.targets.MPY_ABI."
+        )
+
+    # M3 lands here: run make under that toolchain, then collect and verify
+    # the output.
+    sys.stdout.flush()  # keep the output above ahead of the stderr note below
+    print(
+        "\ncibuildmp: the per-target build is not implemented yet (M2 ships "
+        "toolchain resolution). Re-run with --dry-run to get the plan as a "
+        "success.",
+        file=sys.stderr,
+    )
+    return 1
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+
+    if args.clean_cache:
+        root = cache_root()
+        if not root.exists():
+            print(f"cibuildmp: nothing to clean ({root} does not exist)")
+            return 0
+        shutil.rmtree(root)
+        print(f"cibuildmp: removed {root}")
+        return 0
+
+    try:
+        options = Options.load(args.package_dir, args.config_file)
+        if args.output_dir is not None:
+            options.output_dir = args.output_dir
+        if args.archs is not None:
+            options.archs = (
+                list(NATMOD_ARCHS)
+                if args.archs.strip() == "all"
+                else [a.strip() for a in args.archs.split(",") if a.strip()]
+            )
+        targets = options.targets()
+    except (ConfigError, UnknownArchError, SourceError) as exc:
+        if args.debug_traceback:
+            raise
+        print(f"cibuildmp: error: {exc}", file=sys.stderr)
+        return 2
+
+    if args.only is not None:
+        # --only overrides build/skip, matching cibuildwheel's own semantics
+        # for the flag: the caller has already decided what this invocation
+        # is for, and a matrix leg that reached here was selected when the
+        # matrix was generated.
+        targets = [t for t in targets if t.identifier == args.only]
+        if not targets:
+            print(
+                f"cibuildmp: error: --only {args.only!r} matches no target this "
+                f"config can produce",
+                file=sys.stderr,
+            )
+            return 2
+
+    if args.print_build_matrix:
+        print(
+            json.dumps(
+                [
+                    {"only": bo.target.identifier, "os": bo.runs_on}
+                    for bo in (options.build_options(t) for t in targets)
+                ]
+            )
+        )
+        return 0
+
+    if args.print_build_identifiers:
+        identifiers = [t.identifier for t in targets]
+        if args.json:
+            print(json.dumps(identifiers))
+        else:
+            for identifier in identifiers:
+                print(identifier)
+        return 0
+
+    if not targets:
+        if args.allow_empty:
+            print("cibuildmp: no targets selected")
+            return 0
+        print(
+            "cibuildmp: error: no targets selected. Pass --allow-empty if that "
+            "is expected.",
+            file=sys.stderr,
+        )
+        return 2
+
+    if not is_abi_known(options.micropython):
+        print(
+            f"cibuildmp: warning: no recorded .mpy ABI for MicroPython "
+            f"{options.micropython}; assuming {options.abi}. The ABI actually "
+            f"encoded in each built .mpy is verified against its identifier, so "
+            f"a wrong guess fails the build rather than shipping.",
+            file=sys.stderr,
+        )
+
+    if args.dry_run:
+        total = len(targets)
+        print(
+            f"cibuildmp: {total} target(s) against MicroPython "
+            f"{options.micropython} (.mpy ABI {options.abi})"
+        )
+        for index, target in enumerate(targets, 1):
+            print("  " + _plan_line(index, total, options.build_options(target)))
+        return 0
+
+    try:
+        return build(options, targets, toolchain=args.toolchain)
+    except (SourceError, ToolchainError) as exc:
+        if args.debug_traceback:
+            raise
+        print(f"cibuildmp: error: {exc}", file=sys.stderr)
+        return 2

cibuildmp/options.py

@@ -0,0 +1,213 @@
+"""Config loading and option resolution.
+
+Precedence, lowest to highest:
+    defaults -> config file -> matching [[overrides]] -> environment -> CLI
+
+Config lives in cibuildmp.toml at the package root, with the same tree
+accepted under [tool.cibuildmp] in pyproject.toml for the rare MicroPython
+C-module repo that has one. cibuildmp.toml wins when both exist.
+"""
+
+from __future__ import annotations
+
+import os
+import shlex
+import tomllib
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from .targets import (
+    NATMOD_ARCHS,
+    Target,
+    abi_for_tag,
+    matches,
+    natmod_targets,
+    parse_selector,
+    select,
+)
+
+CONFIG_FILENAME = "cibuildmp.toml"
+
+DEFAULT_MICROPYTHON = "v1.28.0"
+DEFAULT_OUTPUT_DIR = "mpyhouse"
+DEFAULT_MODULE_DIR = "natmod"
+DEFAULT_MAKE_TARGET = "dist"
+
+
+class ConfigError(Exception):
+    pass
+
+
+def _as_list(value: Any, key: str) -> list[str]:
+    """Accept a list, or a shell-ish string, for list-valued options.
+
+    The string form exists for the environment layer -- CIBMP_EXTRA_MAKE_ARGS
+    can only ever be a string -- and is accepted in the file too so the two
+    layers do not disagree about what a valid value looks like.
+    """
+    if value is None:
+        return []
+    if isinstance(value, str):
+        return shlex.split(value)
+    if isinstance(value, list):
+        return [str(v) for v in value]
+    raise ConfigError(f"{key}: expected a list or a string, got {type(value).__name__}")
+
+
+@dataclass
+class BuildOptions:
+    """Fully resolved options for a single target."""
+
+    target: Target
+    micropython: str
+    output_dir: Path
+    module_dir: str
+    make_target: str
+    runs_on: str = ""
+    extra_make_args: list[str] = field(default_factory=list)
+    pre_build_command: str = ""
+
+    @property
+    def identifier(self) -> str:
+        return self.target.identifier
+
+
+@dataclass
+class Options:
+    """The whole config, before it is narrowed to a single target."""
+
+    package_dir: Path
+    config_path: Path | None
+    micropython: str
+    output_dir: Path
+    build: list[str]
+    skip: list[str]
+    archs: list[str]
+    micropython_submodules: list[str]
+    mpy_abi: str | None
+    natmod: dict[str, Any]
+    overrides: list[dict[str, Any]]
+
+    # ── Loading ───────────────────────────────────────────────────────────
+
+    @classmethod
+    def load(
+        cls,
+        package_dir: Path,
+        config_file: Path | None = None,
+        env: Mapping[str, str] | None = None,
+    ) -> Options:
+        environ: Mapping[str, str] = os.environ if env is None else env
+        config_path, raw = _read_config(package_dir, config_file)
+
+        natmod = dict(raw.get("natmod") or {})
+        overrides = list(raw.get("overrides") or [])
+        if not isinstance(overrides, list):
+            raise ConfigError("[[overrides]] must be an array of tables")
+
+        def opt(key: str, default: Any = None) -> Any:
+            # Environment beats the file for every global option. Keys are
+            # kebab-case in TOML (matching cibuildwheel) and
+            # CIBMP_SCREAMING_SNAKE in the environment.
+            env_value = environ.get("CIBMP_" + key.replace("-", "_").upper())
+            if env_value is not None:
+                return env_value
+            return raw.get(key, default)
+
+        archs_value = opt("archs") or natmod.get("archs") or list(NATMOD_ARCHS)
+
+        return cls(
+            package_dir=package_dir,
+            config_path=config_path,
+            micropython=str(opt("micropython", DEFAULT_MICROPYTHON)),
+            output_dir=Path(str(opt("output-dir", DEFAULT_OUTPUT_DIR))),
+            build=parse_selector(opt("build", "*")),
+            skip=parse_selector(opt("skip", "")),
+            archs=_as_list(archs_value, "archs"),
+            micropython_submodules=_as_list(
+                opt("micropython-submodules"), "micropython-submodules"
+            ),
+            mpy_abi=(str(opt("mpy-abi")) if opt("mpy-abi") is not None else None),
+            natmod=natmod,
+            overrides=overrides,
+        )
+
+    # ── Resolution ────────────────────────────────────────────────────────
+
+    @property
+    def abi(self) -> str:
+        return abi_for_tag(self.micropython, self.mpy_abi)
+
+    def targets(self) -> list[Target]:
+        """Every target this config selects, in NATMOD_ARCHS order."""
+        return select(natmod_targets(self.archs, self.abi), self.build, self.skip)
+
+    def build_options(
+        self, target: Target, env: Mapping[str, str] | None = None
+    ) -> BuildOptions:
+        """Resolve per-target options: file -> overrides -> environment."""
+        environ: Mapping[str, str] = os.environ if env is None else env
+
+        layers: list[dict[str, Any]] = [self.natmod]
+        for override in self.overrides:
+            selector = override.get("select")
+            if selector is None:
+                raise ConfigError("every [[overrides]] table needs a `select` key")
+            if matches(target.identifier, parse_selector(selector)):
+                layers.append(override)
+
+        def opt(key: str, default: Any = None) -> Any:
+            env_value = environ.get("CIBMP_" + key.replace("-", "_").upper())
+            if env_value is not None:
+                return env_value
+            # Later layers win: a matching override beats [natmod].
+            for layer in reversed(layers):
+                if key in layer:
+                    return layer[key]
+            return default
+
+        return BuildOptions(
+            target=target,
+            micropython=self.micropython,
+            output_dir=self.output_dir,
+            module_dir=str(opt("module-dir", DEFAULT_MODULE_DIR)),
+            make_target=str(opt("make-target", DEFAULT_MAKE_TARGET)),
+            runs_on=str(opt("runs-on", target.default_runner)),
+            extra_make_args=_as_list(opt("extra-make-args"), "extra-make-args"),
+            pre_build_command=str(opt("pre-build-command", "")),
+        )
+
+
+def _read_config(
+    package_dir: Path, config_file: Path | None
+) -> tuple[Path | None, dict[str, Any]]:
+    if config_file is not None:
+        if not config_file.is_file():
+            raise ConfigError(f"config file not found: {config_file}")
+        return config_file, _load_toml_tree(config_file)
+
+    standalone = package_dir / CONFIG_FILENAME
+    if standalone.is_file():
+        return standalone, _load_toml_tree(standalone)
+
+    pyproject = package_dir / "pyproject.toml"
+    if pyproject.is_file():
+        with pyproject.open("rb") as f:
+            data = tomllib.load(f)
+        tool = (data.get("tool") or {}).get("cibuildmp")
+        if tool is not None:
+            return pyproject, dict(tool)
+
+    # No config at all is legitimate: every option has a default, so a repo
+    # following the conventional natmod/ layout builds with none.
+    return None, {}
+
+
+def _load_toml_tree(path: Path) -> dict[str, Any]:
+    with path.open("rb") as f:
+        data = tomllib.load(f)
+    if path.name == "pyproject.toml":
+        return dict((data.get("tool") or {}).get("cibuildmp") or {})
+    return data