brkas 0.1.0__py3-none-any.whl

brkas/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+"""brkas: kas-like project files for Buildroot."""
+__version__ = "0.1.0"

brkas/__main__.py

@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+import sys
+
+from .cli import main
+
+sys.exit(main())

brkas/buildroot.py

@@ -0,0 +1,328 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+"""Drives Buildroot for a project: configure, build, menuconfig, shell."""
+import hashlib
+import json
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import time
+
+from .config import ConfigError, expand
+from .repos import RepoError, log, run
+
+CONFIG_LINE = re.compile(r"^(BR2_[A-Za-z0-9_]+)=(.*)$")
+ENVIRONMENT_SETUP = "BR2_PACKAGE_HOST_ENVIRONMENT_SETUP"
+SPINNER = "|/-\\"
+UNSET_LINE = re.compile(r"^# (BR2_[A-Za-z0-9_]+) is not set$")
+
+
+def render(symbol, value):
+    """One .config line for a value of the config: section."""
+    if value is True or value == "y":
+        return f"{symbol}=y"
+    if value is False or value == "n":
+        return f"# {symbol} is not set"
+    if value == "m" or isinstance(value, int):
+        return f"{symbol}={value}"
+    text = str(value)
+    if len(text) >= 2 and text[0] == text[-1] == '"':
+        return f"{symbol}={text}"
+    return f'{symbol}="' + text.replace("\\", "\\\\").replace('"', '\\"') + '"'
+
+
+def parse_config(text):
+    """{symbol: value} of a .config or fragment, 'n' for unset symbols."""
+    values = {}
+    for line in text.splitlines():
+        line = line.strip()
+        m = CONFIG_LINE.match(line)
+        if m:
+            values[m.group(1)] = m.group(2)
+            continue
+        m = UNSET_LINE.match(line)
+        if m:
+            values[m.group(1)] = "n"
+    return values
+
+
+def read_config(path):
+    try:
+        with open(path) as f:
+            return parse_config(f.read())
+    except OSError:
+        return {}
+
+
+def config_diff(before, after):
+    """Fragment lines that turn before into after."""
+    lines = []
+    for symbol in sorted(set(before) | set(after)):
+        old, new = before.get(symbol, "n"), after.get(symbol, "n")
+        if old != new:
+            lines.append(f"# {symbol} is not set" if new == "n" else f"{symbol}={new}")
+    return lines
+
+
+def symbol_of(line):
+    m = CONFIG_LINE.match(line) or UNSET_LINE.match(line)
+    return m.group(1) if m else None
+
+
+def save_fragment(path, lines):
+    """Writes lines in a fragment file: a symbol already in it takes its
+    new value, other lines are kept."""
+    old = []
+    if os.path.exists(path):
+        with open(path) as f:
+            old = [line.rstrip("\n") for line in f]
+    names = {symbol_of(line) for line in lines}
+    kept = [line for line in old if symbol_of(line) not in names or symbol_of(line) is None]
+    os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True)
+    with open(path, "w") as f:
+        f.write("\n".join(kept + lines) + "\n")
+
+
+class Builder:
+    def __init__(self, project, repos, work_dir, out_dir=None, shell=False):
+        self.project = project
+        self.repos = repos
+        build = project.data.get("build") or {}
+        root = build.get("dir") or "build"
+        if not os.path.isabs(root):
+            root = os.path.join(work_dir, root)
+        self.out = out_dir or os.path.join(root, project.name)
+        self.br = repos["buildroot"].tree
+        self.externals = [repos[name].tree for name in project.externals()]
+        # brkas shell needs host/environment-setup: the shell enables its
+        # package, which then stays on so that a build does not drop it
+        self.environment_setup_on = shell or \
+            read_config(os.path.join(self.out, ".config")).get(ENVIRONMENT_SETUP) == "y"
+
+    def env(self):
+        env = dict(os.environ)
+        for key, value in (self.project.data.get("env") or {}).items():
+            if value is None:
+                env.pop(key, None)
+            else:
+                env[key] = str(value)
+        dl_dir = (self.project.data.get("build") or {}).get("dl_dir")
+        if dl_dir:
+            env["BR2_DL_DIR"] = expand(dl_dir)
+        return env
+
+    def make_cmd(self, *targets):
+        cmd = ["make", "-C", self.br, f"O={self.out}"]
+        cmd.append("BR2_EXTERNAL=" + ":".join(self.externals))
+        return cmd + list(targets)
+
+    def make(self, *targets, interactive=True):
+        cmd = self.make_cmd(*targets)
+        if interactive:
+            if subprocess.run(cmd, env=self.env()).returncode:
+                raise RepoError(f"make {' '.join(targets)} failed")
+            return None
+        p = subprocess.run(cmd, env=self.env(), text=True, capture_output=True)
+        if p.returncode:
+            raise RepoError(f"make {' '.join(targets)} failed:\n{(p.stdout + p.stderr).strip()[-4000:]}")
+        return p.stdout
+
+    def jobs(self):
+        """-j with every available core, unless the project sets it in
+        MAKEFLAGS; Buildroot only runs packages in parallel with
+        BR2_PER_PACKAGE_DIRECTORIES, and ignores it otherwise."""
+        if "-j" in self.env().get("MAKEFLAGS", ""):
+            return []
+        return [f"-j{len(os.sched_getaffinity(0))}"]
+
+    def make_quietly(self, target, what):
+        """make target with its output in a log file, and a spinner on a
+        terminal meanwhile; the end of the log is shown if it fails."""
+        path = os.path.join(self.out, f"brkas-{target}.log")
+        tty = sys.stderr.isatty()
+        log(f"{what}: make {target}, output in {path}")
+        start = time.monotonic()
+        with open(path, "w") as out:
+            p = subprocess.Popen(self.make_cmd(*self.jobs(), target), env=self.env(), stdout=out,
+                                 stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL)
+            try:
+                frame = 0
+                while p.poll() is None:
+                    if tty:
+                        elapsed = int(time.monotonic() - start)
+                        sys.stderr.write(f"\rbrkas: {what} {SPINNER[frame % len(SPINNER)]} "
+                                         f"{elapsed // 60:02d}:{elapsed % 60:02d}")
+                        sys.stderr.flush()
+                        frame += 1
+                    time.sleep(0.2)
+            except KeyboardInterrupt:
+                p.terminate()
+                p.wait()
+                raise
+            finally:
+                if tty:
+                    sys.stderr.write("\r\033[K")
+                    sys.stderr.flush()
+        elapsed = int(time.monotonic() - start)
+        if p.returncode:
+            with open(path, errors="replace") as f:
+                tail = f.readlines()[-30:]
+            sys.stderr.write("".join(tail))
+            raise RepoError(f"make {target} failed, the whole output is in {path}")
+        log(f"{what}: done in {elapsed // 60:02d}:{elapsed % 60:02d}")
+
+    # --- configuration -------------------------------------------------
+
+    def defconfig(self):
+        """('target', name) or ('file', path) or None."""
+        entry = self.project.data.get("defconfig")
+        if entry is None:
+            return None
+        if isinstance(entry, dict):
+            return ("file", self.project.locate(entry))
+        if os.path.isabs(entry):
+            return ("file", entry)
+        name = entry if entry.endswith("_defconfig") else entry + "_defconfig"
+        for tree in [self.br] + self.externals:
+            if os.path.exists(os.path.join(tree, "configs", name)):
+                return ("target", name)
+        raise ConfigError(f"defconfig {name} not found in Buildroot nor in the externals")
+
+    def fragment_files(self):
+        return [(ident, self.project.locate(entry) if isinstance(entry, dict) else entry)
+                for ident, entry in self.project.fragments()]
+
+    def inline_fragment(self):
+        lines = [ENVIRONMENT_SETUP + "=y"] if self.environment_setup_on else []
+        # config: comes last, it may still turn it off
+        return lines + [render(sym, value) for sym, value in self.project.config().items()]
+
+    def digest(self):
+        h = hashlib.sha256()
+        h.update(json.dumps([self.br] + self.externals).encode())
+        kind = self.defconfig()
+        h.update(json.dumps(kind).encode())
+        files = [p for _, p in self.fragment_files()] + ([kind[1]] if kind and kind[0] == "file" else [])
+        if kind and kind[0] == "target":
+            files += [os.path.join(t, "configs", kind[1]) for t in [self.br] + self.externals
+                      if os.path.exists(os.path.join(t, "configs", kind[1]))]
+        for path in files:
+            with open(path, "rb") as f:
+                h.update(f.read())
+        h.update("\n".join(self.inline_fragment()).encode())
+        for repo in self.repos.values():
+            h.update(json.dumps(repo.state(), sort_keys=True).encode())
+        return h.hexdigest()
+
+    def configure(self, force=False):
+        stamp = os.path.join(self.out, ".brkas-config")
+        digest = self.digest()
+        if not force and os.path.exists(os.path.join(self.out, ".config")):
+            try:
+                with open(stamp) as f:
+                    if f.read().strip() == digest:
+                        return False
+            except OSError:
+                pass
+        os.makedirs(self.out, exist_ok=True)
+        kind = self.defconfig()
+        log(f"configuring {self.out}" + (f" from {os.path.basename(kind[1])}" if kind else ""))
+        if kind is None:
+            self.make("defconfig", interactive=False)
+        elif kind[0] == "target":
+            self.make(kind[1], interactive=False)
+        else:
+            self.make("defconfig", f"BR2_DEFCONFIG={kind[1]}", interactive=False)
+
+        fragments = [p for _, p in self.fragment_files()]
+        inline = self.inline_fragment()
+        if inline:
+            path = os.path.join(self.out, "brkas-config.fragment")
+            with open(path, "w") as f:
+                f.write("# config: of the brkas project\n" + "\n".join(inline) + "\n")
+            fragments.append(path)
+        if fragments:
+            run(["sh", os.path.join(self.br, "support/kconfig/merge_config.sh"), "-m",
+                 "-O", self.out, os.path.join(self.out, ".config"), *fragments], cwd=self.out)
+            self.make("olddefconfig", interactive=False)
+        self.check(fragments)
+        with open(stamp, "w") as f:
+            f.write(digest + "\n")
+        return True
+
+    def check(self, fragments):
+        """Warns about values Kconfig did not keep (missing dependencies)."""
+        wanted = {}
+        for path in fragments:
+            with open(path) as f:
+                wanted.update(parse_config(f.read()))
+        got = read_config(os.path.join(self.out, ".config"))
+        for symbol, value in wanted.items():
+            if got.get(symbol, "n") != value:
+                log(f"warning: {symbol}={value} asked, the configuration has "
+                    f"{symbol}={got.get(symbol, 'n')} (a dependency missing?)")
+
+    # --- commands ------------------------------------------------------
+
+    def build(self, targets):
+        self.configure()
+        targets = targets or (self.project.data.get("build") or {}).get("targets") or []
+        self.make(*targets)
+
+    def menuconfig(self, tool, save):
+        self.configure()
+        config = os.path.join(self.out, ".config")
+        before = read_config(config)
+        self.make(tool)
+        lines = config_diff(before, read_config(config))
+        if not lines:
+            log("no change")
+            return lines
+        if save:
+            save_fragment(save, lines)
+            log(f"{len(lines)} changes saved in {save}")
+        else:
+            print("\n".join(lines))
+        # The saved fragment is part of the project: no reconfigure next time
+        with open(os.path.join(self.out, ".brkas-config"), "w") as f:
+            f.write(self.digest() + "\n")
+        return lines
+
+    def sdk_built(self):
+        """make sdk ran: it writes host/share/buildroot/sdk-location."""
+        return os.path.exists(os.path.join(self.out, "host", "share", "buildroot", "sdk-location"))
+
+    def environment_setup(self, reconfigured=False):
+        """host/environment-setup, after 'make sdk' when the SDK is not
+        built yet, or the configuration changed since."""
+        config = read_config(os.path.join(self.out, ".config"))
+        if config.get(ENVIRONMENT_SETUP, "n") != "y":
+            log(f"{ENVIRONMENT_SETUP} is off: a shell without the cross-compilation environment")
+            return None
+        setup = os.path.join(self.out, "host", "environment-setup")
+        if reconfigured or not self.sdk_built() or not os.path.exists(setup):
+            self.make_quietly("sdk", "building the SDK")
+        return setup
+
+    def shell(self, command=None):
+        """bash in the output directory, with host/environment-setup sourced
+        (cross compiler, CC, CFLAGS, ARCH, CROSS_COMPILE...), building the
+        SDK first if needed."""
+        reconfigured = self.configure()
+        env = self.env()
+        env["BRKAS_OUTPUT"] = self.out
+        env["BRKAS_BUILDROOT"] = self.br
+        setup = self.environment_setup(reconfigured)
+        if command:
+            # its aliases (cmake, configure) too: they apply from the next line
+            source = f'shopt -s expand_aliases\n. "{setup}" >/dev/null\n' if setup else ""
+            return subprocess.run(["bash", "-c", source + command], cwd=self.out, env=env).returncode
+        with tempfile.NamedTemporaryFile("w", prefix="brkas-rc.", suffix=".sh") as rc:
+            rc.write('[ -f ~/.bashrc ] && . ~/.bashrc\n')
+            if setup:
+                rc.write(f'. "{setup}"\n')
+            rc.write('PS1="(brkas) $PS1"\n')
+            rc.flush()
+            log(f"shell in {self.out}: exit to leave")
+            return subprocess.run(["bash", "--rcfile", rc.name, "-i"], cwd=self.out, env=env).returncode

brkas/cli.py

@@ -0,0 +1,264 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+"""brkas: set up and build Buildroot projects described in YAML files."""
+import argparse
+import os
+import shutil
+import subprocess
+import sys
+
+import yaml
+
+from . import __version__, config
+from .buildroot import Builder
+from .config import ConfigError
+from .repos import Repo, RepoError, log
+
+
+def make_repos(project, work_dir):
+    repos = {"buildroot": Repo("buildroot", project.buildroot(), work_dir)}
+    for name, spec in project.externals().items():
+        if name == "buildroot":
+            raise ConfigError("externals: 'buildroot' is reserved for Buildroot itself")
+        repos[name] = Repo(name, spec, work_dir)
+    return repos
+
+
+def _patch_deps(repo):
+    return {e.get("repo") for e in (repo.spec.get("patches") or {}).values()
+            if isinstance(e, dict) and e.get("repo")} - {None}
+
+
+def prepare(args, checkout=True):
+    """The project and its repos, checked out with the includes they hold."""
+    work = os.path.abspath(args.work_dir)
+    # lock --update makes a new lock: the old one must not pin the repos
+    locks = not (args.func is cmd_lock and args.update)
+    project = config.load(args.project, locks=locks)
+    for _ in range(8):
+        repos = make_repos(project, work)
+        project.repo_dirs = {name: repo.root for name, repo in repos.items()}
+        if checkout:
+            # repos providing patches to others go first
+            for repo in sorted(repos.values(), key=lambda r: bool(_patch_deps(r))):
+                repo.checkout(project.locate, update=args.update, force=args.force)
+        elif project.missing:
+            raise ConfigError("includes from repos not checked out yet: " + ", ".join(sorted(project.missing))
+                              + " (run brkas checkout first)")
+        if not project.missing:
+            return project, repos
+        project = config.load(args.project, project.repo_dirs, locks)
+        if not checkout:
+            continue
+    raise ConfigError("includes keep pointing to new repos: " + ", ".join(sorted(project.missing)))
+
+
+def builder(args, project, repos, shell=False):
+    return Builder(project, repos, os.path.abspath(args.work_dir), args.output, shell)
+
+
+def lock_data(repos):
+    data = {"header": {"version": config.FORMAT_VERSION}}
+    pins = {}
+    for name, repo in repos.items():
+        pin = repo.lock()
+        if pin:
+            pins[name] = pin
+    if "buildroot" in pins:
+        data["buildroot"] = pins.pop("buildroot")
+    if pins:
+        data["externals"] = pins
+    return data
+
+
+# --- commands ----------------------------------------------------------
+
+def cmd_checkout(args):
+    project, repos = prepare(args)
+    for name, repo in repos.items():
+        log(f"{name}: {repo.describe()} -> {repo.tree}")
+    if not args.no_config:
+        builder(args, project, repos).configure(force=args.reconfigure)
+
+
+def cmd_build(args):
+    project, repos = prepare(args)
+    b = builder(args, project, repos)
+    b.configure(force=args.reconfigure)
+    b.build(args.targets)
+
+
+def cmd_shell(args):
+    project, repos = prepare(args)
+    return builder(args, project, repos, shell=True).shell(args.command)
+
+
+def cmd_menuconfig(args):
+    project, repos = prepare(args)
+    builder(args, project, repos).menuconfig(args.tool, args.save)
+
+
+CONFIG_TOOLS = {
+    "menuconfig": "change the configuration in menuconfig, save the changes as a fragment",
+    "xconfig": "the same, in xconfig (Qt)",
+}
+
+
+def cmd_dump(args):
+    project, repos = prepare(args, checkout=not args.no_checkout)
+    data = lock_data(repos) if args.lock else dict({"header": {"version": config.FORMAT_VERSION}}, **project.data)
+    yaml.safe_dump(data, sys.stdout, sort_keys=False, default_flow_style=False)
+
+
+def cmd_lock(args):
+    project, repos = prepare(args)
+    path = config.lock_path(project.files[-1])
+    with open(path, "w") as f:
+        f.write("# Written by brkas lock: the exact sources of the project\n")
+        yaml.safe_dump(lock_data(repos), f, sort_keys=False, default_flow_style=False)
+    log(f"wrote {path}")
+
+
+def cmd_for_all_repos(args):
+    project, repos = prepare(args, checkout=not args.no_checkout)
+    status = 0
+    for name, repo in repos.items():
+        env = dict(os.environ, BRKAS_REPO_NAME=name, BRKAS_REPO_DIR=repo.root,
+                   BRKAS_REPO_URL=repo.spec.get("url", ""),
+                   BRKAS_REPO_COMMIT=repo.state().get("base", ""))
+        log(f"{name}: {args.command}")
+        status |= subprocess.run(args.command, shell=True, cwd=repo.root, env=env).returncode
+    return status
+
+
+def remove(path):
+    if os.path.isdir(path):
+        log(f"removing {path}")
+        shutil.rmtree(path)
+
+
+def download_cache(project, work):
+    """Where Buildroot downloads for this project: build.dl_dir, else
+    $BR2_DL_DIR, else dl/ in the Buildroot tree."""
+    dl_dir = (project.data.get("build") or {}).get("dl_dir") or os.environ.get("BR2_DL_DIR")
+    if dl_dir:
+        return os.path.abspath(config.expand(dl_dir))
+    return os.path.join(make_repos(project, work)["buildroot"].tree, "dl")
+
+
+def remove_cache(path):
+    """Empties the cache; the directory itself may stay, as when it is a
+    mount point in brkas-container."""
+    if not os.path.isdir(path):
+        return
+    log(f"removing {path}")
+    for entry in os.listdir(path):
+        entry = os.path.join(path, entry)
+        if os.path.isdir(entry) and not os.path.islink(entry):
+            shutil.rmtree(entry)
+        else:
+            os.unlink(entry)
+    try:
+        os.rmdir(path)
+    except OSError:
+        pass
+
+
+def clean_work_dir(work):
+    """clean --all without a project: what brkas made in the work
+    directory, i.e. build/, downloads/ and the repos it checked out (those
+    with its state file)."""
+    for name in ("build", "downloads"):
+        remove(os.path.join(work, name))
+    for entry in sorted(os.listdir(work)) if os.path.isdir(work) else []:
+        path = os.path.join(work, entry)
+        if os.path.exists(os.path.join(path, ".git", "brkas.json")) or \
+                os.path.exists(os.path.join(path, ".brkas.json")):
+            remove(path)
+
+
+def cmd_clean(args):
+    work = os.path.abspath(args.work_dir)
+    if not args.project:
+        if args.cache:
+            raise ConfigError("clean --cache needs a project: the cache is its build.dl_dir")
+        if not args.all or args.output:
+            raise ConfigError("clean needs a project, or --all to clean the whole work directory")
+        return clean_work_dir(work)
+    project = config.load(args.project)
+    if args.cache:
+        remove_cache(download_cache(project, work))
+        if not args.all:
+            return  # only the cache: the output stays
+    build = project.data.get("build") or {}
+    root = build.get("dir") or "build"
+    out_dir = args.output or os.path.join(root if os.path.isabs(root) else os.path.join(work, root), project.name)
+    remove(out_dir)
+    if args.all:
+        for name, repo in make_repos(project, work).items():
+            if repo.kind != "local":
+                remove(repo.root)
+
+
+# --- arguments ---------------------------------------------------------
+
+def parser():
+    p = argparse.ArgumentParser(prog="brkas", description=__doc__)
+    p.add_argument("--version", action="version", version=f"brkas {__version__}")
+    p.add_argument("--work-dir", default=os.environ.get("BRKAS_WORK_DIR", "."),
+                   help="where repos and build/ go (default: $BRKAS_WORK_DIR or .)")
+    sub = p.add_subparsers(dest="command", required=True, metavar="command")
+
+    def add(name, func, help, checkout=True, project="project file(s), a.yml or a.yml:b.yml"):
+        s = sub.add_parser(name, help=help, description=help)
+        if project:
+            s.add_argument("project", help=project)
+        s.set_defaults(func=func)
+        if checkout:
+            s.add_argument("--update", action="store_true", help="fetch the repos, to follow their branches")
+            s.add_argument("--force", action="store_true", help="discard local changes in the repos")
+            s.add_argument("-o", "--output", help="output directory (default: build/<name>)")
+        return s
+
+    s = add("checkout", cmd_checkout, "check out the repos and configure the output directory")
+    s.add_argument("--no-config", action="store_true", help="only check out the repos")
+    s.add_argument("--reconfigure", action="store_true", help="configure again from scratch")
+    s = add("build", cmd_build, "check out, configure and build")
+    s.add_argument("--reconfigure", action="store_true", help="configure again from scratch")
+    s.add_argument("targets", nargs="*", help="make targets (default: build.targets, else all)")
+    s = add("shell", cmd_shell, "bash in the output directory with the SDK environment, after make sdk if needed")
+    s.add_argument("-c", "--command", help="run this command instead of an interactive shell")
+    for tool, help in CONFIG_TOOLS.items():
+        s = add(tool, cmd_menuconfig, help)
+        s.set_defaults(tool=tool)
+        s.add_argument("--save", metavar="FRAGMENT", help="fragment file to write the changes to (default: print them)")
+    s = add("dump", cmd_dump, "print the merged project, or with --lock its exact sources")
+    s.add_argument("--lock", action="store_true", help="print the lock file contents")
+    s.add_argument("--no-checkout", action="store_true", help="use the repos as they are")
+    add("lock", cmd_lock, "pin the repos: write <last project file>.lock.yml")
+    s = add("for-all-repos", cmd_for_all_repos, "run a shell command in every repo")
+    s.add_argument("--no-checkout", action="store_true", help="use the repos as they are")
+    s.add_argument("command", help="command, with BRKAS_REPO_{NAME,DIR,URL,COMMIT} set")
+    s = add("clean", cmd_clean, "remove the output directory", checkout=False, project=None)
+    s.add_argument("project", nargs="?", help="project file(s); without it, --all cleans the whole "
+                                              "work directory")
+    s.add_argument("-o", "--output", help="output directory (default: build/<name>)")
+    s.add_argument("--all", action="store_true", help="also remove the repos brkas checked out")
+    s.add_argument("--cache", action="store_true",
+                   help="remove the download cache of the project (build.dl_dir) and nothing else; "
+                        "with --all, the cache as well")
+    return p
+
+
+def main(argv=None):
+    args = parser().parse_args(argv)
+    try:
+        return args.func(args) or 0
+    except (ConfigError, RepoError) as e:
+        print(f"brkas: error: {e}", file=sys.stderr)
+        return 1
+    except KeyboardInterrupt:
+        return 130
+
+
+if __name__ == "__main__":
+    sys.exit(main())