binary-manager 0.1.1__tar.gz → 0.1.2__tar.gz

{binary_manager-0.1.1 → binary_manager-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: binary-manager
-Version: 0.1.1
+Version: 0.1.2
 Summary: Package firmware binaries into images described by a devicetree
 Author-email: Simon Glass <sjg@chromium.org>
 License-Expression: GPL-2.0-or-later

{binary_manager-0.1.1 → binary_manager-0.1.2}/binary_manager.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: binary-manager
-Version: 0.1.1
+Version: 0.1.2
 Summary: Package firmware binaries into images described by a devicetree
 Author-email: Simon Glass <sjg@chromium.org>
 License-Expression: GPL-2.0-or-later

{binary_manager-0.1.1 → binary_manager-0.1.2}/binary_manager.egg-info/SOURCES.txt

@@ -12,6 +12,10 @@ binman/__main__.py
 binman/binman.rst
 binman/bintool.py
 binman/bintool_test.py
+binman/blob.py
+binman/blob_test.py
+binman/blobstore.py
+binman/blobstores.yaml
 binman/cbfs_util.py
 binman/cbfs_util_test.py
 binman/cmdline.py
@@ -21,6 +25,7 @@ binman/elf_test.py
 binman/entry.py
 binman/entry_test.py
 binman/fdt_test.py
+binman/fetchbase.py
 binman/fip_util.py
 binman/fip_util_test.py
 binman/fmap_util.py
@@ -48,6 +53,9 @@ binman/_vendor/u_boot_pylib/test_gitutil.py
 binman/_vendor/u_boot_pylib/test_util.py
 binman/_vendor/u_boot_pylib/tools.py
 binman/_vendor/u_boot_pylib/tout.py
+binman/blobs/__init__.py
+binman/blobs/_testing.py
+binman/blobs/atf.py
 binman/btool/_testing.py
 binman/btool/bootgen.py
 binman/btool/btool_gzip.py
@@ -223,6 +231,8 @@ binman/test/blob/ext_missing.dts
 binman/test/blob/ext_missing_sect.dts
 binman/test/blob/ext_opt.dts
 binman/test/blob/fake.dts
+binman/test/blob/fetch.dts
+binman/test/blob/fetch_other.dts
 binman/test/blob/missing.dts
 binman/test/blob/named_by_arg.dts
 binman/test/blob/symbol.dts

{binary_manager-0.1.1 → binary_manager-0.1.2}/binman/_vendor/u_boot_pylib/test_util.py

@@ -24,7 +24,7 @@ except:
 
 def run_test_coverage(prog, filter_fname, exclude_list, build_dir,
                       required=None, extra_args=None, single_thread='-P1',
-                      args=None):
+                      args=None, test_cmd=None):
     """Run tests and check that we get 100% coverage
 
     Args:
@@ -43,6 +43,9 @@ def run_test_coverage(prog, filter_fname, exclude_list, build_dir,
             single-threaded. This is necessary to get proper coverage results.
             The default is '-P0'
         args (list of str): List of tests to run, or None to run all
+        test_cmd (str): Argument which tells the tool to run its tests, or
+            None to choose from the name of the program: 'test' for binman,
+            patman and pickman, else '-t'
 
     Raises:
         ValueError if the code coverage is not 100%
@@ -57,8 +60,9 @@ def run_test_coverage(prog, filter_fname, exclude_list, build_dir,
     glob_list += exclude_list
     glob_list += ['*libfdt.py', '*/site-packages/*', '*/dist-packages/*']
     glob_list += ['*concurrencytest*']
-    use_test = 'binman' in prog or 'patman' in prog or 'pickman' in prog
-    test_cmd = 'test' if use_test else '-t'
+    if not test_cmd:
+        name = os.path.basename(prog)
+        test_cmd = 'test' if name in ('binman', 'patman', 'pickman') else '-t'
     prefix = ''
     if build_dir:
         prefix = 'PYTHONPATH=$PYTHONPATH:%s/sandbox_spl/tools ' % build_dir

{binary_manager-0.1.1 → binary_manager-0.1.2}/binman/binman.rst

@@ -1644,15 +1644,199 @@ is auto-generated from the source code.
 
    bintools
 
+Fetching firmware blobs
+=======================
+
+Images often need firmware which is built by another project, such as the
+BL31 image from ARM Trusted Firmware (TF-A). Binman can fetch some of these
+blobs, preferring to build them from source over downloading prebuilt binaries.
+
+Each blob type is identified by a compatible string, such as
+``arm,trusted-firmware-a``. To see the blob types which binman knows about, and
+the stores it can fetch them from, use::
+
+    binman blob --list
+    binman blob --list-stores
+    binman blob --info arm,trusted-firmware-a
+
+To fetch a blob, give its version along with the architecture and platform it
+is for::
+
+    binman blob --fetch arm,trusted-firmware-a --version 2.12 \
+        --arch aarch64 --plat rk3399
+
+Binman first tries to build the blob from source, which needs the relevant
+toolchain. For TF-A this is an aarch64 cross-compiler, given by
+``CROSS_COMPILE`` (default ``aarch64-linux-gnu-``). If that fails, binman tries
+the blob's other stores in priority order, lowest number first. Use
+``--source-only`` to only build from source, or ``--no-source`` to skip
+building. Binman exits with an error if the blob cannot be fetched.
+
+Blobs are built in ``~/.cache/binman/blob-build``, or under ``XDG_CACHE_HOME``
+if that is set, with a directory for each git repository. This holds the
+repository in ``src``, with a working tree for each version which is built,
+e.g.::
+
+    ~/.cache/binman/blob-build/trusted-firmware-a-1a2b3c4d/v2.12
+
+The build is kept, so building the same version again reuses it, which is
+faster and does not need to fetch the source again. If a build fails, binman
+says where it is, so that you can look at what went wrong. Use the
+``--blob-build-dir`` option or the ``BINMAN_BLOB_BUILD_DIR`` environment
+variable to choose a different directory. Use ``--temp-build`` to build in a
+temporary directory which is removed afterwards, and ``--clean-builds`` to
+remove all the builds::
+
+    binman blob --clean-builds
+
+Fetched blobs are cached in ``~/.cache/binman/blobs``, under the compatible
+string, version, architecture and platform, e.g.::
+
+    ~/.cache/binman/blobs/arm,trusted-firmware-a/2.12/aarch64/sun50i_a64/bl31.bin
+
+For TF-A, this is the ELF file ``bl31.elf`` on Rockchip platforms, since TF-A
+does not produce ``bl31.bin`` for them.
+
+If ``XDG_CACHE_HOME`` is set, the cache is in ``$XDG_CACHE_HOME/binman/blobs``
+instead. Use the ``--blob-dir`` option or the ``BINMAN_BLOB_DIR`` environment
+variable to choose a different directory, for example one which is shared by
+several users or kept between CI runs.
+
+To put a blob which you have obtained some other way into the cache, use::
+
+    binman blob --add arm,trusted-firmware-a --file bl31.bin --version 2.12 \
+        --arch aarch64 --plat sun50i_a64
+
+Building an image also uses blobs. When an image needs an external blob which
+is not in the input directories (see `External blobs`_), binman looks for a
+blob type which provides that file for the board (see `Blobs for a board`_) and
+uses the file from the cache, fetching it first if needed. If no blob type
+provides the file, or it cannot be fetched, the blob is missing as usual.
+
+Blob stores
+-----------
+
+The blob types and stores are described in ``blobstores.yaml`` in the binman
+package, followed by the user's own configuration, so that, for example, a
+private server can be used without changing binman. Binman reads
+``~/.config/binman/blobstores.yaml``, if it exists, then any files listed in the
+``BINMAN_BLOBSTORES`` environment variable, separated by ``:``. Each file has
+the same format and adds stores and blob types, replacing any with the same
+name.
+
+A store has a type, a priority and a description:
+
+build
+    Build from source, using the blob type's handler
+
+local
+    Copy a local file
+
+url
+    Download a file
+
+For the local and url types, each blob type gives a ``pattern``, which is the
+file's path or URL. A url store may give a ``repo`` URL, which is put in front
+of a pattern which is not a full URL. Patterns may contain ``{version}``,
+``{arch}`` and ``{plat}``, which are replaced with the values requested. These
+stores must provide the blob file itself, since binman does not unpack
+archives. For example, to fetch TF-A from a mirror of prebuilt binaries::
+
+    stores:
+      my-mirror:
+        type: url
+        repo: https://example.com/firmware
+        priority: 50
+        desc: Prebuilt firmware mirror
+
+    blobs:
+      arm,trusted-firmware-a:
+        desc: ARM Trusted Firmware-A (BL31)
+        handler: atf
+        stores:
+          - store: source-build
+          - store: my-mirror
+            pattern: tf-a/{version}/{plat}/bl31.bin
+
+Blobs for a board
+-----------------
+
+A board often needs several blobs, such as the FSP, flash descriptor and video
+BIOS table on an x86 board, which may only be available privately. A blob type
+can list the files it provides, with the version to fetch. The ``{file}``
+placeholder in a store's pattern is replaced with the filename. A blob type
+with no handler is only fetched from its stores, so no code is needed.
+
+Such a blob type is named after a compatible string, and is used when building
+an image for a board whose devicetree root node has that compatible string.
+Since the root node lists the board's compatible string first, followed by
+more general ones such as the SoC's, a board's own blobs take precedence over
+blobs which are shared by all boards with that SoC. If the board's blob type
+cannot provide a file, binman falls back to the next one.
+
+For example, Chromebook Coral's devicetree has
+``compatible = "google,coral", "intel,apollolake"``, so its blobs could be
+provided from a private server, sharing the FSP with other Apollo Lake boards::
+
+    stores:
+      lab-server:
+        type: url
+        repo: https://blobs.example.com
+        priority: 50
+        desc: Lab blob server
+
+    blobs:
+      google,coral:
+        desc: Chromebook Coral firmware blobs
+        version: '1'
+        files: [descriptor.bin, fitimage.bin, vbt.bin]
+        stores:
+          - store: lab-server
+            pattern: coral/{version}/{file}
+
+      intel,apollolake:
+        desc: Apollo Lake FSP
+        version: '1'
+        files: [fsp_m.bin, fsp_s.bin]
+        stores:
+          - store: lab-server
+            pattern: apollolake/{version}/{file}
+
+The server then just serves the files, e.g. at
+``https://blobs.example.com/coral/1/vbt.bin``, giving an HTTP error for a file
+which does not exist. Change the version when the blobs change, so that they
+are fetched afresh rather than taken from the cache.
+
+To fetch all the files of such a blob type in advance, for example to fill the
+cache, use::
+
+    binman blob --fetch google,coral
+
+The files are cached under the blob type and version, e.g.
+``~/.cache/binman/blobs/google,coral/1/vbt.bin``
+
+Adding a blob type
+------------------
+
+Each blob type has a handler in the ``blobs/`` directory of the binman package,
+named in ``blobstores.yaml``. The handler is a module defining a class named
+``Blob<handler>`` which is a subclass of ``Blob``. To support building from
+source, it implements ``build()``, typically using ``build_from_git()``, which
+checks out a repository in the build directory, runs ``make`` and returns the
+file which was built. See ``blobs/atf.py`` for an example.
+
 Binman commands and arguments
 =============================
 
 Usage::
 
     binman [-h] [-B BUILD_DIR] [-D] [--tooldir TOOLDIR] [-H]
-        [--toolpath TOOLPATH] [-T THREADS] [--test-section-timeout]
+        [--toolpath TOOLPATH] [--blob-dir BLOB_DIR]
+        [--blob-build-dir BLOB_BUILD_DIR] [-T THREADS]
+        [--test-section-timeout]
         [-v VERBOSITY] [-V]
-        {build,bintool-docs,entry-docs,ls,extract,replace,test,tool} ...
+        {build,bintool-docs,entry-docs,ls,extract,replace,sign,test,tool,blob}
+        ...
 
 Binman provides the following commands:
 
@@ -1662,8 +1846,10 @@ Binman provides the following commands:
 - **ls** - list an image
 - **extract** - extract files from an image
 - **replace** - replace one or more entries in an image
+- **sign** - sign entries in an image
 - **test** - run tests
 - **tool** - manage bintools
+- **blob** - fetch firmware blobs (see `Fetching firmware blobs`_)
 
 Options:
 
@@ -1684,6 +1870,14 @@ Options:
 --toolpath TOOLPATH
     Add a path to the list of directories containing tools
 
+--blob-dir BLOB_DIR
+    Set the directory to cache blobs in (default ``$BINMAN_BLOB_DIR`` or
+    ``~/.cache/binman/blobs``). See `Fetching firmware blobs`_.
+
+--blob-build-dir BLOB_BUILD_DIR
+    Set the directory to build blobs in (default ``$BINMAN_BLOB_BUILD_DIR`` or
+    ``~/.cache/binman/blob-build``). See `Fetching firmware blobs`_.
+
 -T THREADS, --threads THREADS
     Number of threads to use (0=single-thread). Note that -T0 is useful for
     debugging since everything runs in one thread.
@@ -2101,7 +2295,7 @@ This is useful in CI systems which want to check that everything is correct but
 don't have access to the blobs.
 
 If the blobs are in a different directory, you can specify this with the `-I`
-option.
+option. Binman can fetch some blobs for you; see `Fetching firmware blobs`_.
 
 For U-Boot, you can set the BINMAN_INDIRS environment variable to provide a
 space-separated list of directories to search for binary blobs::

{binary_manager-0.1.1 → binary_manager-0.1.2}/binman/bintool.py

@@ -10,38 +10,34 @@ the tool, checking its version and fetching it if needed.
 """
 
 import collections
-import glob
-import importlib
-import multiprocessing
 import os
 import shutil
 import tempfile
 import urllib.error
 
+from binman import fetchbase
+
 from u_boot_pylib import command
 from u_boot_pylib import terminal
 from u_boot_pylib import tools
 from u_boot_pylib import tout
 
-BINMAN_DIR = os.path.dirname(os.path.realpath(__file__))
-
 # Format string for listing bintools, see also the header in list_all()
 FORMAT = '%-16.16s %-12.12s %-26.26s %s'
 
 # List of known modules, to avoid importing the module multiple times
 modules = {}
 
-# Possible ways of fetching a tool (FETCH_COUNT is number of ways)
-FETCH_ANY, FETCH_BIN, FETCH_BUILD, FETCH_COUNT = range(4)
-
-FETCH_NAMES = {
-    FETCH_ANY: 'any method',
-    FETCH_BIN: 'binary download',
-    FETCH_BUILD: 'build from source'
-    }
-
-# Status of tool fetching
-FETCHED, FAIL, PRESENT, STATUS_COUNT = range(4)
+# Re-export constants for backwards compatibility
+FETCH_ANY = fetchbase.FETCH_ANY
+FETCH_BIN = fetchbase.FETCH_BIN
+FETCH_BUILD = fetchbase.FETCH_BUILD
+FETCH_COUNT = fetchbase.FETCH_COUNT
+FETCH_NAMES = fetchbase.FETCH_NAMES
+FETCHED = fetchbase.FETCHED
+FAIL = fetchbase.FAIL
+PRESENT = fetchbase.PRESENT
+STATUS_COUNT = fetchbase.STATUS_COUNT
 
 class Bintool:
     """Tool which operates on binaries to help produce entry contents
@@ -76,28 +72,8 @@ class Bintool:
                 module name that could not be found
                 exception received
         """
-        # Convert something like 'u-boot' to 'u_boot' since we are only
-        # interested in the type.
-        module_name = btype.replace('-', '_')
-        module = modules.get(module_name)
-        class_name = f'Bintool{module_name}'
-
-        # Import the module if we have not already done so
-        if not module:
-            try:
-                module = importlib.import_module('binman.btool.' + module_name)
-            except ImportError as exc:
-                try:
-                    # Deal with classes which must be renamed due to conflicts
-                    # with Python libraries
-                    module = importlib.import_module('binman.btool.btool_' +
-                                                     module_name)
-                except ImportError:
-                    return module_name, exc
-            modules[module_name] = module
-
-        # Look up the expected class name
-        return getattr(module, class_name)
+        return fetchbase.find_module_class(
+            modules, btype, 'binman.btool', 'Bintool', fallback_prefix='btool_')
 
     @staticmethod
     def create(name):
@@ -142,15 +118,7 @@ class Bintool:
         Returns:
             list of str: names of all tools known to binman
         """
-        files = glob.glob(os.path.join(BINMAN_DIR, 'btool/*'))
-        names = [os.path.splitext(os.path.basename(fname))[0]
-                 for fname in files]
-        names = [name for name in names if name[0] != '_']
-        names = [name[6:] if name.startswith('btool_') else name
-                 for name in names]
-        if include_testing:
-            names.append('_testing')
-        return sorted(names)
+        return fetchbase.get_module_list('btool', include_testing)
 
     @staticmethod
     def list_all():
@@ -355,29 +323,9 @@ class Bintool:
                 str: Name of temp directory to remove, or None
             or None on error
         """
-        tmpdir = tempfile.mkdtemp(prefix='binmanf.')
-        print(f"- clone git repo '{git_repo}' to '{tmpdir}'")
-        if git_branch:
-            tools.run('git', 'clone', '--depth', '1', '--branch', git_branch,
-                      git_repo, tmpdir)
-        else:
-            tools.run('git', 'clone', '--depth', '1', git_repo, tmpdir)
-        for target in make_targets:
-            print(f"- build target '{target}'")
-            makedir = tmpdir
-            if make_path:
-                makedir = os.path.join(tmpdir, make_path)
-            cmd = ['make', '-C', makedir, '-j', f'{multiprocessing.cpu_count()}',
-                   target]
-            if flags:
-                cmd += flags
-            tools.run(*cmd)
-
-        fname = os.path.join(tmpdir, bintool_path)
-        if not os.path.exists(fname):
-            print(f"- File '{fname}' was not produced")
-            return None
-        return fname, tmpdir
+        return fetchbase.build_from_git(
+            git_repo, make_targets, bintool_path, git_branch=git_branch,
+            make_flags=flags, make_path=make_path)
 
     @classmethod
     def fetch_from_url(cls, url):
@@ -391,9 +339,7 @@ class Bintool:
                 str: Filename of fetched file to copy to a suitable directory
                 str: Name of temp directory to remove, or None
         """
-        fname, tmpdir = tools.download(url)
-        tools.run('chmod', 'a+x', fname)
-        return fname, tmpdir
+        return fetchbase.fetch_from_url(url, make_executable=True)
 
     @classmethod
     def fetch_from_drive(cls, drive_id):

{binary_manager-0.1.1 → binary_manager-0.1.2}/binman/bintool_test.py

@@ -14,6 +14,7 @@ import unittest.mock
 import urllib.error
 
 from binman import bintool
+from binman import fetchbase
 from binman.bintool import Bintool
 
 from u_boot_pylib import command
@@ -241,7 +242,7 @@ class TestBintool(unittest.TestCase):
                 str: Filename of written file (or missing 'make' output)
                 str: Contents of stdout
         """
-        def fake_run(*cmd):
+        def fake_run(*cmd, **_kwargs):
             if cmd[0] == 'make':
                 # See Bintool.build_from_git()
                 tmpdir = cmd[2]
@@ -254,7 +255,8 @@ class TestBintool(unittest.TestCase):
         self.fname = None
         with unittest.mock.patch.object(bintool.Bintool, 'tooldir',
                                         self._indir):
-            with unittest.mock.patch.object(tools, 'run', side_effect=fake_run):
+            with unittest.mock.patch.object(fetchbase, 'run_build',
+                                            side_effect=fake_run):
                 with terminal.capture() as (stdout, _):
                     btest.fetch_tool(bintool.FETCH_BUILD, col, False)
         fname = os.path.join(self._indir, '_testing')
@@ -298,7 +300,7 @@ class TestBintool(unittest.TestCase):
             tools.write_file(fname, expected)
             return fname, dirname
 
-        def fake_run(*cmd):
+        def fake_run(*cmd, **_kwargs):
             if cmd[0] == 'make':
                 # See Bintool.build_from_git()
                 tmpdir = cmd[2]
@@ -311,7 +313,9 @@ class TestBintool(unittest.TestCase):
         os.mkdir(dirname)
         fname = os.path.join(dirname, 'downloaded')
 
-        with unittest.mock.patch.object(tools, 'run', side_effect=fake_run):
+        with unittest.mock.patch.object(tools, 'run', side_effect=fake_run), \
+                unittest.mock.patch.object(fetchbase, 'run_build',
+                                           side_effect=fake_run):
             with unittest.mock.patch.object(tools, 'download',
                                             side_effect=handle_download):
                 with terminal.capture() as _: