binary-manager 0.2.0__tar.gz → 0.2.1__tar.gz

{binary_manager-0.2.0 → binary_manager-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: binary-manager
-Version: 0.2.0
+Version: 0.2.1
 Summary: Package firmware binaries into images described by a devicetree
 Author-email: Simon Glass <sjg@chromium.org>
 License-Expression: GPL-2.0-or-later
@@ -25,6 +25,7 @@ Requires-Dist: jsonschema
 Requires-Dist: PyYAML
 Requires-Dist: yamllint
 Requires-Dist: u-boot-pylib>=0.1.0
+Requires-Dist: dtoc>=0.1.0
 Provides-Extra: test
 Requires-Dist: concurrencytest; extra == "test"
 Requires-Dist: coverage; extra == "test"
@@ -146,6 +147,6 @@ since the expected output tracks its development; pass ``--toolpath`` to point
 binman at a U-Boot ``tools/`` build directory. ``binman test -T`` checks that
 the tests cover all of the code.
 
-The ``dtoc`` library is vendored from the U-Boot tree and ``u_boot_pylib`` is
-installed from PyPI as ``u-boot-pylib``, so no surrounding U-Boot source is
+The ``dtoc`` and ``u_boot_pylib`` libraries are installed from PyPI, as the
+``dtoc`` and ``u-boot-pylib`` packages, so no surrounding U-Boot source is
 required.

{binary_manager-0.2.0 → binary_manager-0.2.1}/README.rst

@@ -114,6 +114,6 @@ since the expected output tracks its development; pass ``--toolpath`` to point
 binman at a U-Boot ``tools/`` build directory. ``binman test -T`` checks that
 the tests cover all of the code.
 
-The ``dtoc`` library is vendored from the U-Boot tree and ``u_boot_pylib`` is
-installed from PyPI as ``u-boot-pylib``, so no surrounding U-Boot source is
+The ``dtoc`` and ``u_boot_pylib`` libraries are installed from PyPI, as the
+``dtoc`` and ``u-boot-pylib`` packages, so no surrounding U-Boot source is
 required.

{binary_manager-0.2.0 → binary_manager-0.2.1}/binary_manager.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: binary-manager
-Version: 0.2.0
+Version: 0.2.1
 Summary: Package firmware binaries into images described by a devicetree
 Author-email: Simon Glass <sjg@chromium.org>
 License-Expression: GPL-2.0-or-later
@@ -25,6 +25,7 @@ Requires-Dist: jsonschema
 Requires-Dist: PyYAML
 Requires-Dist: yamllint
 Requires-Dist: u-boot-pylib>=0.1.0
+Requires-Dist: dtoc>=0.1.0
 Provides-Extra: test
 Requires-Dist: concurrencytest; extra == "test"
 Requires-Dist: coverage; extra == "test"
@@ -146,6 +147,6 @@ since the expected output tracks its development; pass ``--toolpath`` to point
 binman at a U-Boot ``tools/`` build directory. ``binman test -T`` checks that
 the tests cover all of the code.
 
-The ``dtoc`` library is vendored from the U-Boot tree and ``u_boot_pylib`` is
-installed from PyPI as ``u-boot-pylib``, so no surrounding U-Boot source is
+The ``dtoc`` and ``u_boot_pylib`` libraries are installed from PyPI, as the
+``dtoc`` and ``u-boot-pylib`` packages, so no surrounding U-Boot source is
 required.

{binary_manager-0.2.0 → binary_manager-0.2.1}/binary_manager.egg-info/SOURCES.txt

@@ -38,9 +38,6 @@ binman/image_test.py
 binman/main.py
 binman/missing-blob-help
 binman/state.py
-binman/_vendor/dtoc/__init__.py
-binman/_vendor/dtoc/fdt.py
-binman/_vendor/dtoc/fdt_util.py
 binman/blobs/__init__.py
 binman/blobs/_testing.py
 binman/blobs/atf.py
@@ -426,6 +423,7 @@ binman/test/fit/fit.dts
 binman/test/fit/gennode.dts
 binman/test/fit/image_subentry_alignment.dts
 binman/test/fit/key_name_hint_dir_signature.dts
+binman/test/fit/large.dts
 binman/test/fit/missing_blob.dts
 binman/test/fit/openssl.conf
 binman/test/fit/replace_sibling.dts

{binary_manager-0.2.0 → binary_manager-0.2.1}/binary_manager.egg-info/requires.txt

@@ -5,6 +5,7 @@ jsonschema
 PyYAML
 yamllint
 u-boot-pylib>=0.1.0
+dtoc>=0.1.0
 
 [test]
 concurrencytest

binary_manager-0.2.1/binman/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: GPL-2.0+
+
+"""Binman firmware packager"""

{binary_manager-0.2.0 → binary_manager-0.2.1}/binman/bintool.py

@@ -374,7 +374,8 @@ class Bintool:
             args = ['sudo', 'apt-get', 'update', '-y']
             print('- %s' % ' '.join(args))
             tools.run(*args)
-            cls.apt_updated = True
+            # Set this on the base class, so that it covers every bintool
+            Bintool.apt_updated = True
         args = ['sudo', 'apt-get', 'install', '-y', package]
         print('- %s' % ' '.join(args))
         tools.run(*args)

{binary_manager-0.2.0 → binary_manager-0.2.1}/binman/bintool_test.py

@@ -8,6 +8,7 @@
 import collections
 import os
 import shutil
+import sys
 import tempfile
 import unittest
 import unittest.mock
@@ -22,6 +23,23 @@ from u_boot_pylib import terminal
 from u_boot_pylib import test_util
 from u_boot_pylib import tools
 
+# Fake 'sudo' which records each command it is given. It handles
+# 'apt-get install' by creating a stand-in for the tool in the package, so
+# that the tool can then be found on the PATH
+FAKE_SUDO = '''#!{python}
+import os
+import sys
+
+bindir = os.path.dirname(os.path.abspath(sys.argv[0]))
+with open(os.path.join(bindir, 'log'), 'a') as outf:
+    print(' '.join(sys.argv[1:]), file=outf)
+if sys.argv[1:3] == ['apt-get', 'install']:
+    fname = os.path.join(bindir, sys.argv[-1])
+    with open(fname, 'w') as outf:
+        print('#!/bin/sh', file=outf)
+    os.chmod(fname, 0o755)
+'''
+
 # pylint: disable=R0904
 class TestBintool(unittest.TestCase):
     """Tests for the Bintool class"""
@@ -284,6 +302,36 @@ class TestBintool(unittest.TestCase):
                 result = btest.fetch_tool(bintool.FETCH_BIN, col, False)
         self.assertEqual(bintool.FETCHED, result)
 
+    def test_apt_install_missing(self):
+        """Test that fetching missing tools installs their packages with apt"""
+        bindir = os.path.join(self._indir, 'bin')
+        os.mkdir(bindir)
+        sudo = os.path.join(bindir, 'sudo')
+        tools.write_file(sudo, FAKE_SUDO.format(python=sys.executable),
+                         binary=False)
+        os.chmod(sudo, 0o755)
+
+        # Make sure that only the fake tools can be found
+        btools = [Bintool.create(name) for name in ['lz4', 'lzop']]
+        with unittest.mock.patch.dict('os.environ', {'PATH': bindir}), \
+                unittest.mock.patch.object(tools, 'tool_search_paths', []), \
+                unittest.mock.patch.object(Bintool, 'apt_updated', False):
+            for btool in btools:
+                self.assertFalse(btool.is_present())
+            with terminal.capture() as (stdout, _):
+                self.assertTrue(Bintool.fetch_tools(bintool.FETCH_ANY,
+                                                    ['lz4', 'lzop']))
+            for btool in btools:
+                self.assertTrue(btool.is_present())
+        self.assertIn('- sudo apt-get install -y lzop', stdout.getvalue())
+
+        # The package list should be updated only once
+        self.assertEqual(
+            ['apt-get update -y', 'apt-get install -y lz4',
+             'apt-get install -y lzop'],
+            tools.read_file(os.path.join(bindir, 'log'),
+                            binary=False).splitlines())
+
     def test_no_fetch(self):
         """Test fetching when there is no method"""
         btest = Bintool.create('_testing')

{binary_manager-0.2.0 → binary_manager-0.2.1}/binman/blob_test.py

@@ -900,7 +900,9 @@ class TestBlobFiles(unittest.TestCase):
             tools.write_file(fname, data)
         server = http.server.ThreadingHTTPServer(
             ('127.0.0.1', 0), functools.partial(QuietHandler, directory=root))
-        thread = threading.Thread(target=server.serve_forever, daemon=True)
+        # Poll often so that shutdown() returns quickly
+        thread = threading.Thread(target=server.serve_forever, args=(0.01,),
+                                  daemon=True)
         thread.start()
         self.addCleanup(server.server_close)
         self.addCleanup(server.shutdown)

{binary_manager-0.2.0 → binary_manager-0.2.1}/binman/etype/fit.py

@@ -25,6 +25,20 @@ OPERATIONS = {
     'split-elf': OP_SPLIT_ELF,
     }
 
+class GrowingFdtSw(libfdt.FdtSw):
+    """FdtSw which doubles the size of its buffer when it runs out of space
+
+    FdtSw grows its buffer by INC_SIZE bytes each time, copying the contents
+    to a new buffer. When the FIT holds large images this takes time which is
+    proportional to the square of the FIT size. Grow the buffer geometrically
+    instead.
+    """
+    INC_SIZE = 65536
+
+    def resize(self, size):
+        super().resize(size)
+        self.INC_SIZE = size
+
 # pylint: disable=invalid-name
 class Entry_fit(Entry_section):
 
@@ -865,11 +879,11 @@ class Entry_fit(Entry_section):
                                 pass
                             elif pname == 'fit,compatible':
                                 fdt_phase = fdt_util.GetString(node, pname)
-                                data = tools.read_file(fname)
-                                fdt = Fdt.FromData(data)
-                                fdt.Scan()
-                                prop = fdt.GetRoot().props['compatible']
-                                fsw.property('compatible', prop.bytes)
+                                # Read just this property, since scanning
+                                # the whole devicetree is slow
+                                fdt_obj = libfdt.Fdt(tools.read_file(fname))
+                                compat = fdt_obj.getprop(0, 'compatible')
+                                fsw.property('compatible', bytes(compat))
                             elif pname == 'fit,fdt-phase':
                                 fdt_phase = fdt_util.GetString(node, pname)
                             elif pname.startswith('fit,'):
@@ -1038,8 +1052,7 @@ class Entry_fit(Entry_section):
 
         # Build a new tree with all nodes and properties starting from the
         # entry node
-        fsw = libfdt.FdtSw()
-        fsw.INC_SIZE = 65536
+        fsw = GrowingFdtSw()
         fsw.finish_reservemap()
         to_remove = []
         loadables = []
@@ -1079,24 +1092,68 @@ class Entry_fit(Entry_section):
         if self.mkimage in self.missing_bintools:
             return
 
-        fdt = Fdt.FromData(self.GetData())
-        fdt.Scan()
+        # Look up just the properties needed, since scanning the whole FIT is
+        # slow when it holds large images
+        fdt_obj = libfdt.Fdt(self.GetData())
+
+        def _get_int(node_off, name):
+            """Read an integer property from a node in the FIT
+
+            Args:
+                node_off (int): Offset of the node
+                name (str): Name of the property
+
+            Returns:
+                int: Value of the property, or None if it is not present
+            """
+            # This returns an error code instead of a Property if the
+            # property is not present
+            prop = fdt_obj.getprop(node_off, name, libfdt.QUIET_NOTFOUND)
+            if prop == -libfdt.NOTFOUND:
+                return None
+            return prop.as_uint32()
+
+        def _find_prop(node_off, name):
+            """Find a property in a node in the FIT
+
+            getprop() provides the value of a property but not its offset,
+            which is needed to work out where the value is in the FIT. So
+            look through the node's properties for the one with this name.
+
+            Args:
+                node_off (int): Offset of the node
+                name (str): Name of the property
+
+            Returns:
+                tuple:
+                    int: Offset of the property, or None if not present
+                    int: Length of its value, or None if not present
+            """
+            poffset = fdt_obj.first_property_offset(node_off,
+                                                    libfdt.QUIET_NOTFOUND)
+            while poffset >= 0:
+                prop = fdt_obj.get_property_by_offset(poffset)
+                if prop.name == name:
+                    return poffset, len(prop)
+                poffset = fdt_obj.next_property_offset(poffset,
+                                                       libfdt.QUIET_NOTFOUND)
+            return None, None
 
         for image_name, entry in self._entries.items():
             path = f"/images/{image_name}"
-            node = fdt.GetNode(path)
+            node_off = fdt_obj.path_offset(path)
 
-            data_prop = node.props.get("data")
-            data_pos = fdt_util.GetInt(node, "data-position")
-            data_offset = fdt_util.GetInt(node, "data-offset")
-            data_size = fdt_util.GetInt(node, "data-size")
+            data_poffset, data_len = _find_prop(node_off, 'data')
+            data_pos = _get_int(node_off, "data-position")
+            data_offset = _get_int(node_off, "data-offset")
+            data_size = _get_int(node_off, "data-size")
 
             # Contents are inside the FIT
-            if data_prop is not None:
-                # GetOffset() returns offset of a fdt_property struct,
-                # which has 3 fdt32_t members before the actual data.
-                offset = data_prop.GetOffset() + 12
-                size = len(data_prop.bytes)
+            if data_poffset is not None:
+                # The property offset is that of a fdt_property struct, which
+                # has 3 fdt32_t members before the actual data
+                offset = fdt_obj.off_dt_struct() + data_poffset + 12
+                size = data_len
 
             # External offset from the base of the FIT
             elif data_pos is not None:
@@ -1105,7 +1162,7 @@ class Entry_fit(Entry_section):
 
             # External offset from the end of the FIT, not used in binman
             elif data_offset is not None: # pragma: no cover
-                offset = fdt.GetFdtObj().totalsize() + data_offset
+                offset = fdt_obj.totalsize() + data_offset
                 size = data_size
 
             # This should never happen

{binary_manager-0.2.0 → binary_manager-0.2.1}/binman/etype/ti_board_config.py

@@ -27,6 +27,11 @@ BOARDCFG_PM = 0xE
 BOARDCFG_RM = 0xC
 BOARDCFG_NUM_ELEMS = 4
 
+# Board-config binaries already compiled, keyed by the contents of the config
+# and schema files. An image often includes the same board config several
+# times, and linting and validating the YAML is slow
+compiled_cache = {}
+
 class Entry_ti_board_config(Entry_section):
     """An entry containing a TI schema validated board config binary
 
@@ -245,13 +250,17 @@ class Entry_ti_board_config(Entry_section):
             return data
 
         else:
-            with open(self._config_file, 'r') as f:
-                self.file_yaml = yaml.safe_load(f)
-            with open(self._schema_file, 'r') as sch:
-                self.schema_yaml = yaml.safe_load(sch)
+            config_text = tools.read_file(self._config_file, binary=False)
+            schema_text = tools.read_file(self._schema_file, binary=False)
+            key = (config_text, schema_text)
+            data = compiled_cache.get(key)
+            if data is not None:
+                return bytearray(data)
+            self.file_yaml = yaml.safe_load(config_text)
+            self.schema_yaml = yaml.safe_load(schema_text)
 
             yaml_config = config.YamlLintConfig("extends: default")
-            for p in yamllint.linter.run(open(self._config_file, "r"), yaml_config):
+            for p in yamllint.linter.run(config_text, yaml_config):
                 self.Raise(f"Yamllint error: Line {p.line} in {self._config_file}: {p.rule}")
             try:
                 validate(self.file_yaml, self.schema_yaml)
@@ -259,6 +268,7 @@ class Entry_ti_board_config(Entry_section):
                 self.Raise(f"Schema validation error: {e}")
 
             data = self._generate_binaries()
+            compiled_cache[key] = bytes(data)
             return data
 
     def SetImagePos(self, image_pos):

{binary_manager-0.2.0 → binary_manager-0.2.1}/binman/ftest.py

@@ -161,6 +161,10 @@ class TestFunctional(unittest.TestCase):
     In some cases a 'real' file must be used - these are also supplied in
     the test/ diurectory.
     """
+    # Private and public key used by the signing tests, as a tuple of bytes.
+    # Generating an RSA4096 key takes a while, so this is done only once
+    _sign_keys = None
+
     @classmethod
     def setUpClass(cls):
         global entry
@@ -2078,8 +2082,9 @@ class TestFunctional(unittest.TestCase):
                                 tmpdir, argv[argv.index('--directory') + 1]))
                         server = http.server.ThreadingHTTPServer(
                             ('localhost', 0), handler)
+                        # Poll often so that shutdown() returns quickly
                         threading.Thread(target=server.serve_forever,
-                                         daemon=True).start()
+                                         args=(0.01,), daemon=True).start()
                         ports[argv[-1]] = server.server_address[1]
                     else:
                         result = command.run_one(
@@ -4941,6 +4946,22 @@ FLASH@0xffff0000 0x10000 {
             self.assertEqual(len(expected), size)
             self.assertEqual(expected, data[image_pos:image_pos+size])
 
+    def testFitLarge(self):
+        """Test a FIT which is larger than its initial buffer"""
+        data, _, _, out_dtb_fname = self._DoReadFileDtb('fit/large.dts',
+                                                        update_dtb=True)
+        dtb = fdt.Fdt(out_dtb_fname)
+        dtb.Scan()
+        props = self._GetPropTree(dtb, ['image-pos', 'size'])
+        for node, expected in [
+            ('fit/images/kernel', tools.get_bytes(0xaa, 0x30000)),
+            ('fit/images/ramdisk', tools.get_bytes(0xbb, 0x50000)),
+        ]:
+            image_pos = props[f'{node}:image-pos']
+            size = props[f'{node}:size']
+            self.assertEqual(len(expected), size)
+            self.assertEqual(expected, data[image_pos:image_pos + size])
+
     def testFitExternal(self):
         """Test an image with an FIT with external images"""
         data = self._DoReadFile('fit/external.dts')
@@ -7845,9 +7866,15 @@ fdt         fdtmap                Extract the devicetree blob from the fdtmap
         fit = tools.get_output_filename('fit.fit')
         key_dir = tools.get_output_dir()
 
-        tools.run('openssl', 'req', '-batch' , '-newkey', 'rsa:4096',
-                  '-sha256', '-new',  '-nodes',  '-x509', '-keyout',
-                  private_key, '-out', public_key)
+        if TestFunctional._sign_keys:
+            tools.write_file(private_key, TestFunctional._sign_keys[0])
+            tools.write_file(public_key, TestFunctional._sign_keys[1])
+        else:
+            tools.run('openssl', 'req', '-batch' , '-newkey', 'rsa:4096',
+                      '-sha256', '-new',  '-nodes',  '-x509', '-keyout',
+                      private_key, '-out', public_key)
+            TestFunctional._sign_keys = (tools.read_file(private_key),
+                                         tools.read_file(public_key))
         tools.run('fdt_add_pubkey', '-a', 'sha256,rsa4096', '-k', key_dir,
                   '-n', 'test_key', '-r', 'conf', dtb)
 

{binary_manager-0.2.0 → binary_manager-0.2.1}/binman/main.py

@@ -24,7 +24,6 @@ sys.path.insert(2, os.path.dirname(our_path))
 from binman import bintool
 from binman import cmdline
 from binman import control
-from u_boot_pylib import test_util
 
 def RunTests(debug, verbosity, processes, test_preserve_dirs, args, toolpath):
     """Run the functional tests and any embedded doctests
@@ -55,6 +54,7 @@ def RunTests(debug, verbosity, processes, test_preserve_dirs, args, toolpath):
     from binman import blob
     from binman import blobstore
     import doctest
+    from u_boot_pylib import test_util
 
     # Keep the tests away from the user's blob configuration, so that it does
     # not affect them. Tests which need a configuration provide their own
@@ -96,6 +96,8 @@ def RunTestCoverage(toolpath, build_dir, args):
         for path in toolpath:
             extra_args += ' --toolpath %s' % path
 
+    from u_boot_pylib import test_util
+
     test_util.run_test_coverage(os.path.join(our_path, 'main.py'), None,
             ['*test*', '*main.py', '*/dtoc/*', '*/u_boot_pylib/*'],
             build_dir, all_set, extra_args or None, args=args,
@@ -119,6 +121,9 @@ def RunBinman(args):
         args.toolpath = ['./tools', 'build-sandbox/tools']
 
     if args.cmd == 'test':
+        # Only the tests need this, and it takes a while to import
+        from u_boot_pylib import test_util
+
         if args.test_coverage:
             RunTestCoverage(args.toolpath, args.build_dir, args.tests)
         else:

binary_manager-0.2.1/binman/test/fit/large.dts

@@ -0,0 +1,51 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+/dts-v1/;
+
+/ {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	binman {
+		fit {
+			description = "test-desc";
+			#address-cells = <1>;
+
+			images {
+				kernel {
+					description = "Large kernel";
+					type = "kernel";
+					arch = "arm64";
+					os = "linux";
+					compression = "none";
+					load = <00000000>;
+					entry = <00000000>;
+					fill {
+						size = <0x30000>;
+						fill-byte = [aa];
+					};
+				};
+				ramdisk {
+					description = "Large ramdisk";
+					type = "ramdisk";
+					arch = "arm64";
+					os = "linux";
+					compression = "none";
+					fill {
+						size = <0x50000>;
+						fill-byte = [bb];
+					};
+				};
+			};
+
+			configurations {
+				default = "conf-1";
+				conf-1 {
+					description = "Boot the kernel with a ramdisk";
+					kernel = "kernel";
+					ramdisk = "ramdisk";
+				};
+			};
+		};
+	};
+};

{binary_manager-0.2.0 → binary_manager-0.2.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "binary-manager"
-version = "0.2.0"
+version = "0.2.1"
 authors = [
   { name="Simon Glass", email="sjg@chromium.org" },
 ]
@@ -30,6 +30,7 @@ dependencies = [
     "PyYAML",
     "yamllint",
     "u-boot-pylib >= 0.1.0",
+    "dtoc >= 0.1.0",
 ]
 
 [project.optional-dependencies]
@@ -46,8 +47,7 @@ Repository = "https://github.com/deinde-dev/binman"
 binman = "binman.main:start_binman"
 
 [tool.setuptools.packages.find]
-# binman.test and its subdirs hold the test data, found as namespace packages.
-# binman._vendor holds the vendored dtoc library (see binman/__init__.py)
+# binman.test and its subdirs hold the test data, found as namespace packages
 include = ["binman*"]
 
 [tool.setuptools.package-data]

binary_manager-0.2.0/binman/__init__.py

@@ -1,17 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0+
-
-"""Binman firmware packager
-
-binman bundles a copy of dtoc's devicetree modules in _vendor/. Make these
-importable by their usual name, ahead of any other copies, e.g. in a U-Boot
-tree. They are kept out of the top level of site-packages, where they would
-clash with U-Boot's tools and with the separate dtoc package.
-"""
-
-import os
-import sys
-
-_VENDOR_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)),
-                           '_vendor')
-if _VENDOR_DIR not in sys.path:
-    sys.path.insert(0, _VENDOR_DIR)