BuildStream 2.2.0.dev4__tar.gz → 2.2.1.dev0__tar.gz

{buildstream-2.2.0.dev4 → buildstream-2.2.1.dev0}/NEWS

@@ -1,3 +1,16 @@
+=================
+buildstream 2.2.1
+=================
+
+  o Fix parsing of keepalive time (#1916)
+
+  o Fix alias mapping when using source mirror plugins (#1918)
+
+  o Fix typing of SourceFetcher.fetch() and Source.translate_url():
+    the `alias_override` parameter should be considered opaque.
+
+  o Update documentation regarding the buildbox merge (#1919)
+
 =================
 buildstream 2.2.0
 =================

{buildstream-2.2.0.dev4 → buildstream-2.2.1.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: BuildStream
-Version: 2.2.0.dev4
+Version: 2.2.1.dev0
 Summary: A framework for modelling build pipelines in YAML
 Home-page: https://buildstream.build
 Author: The Apache Software Foundation

{buildstream-2.2.0.dev4 → buildstream-2.2.1.dev0}/doc/source/main_install.rst

@@ -117,26 +117,14 @@ e.g., ``~/.local/bin``.
 Build from source
 ~~~~~~~~~~~~~~~~~
 
-Each of the 4 buildbox components can be installed separately from their
-respective git repositiories, and each respository has individual install
-instructions. We recommend installing the latest release tag of each
-component.
-
-| **Buildbox-common:** See the installation section in:
-| https://gitlab.com/BuildGrid/buildbox/buildbox-common/-/blob/master/README.rst
-| (Be sure to install from the latest stable release tag.)
-
-| **Buildbox-casd:** See the installation section in:
-| https://gitlab.com/BuildGrid/buildbox/buildbox-casd/-/blob/master/README.rst \
-| (Be sure to install from the latest stable release tag.)
-
-| **Buildbox-fuse:** See
-| https://gitlab.com/BuildGrid/buildbox/buildbox-fuse/-/blob/master/INSTALL.rst
-| (Be sure to install from the latest stable release tag.)
-
-| **Buildbox-run-bublewrap:** See the installation section in:
-| https://gitlab.com/BuildGrid/buildbox/buildbox-run-bubblewrap/-/blob/master/README.rst
-| (Be sure to install from the latest stable release tag.)
+Buildbox components can be installed from their git repository.
+We recommend installing from the latest release tag.
+
+See the <"Installation" section here <https://gitlab.com/BuildGrid/buildbox/buildbox/-/blob/master/README.rst#installation>`_.
+
+Buildbox contains many components, BuildStream needs only ``buildbox-casd``,
+``buildbox-fuse`` and ``buildbox-run-bubblewrap``, which can be selected by
+passing options ``-DTOOLS=OFF -DCASD=ON -DFUSE=ON -DRUN_BUBBLEWRAP=ON`` to CMake.
 
 Finally, configure buildbox-run-bubblewrap as the default buildbox-run
 implementation::

{buildstream-2.2.0.dev4 → buildstream-2.2.1.dev0}/src/BuildStream.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: BuildStream
-Version: 2.2.0.dev4
+Version: 2.2.1.dev0
 Summary: A framework for modelling build pipelines in YAML
 Home-page: https://buildstream.build
 Author: The Apache Software Foundation

{buildstream-2.2.0.dev4 → buildstream-2.2.1.dev0}/src/buildstream/_cas/casremote.py

@@ -13,7 +13,6 @@
 #
 
 import grpc
-from google.protobuf.duration_pb2 import Duration  # pylint: disable=no-name-in-module
 
 from .._protos.google.rpc import code_pb2
 from .._protos.build.buildgrid import local_cas_pb2
@@ -71,7 +70,7 @@ class CASRemote(BaseRemote):
         if self.spec.client_cert:
             cas_endpoint.client_cert = self.spec.client_cert
         if self.spec.keepalive_time is not None:
-            cas_endpoint.keepalive_time = Duration().FromSeconds(self.spec.keepalive_time)
+            cas_endpoint.keepalive_time.FromSeconds(self.spec.keepalive_time)
         try:
             response = local_cas.GetInstanceNameForRemotes(request)
         except grpc.RpcError as e:

{buildstream-2.2.0.dev4 → buildstream-2.2.1.dev0}/src/buildstream/_project.py

@@ -40,7 +40,7 @@ from ._includes import Includes
 from ._workspaces import WORKSPACE_PROJECT_FILE
 from ._remotespec import RemoteSpec
 from .sourcemirror import SourceMirror
-from .source import SourceError
+from .source import AliasSubstitution, SourceError
 
 
 if TYPE_CHECKING:
@@ -453,7 +453,7 @@ class Project:
     #
     def get_alias_uris(
         self, alias: str, *, first_pass: bool = False, tracking: bool = False
-    ) -> Sequence[Union[SourceMirror, str, None]]:
+    ) -> Sequence[Optional[AliasSubstitution]]:
 
         if first_pass:
             config = self.first_pass_config
@@ -489,7 +489,7 @@ class Project:
         if policy in (_SourceUriPolicy.ALL, _SourceUriPolicy.ALIASES):
             uri_list.append(config._aliases.get_str(alias))
 
-        return uri_list
+        return [AliasSubstitution(alias, mirror) for mirror in uri_list]
 
     # load_elements()
     #

{buildstream-2.2.0.dev4 → buildstream-2.2.1.dev0}/src/buildstream/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2024-04-18T09:05:28+0100",
+ "date": "2024-05-17T08:33:28+0100",
  "dirty": false,
  "error": null,
- "full-revisionid": "8080e8552bdf5ea078ccdc4f7d9f66b641f276ac",
- "version": "2.2.0.dev4"
+ "full-revisionid": "db0a4f3e0dc3f3a499b27142b9d16ca0dc95560f",
+ "version": "2.2.1.dev0"
 }
 '''  # END VERSION_JSON
 

{buildstream-2.2.0.dev4 → buildstream-2.2.1.dev0}/src/buildstream/source.py

@@ -218,6 +218,7 @@ Class Reference
 import os
 from contextlib import contextmanager
 from typing import Iterable, Iterator, Optional, Tuple, Dict, Any, Set, TYPE_CHECKING, Union
+from dataclasses import dataclass
 
 from . import _yaml, utils
 from .node import MappingNode
@@ -259,6 +260,12 @@ class SourceError(BstError):
         super().__init__(message, detail=detail, domain=ErrorDomain.SOURCE, reason=reason, temporary=temporary)
 
 
+@dataclass
+class AliasSubstitution:
+    _effective_alias: str
+    _mirror: Union[SourceMirror, str]
+
+
 class SourceFetcher:
     """SourceFetcher()
 
@@ -280,7 +287,7 @@ class SourceFetcher:
     #############################################################
     #                      Abstract Methods                     #
     #############################################################
-    def fetch(self, alias_override: Optional[Union[str, SourceMirror]] = None, **kwargs) -> None:
+    def fetch(self, alias_override: Optional[AliasSubstitution] = None, **kwargs) -> None:
         """Fetch remote sources and mirror them locally, ensuring at least
         that the specific reference is cached locally.
 
@@ -387,7 +394,7 @@ class Source(Plugin):
         meta: MetaSource,
         variables: Variables,
         *,
-        alias_override: Optional[Tuple[str, Union[str, SourceMirror]]] = None,
+        alias_override: Optional[Tuple[str, AliasSubstitution]] = None,
         unique_id: Optional[int] = None,
     ):
         # Set element_name member before parent init, as needed for debug messaging
@@ -699,7 +706,7 @@ class Source(Plugin):
         self,
         url: str,
         *,
-        alias_override: Optional[Union[str, SourceMirror]] = None,
+        alias_override: Optional[AliasSubstitution] = None,
         primary: bool = True,
         suffix: Optional[str] = None,
         extra_data: Optional[Dict[str, Any]] = None,
@@ -745,7 +752,7 @@ class Source(Plugin):
 
             if self.__alias_override is not None:
                 override_alias = self.__alias_override[0]
-                override_mirror = self.__alias_override[1]
+                override_subst = self.__alias_override[1]
 
                 # Implicit alias overrides may only be done for one
                 # specific alias, so that sources that fetch from multiple
@@ -756,16 +763,19 @@ class Source(Plugin):
                     return url
 
             elif alias_override is not None:
-                override_mirror = alias_override
+                override_subst = alias_override
 
             # The default source mirror will give prefix URLs
-            if isinstance(override_mirror, str):
-                return override_mirror + url_body
+            if isinstance(override_subst._mirror, str):
+                return override_subst._mirror + url_body
             #
             # Delegate the URL translation to the SourceMirror plugin
             #
-            return override_mirror.translate_url(
-                alias=url_alias, alias_url=project_alias_url, source_url=url_body, extra_data=extra_data
+            return override_subst._mirror.translate_url(
+                alias=override_subst._effective_alias,
+                alias_url=project_alias_url,
+                source_url=url_body,
+                extra_data=extra_data,
             )
         else:
             return project.translate_url(url, source=self, first_pass=self.__first_pass)

{buildstream-2.2.0.dev4 → buildstream-2.2.1.dev0}/tests/artifactcache/pull.py

@@ -47,8 +47,17 @@ def tree_maker(cas, tree, directory):
         tree_maker(cas, tree, child_directory)
 
 
+@pytest.mark.parametrize(
+    "connection_config",
+    (
+        None,
+        {
+            "keepalive-time": 60,
+        },
+    ),
+)
 @pytest.mark.datafiles(DATA_DIR)
-def test_pull(cli, tmpdir, datafiles):
+def test_pull(cli, tmpdir, datafiles, connection_config):
     project_dir = str(datafiles)
 
     # Set up an artifact cache.
@@ -56,18 +65,17 @@ def test_pull(cli, tmpdir, datafiles):
         # Configure artifact share
         cache_dir = os.path.join(str(tmpdir), "cache")
         user_config_file = str(tmpdir.join("buildstream.conf"))
+        server = {
+            "url": share.repo,
+            "push": True,
+        }
         user_config = {
             "scheduler": {"pushers": 1},
-            "artifacts": {
-                "servers": [
-                    {
-                        "url": share.repo,
-                        "push": True,
-                    }
-                ]
-            },
+            "artifacts": {"servers": [server]},
             "cachedir": cache_dir,
         }
+        if connection_config is not None:
+            server["connection-config"] = connection_config
 
         # Write down the user configuration file
         _yaml.roundtrip_dump(user_config, file=user_config_file)

{buildstream-2.2.0.dev4 → buildstream-2.2.1.dev0}/tests/frontend/mirror.py

@@ -889,16 +889,46 @@ def test_source_mirror_plugin(cli, tmpdir):
         assert bar_str in contents
 
 
+# Test subproject alias mapping. As there are quite a few corner cases and interactions with other
+# features that need to be tested, this test relies heavily on parametrization. Here is what each
+# parameter means:
+# * subproject_mirrors: whether the subproject defines a (failing) mirror
+# * unaliased_sources: whether the subproject has unaliased sources
+# * disallow_subproject_uris: define disallow-subproject-uris in the parent project
+# * fetch_source: whether to fetch from aliases or mirrors
+# * alias_override: which aliases to override ("global" means use "map-aliases")
+# * alias_mapping: how to map aliases to the overrides
+# * source_mirror: whether to use a custom source mirror plugin
+
+
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.usefixtures("datafiles")
 @pytest.mark.parametrize("subproject_mirrors", [True, False])
 @pytest.mark.parametrize("unaliased_sources", [True, False])
 @pytest.mark.parametrize("disallow_subproject_uris", [True, False])
 @pytest.mark.parametrize("fetch_source", ["aliases", "mirrors"])
-@pytest.mark.parametrize("alias_override", [["foo"], ["foo", "bar"], "global", "invalid"])
+@pytest.mark.parametrize("alias_override", [["foo"], ["foo", "bar"], "global"])
+@pytest.mark.parametrize("alias_mapping", ["identity", "project-prefix", "invalid"])
+@pytest.mark.parametrize("source_mirror", [True, False])
 def test_mirror_subproject_aliases(
-    cli, tmpdir, subproject_mirrors, unaliased_sources, disallow_subproject_uris, fetch_source, alias_override
+    cli,
+    tmpdir,
+    subproject_mirrors,
+    unaliased_sources,
+    disallow_subproject_uris,
+    fetch_source,
+    alias_override,
+    alias_mapping,
+    source_mirror,
 ):
+    if alias_override == "global":
+        if alias_mapping == "invalid":
+            # we can't have an invalid mapping using a predefined option
+            pytest.skip()
+        elif alias_mapping == "project-prefix":
+            # project-prefix alias mapping not yet implemented
+            pytest.xfail()
+
     output_file = os.path.join(str(tmpdir), "output.txt")
     project_dir = tmpdir
 
@@ -941,7 +971,69 @@ def test_mirror_subproject_aliases(
     # copy the source plugin to the subproject
     shutil.copytree(project_dir / "sources", subproject_dir / "sources")
 
-    project = generate_project(MirrorConfig.SUCCESS_MIRRORS, fetch_source == "aliases")
+    if alias_mapping == "identity":
+
+        def map_alias(x):
+            return x
+
+    elif alias_mapping == "project-prefix":
+
+        def map_alias(x):
+            return subproject["name"] + "/" + x
+
+    else:
+
+        def map_alias(x):
+            return "invalid-" + x
+
+    if alias_mapping != "invalid":
+        map_alias_valid = map_alias
+    else:
+
+        def map_alias_valid(x):
+            return x
+
+    project = {
+        "name": "test",
+        "min-version": "2.0",
+        "element-path": "elements",
+        "aliases": {
+            map_alias_valid("foo"): "FOO/",
+            map_alias_valid("bar"): "RAB/" if fetch_source == "aliases" else "BAR/",
+        },
+        "plugins": [{"origin": "local", "path": "sources", "sources": ["fetch_source"]}],
+        # Copy of SUCCESS_MIRROR_LIST from above
+        "mirrors": [
+            {
+                "name": "middle-earth",
+                "aliases": {
+                    map_alias_valid("foo"): ["OOF/"],
+                    map_alias_valid("bar"): ["RAB/"],
+                },
+            },
+            {
+                "name": "arrakis",
+                "aliases": {
+                    map_alias_valid("foo"): ["FOO/"],
+                    map_alias_valid("bar"): ["RBA/"],
+                },
+            },
+            {
+                "name": "oz",
+                "aliases": {
+                    map_alias_valid("foo"): ["ooF/"],
+                    map_alias_valid("bar"): ["raB/"],
+                },
+            },
+        ],
+    }
+    if source_mirror:
+        project["plugins"].append({"origin": "local", "path": "sourcemirrors", "source-mirrors": ["mirror"]})
+
+        mirrors = []
+        for mirror in project["mirrors"]:
+            mirrors.append({"name": mirror["name"], "kind": "mirror", "config": {"aliases": mirror["aliases"]}})
+        project["mirrors"] = mirrors
 
     if disallow_subproject_uris:
         project["junctions"] = {"disallow-subproject-uris": "true"}
@@ -960,16 +1052,9 @@ def test_mirror_subproject_aliases(
     }
 
     if alias_override == "global":
-        junction["config"] = {"map-aliases": "identity"}
-    elif alias_override == "invalid":
-        junction["config"] = {
-            "aliases": {
-                "foo": "invalid-foo",
-                "bar": "invalid-bar",
-            }
-        }
+        junction["config"] = {"map-aliases": alias_mapping}
     else:
-        junction["config"] = {"aliases": {alias: alias for alias in alias_override}}
+        junction["config"] = {"aliases": {alias: map_alias(alias) for alias in alias_override}}
 
     _yaml.roundtrip_dump(junction, str(element_dir / junction_name))
 
@@ -977,7 +1062,7 @@ def test_mirror_subproject_aliases(
     cli.configure(userconfig)
 
     result = cli.run(project=project_dir, args=["source", "fetch", "{}:{}".format(junction_name, element_name)])
-    if alias_override == "invalid":
+    if alias_mapping == "invalid":
         # Mapped alias does not exist in the parent project
         result.assert_main_error(ErrorDomain.SOURCE, "invalid-source-alias")
     elif disallow_subproject_uris and unaliased_sources: