BuildStream 2.2.0.dev2__tar.gz → 2.2.0.dev3__tar.gz

{BuildStream-2.2.0.dev2 → buildstream-2.2.0.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: BuildStream
-Version: 2.2.0.dev2
+Version: 2.2.0.dev3
 Summary: A framework for modelling build pipelines in YAML
 Home-page: https://buildstream.build
 Author: The Apache Software Foundation

{BuildStream-2.2.0.dev2 → buildstream-2.2.0.dev3}/doc/source/format_project.rst

@@ -204,6 +204,9 @@ URLs which are to be used in the individual ``.bst`` files.
      foo: git://git.foo.org/
      bar: http://bar.com/downloads/
 
+If you want this project's alias definitions to also be used for subprojects,
+see :ref:`Mapping source aliases of subprojects <project_junctions_source_aliases>`.
+
 
 Sandbox options
 ~~~~~~~~~~~~~~~
@@ -319,6 +322,9 @@ The mirrors can be overridden on a per project basis using
 be used first in the :ref:`user configuration <config_default_mirror>`, or using
 the  :ref:`--default-mirror <invoking_bst>` command-line argument.
 
+If you want this project's mirrors to also be used for subprojects,
+see :ref:`Mapping source aliases of subprojects <project_junctions_source_aliases>`.
+
 
 .. _project_plugins:
 
@@ -989,6 +995,35 @@ subproject.
    will not accrue runtime dependencies on elements in your *internal* subproject.
 
 
+.. _project_junctions_source_aliases:
+
+Mapping source aliases of subprojects
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+:mod:`junction <elements.junction>` elements allow source aliases of subprojects
+to be mapped to aliases of the parent project. This makes it possible to control
+the translation of aliases to URLs including mirror configuration across multiple
+project levels.
+
+To ensure that there are mappings for all aliases of all subprojects, you can set the
+``disallow-subproject-uris`` flag in the ``junctions`` group here in ``project.conf``.
+
+top-level
+
+.. code:: yaml
+
+   junctions:
+     disallow-subproject-uris: True
+
+This will raise an error if an alias without a mapping is encountered. This flag
+is applied recursively across all junctions.
+
+It also configures ``unaliased-url`` as a fatal warning in all subprojects to
+ensure that the current project is in full control over all source URLs.
+As the fatal warning configuration contributes to the cache key, this flag will
+affect the cache key of subprojects that haven't already configured
+``unaliased-url`` as a fatal warning.
+
+
 .. _project_defaults:
 
 Element default configuration

{BuildStream-2.2.0.dev2 → buildstream-2.2.0.dev3}/src/BuildStream.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: BuildStream
-Version: 2.2.0.dev2
+Version: 2.2.0.dev3
 Summary: A framework for modelling build pipelines in YAML
 Home-page: https://buildstream.build
 Author: The Apache Software Foundation

{BuildStream-2.2.0.dev2 → buildstream-2.2.0.dev3}/src/BuildStream.egg-info/SOURCES.txt

@@ -2252,6 +2252,7 @@ tests/sources/tar/fetch/target-lz.bst
 tests/sources/tar/fetch/target.bst
 tests/sources/tar/fetch/content/a/c
 tests/sources/tar/fetch/content/a/b/d
+tests/sources/tar/fetch/sourcemirrors/bearermirror.py
 tests/sources/tar/no-basedir/target.bst
 tests/sources/tar/no-basedir/content/a/c
 tests/sources/tar/no-basedir/content/a/b/d
@@ -2273,6 +2274,7 @@ tests/sources/variables/unresolveable-target.bst
 tests/sources/variables/folder/file.txt
 tests/testutils/__init__.py
 tests/testutils/artifactshare.py
+tests/testutils/bearer_http_server.py
 tests/testutils/constants.py
 tests/testutils/context.py
 tests/testutils/element_generators.py

{BuildStream-2.2.0.dev2 → buildstream-2.2.0.dev3}/src/buildstream/_project.py

@@ -40,10 +40,12 @@ from ._includes import Includes
 from ._workspaces import WORKSPACE_PROJECT_FILE
 from ._remotespec import RemoteSpec
 from .sourcemirror import SourceMirror
+from .source import SourceError
 
 
 if TYPE_CHECKING:
     from ._context import Context
+    from .plugins.elements.junction import JunctionElement
 
 
 # Project Configuration file
@@ -82,7 +84,7 @@ class Project:
         directory: Optional[str],
         context: "Context",
         *,
-        junction: Optional[object] = None,
+        junction: Optional["JunctionElement"] = None,
         cli_options: Optional[Dict[str, str]] = None,
         default_mirror: Optional[str] = None,
         parent_loader: Optional[Loader] = None,
@@ -98,11 +100,12 @@ class Project:
 
         self.load_context: LoadContext  # The LoadContext
         self.loader: Optional[Loader] = None  # The loader associated to this project
-        self.junction: Optional[object] = junction  # The junction Element object, if this is a subproject
+        self.junction: Optional["JunctionElement"] = junction  # The junction Element object, if this is a subproject
 
         self.ref_storage: Optional[ProjectRefStorage] = None  # Where to store source refs
         self.refs: Optional[ProjectRefs] = None
         self.junction_refs: Optional[ProjectRefs] = None
+        self.disallow_subproject_uris: bool = False
 
         self.config: ProjectConfig = ProjectConfig()
         self.first_pass_config: ProjectConfig = ProjectConfig()
@@ -234,12 +237,28 @@ class Project:
     # This method is provided for :class:`.Source` objects to resolve
     # fully qualified urls based on the shorthand which is allowed
     # to be specified in the YAML
-    def translate_url(self, url, *, first_pass=False):
+    def translate_url(self, url, *, source, first_pass=False):
 
         if url and utils._ALIAS_SEPARATOR in url:
             url_alias, url_body = url.split(utils._ALIAS_SEPARATOR, 1)
             alias_url = self.get_alias_url(url_alias, first_pass=first_pass)
             if alias_url:
+                if self.junction:
+                    parent_project = self.junction._get_project()
+                    parent_alias = self.junction.get_parent_alias(url_alias)
+                    if parent_alias:
+                        # Delegate translation to parent project
+                        return parent_project.translate_url(
+                            parent_alias + utils._ALIAS_SEPARATOR + url_body, source=source, first_pass=first_pass
+                        )
+                    elif parent_project.disallow_subproject_uris:
+                        raise SourceError(
+                            "{}: Parent project did not provide a mapping for alias '{}' and disallowed usage of unmapped aliases".format(
+                                source, url_alias
+                            ),
+                            reason="missing-alias-mapping",
+                        )
+
                 url = alias_url + url_body
 
         return url
@@ -391,12 +410,33 @@ class Project:
     # Returns:
     #    bool: Whether the alias is declared in the scope of this project
     #
-    def alias_exists(self, alias, *, first_pass=False):
+    def alias_exists(self, alias, *, source, first_pass=False):
         if first_pass:
             config = self.first_pass_config
         else:
             config = self.config
 
+        if self.junction:
+            parent_project = self.junction._get_project()
+            parent_alias = self.junction.get_parent_alias(alias)
+            if parent_alias:
+                if parent_project.alias_exists(parent_alias, source=source, first_pass=first_pass):
+                    return True
+                else:
+                    raise SourceError(
+                        "{}: Mapped alias '{}' for subproject alias '{}' is invalid in the parent project".format(
+                            self.junction, parent_alias, alias
+                        ),
+                        reason="invalid-source-alias",
+                    )
+            elif parent_project.disallow_subproject_uris:
+                raise SourceError(
+                    "{}: Parent project did not provide a mapping for alias '{}' and disallowed usage of unmapped aliases".format(
+                        source, alias
+                    ),
+                    reason="missing-alias-mapping",
+                )
+
         return config._aliases.get_str(alias, default=None) is not None
 
     # get_alias_uris()
@@ -423,6 +463,15 @@ class Project:
         if not alias or alias not in config._aliases:  # pylint: disable=unsupported-membership-test
             return [None]
 
+        if self.junction:
+            parent_project = self.junction._get_project()
+            parent_alias = self.junction.get_parent_alias(alias)
+            if parent_alias:
+                # Delegate translation to parent project
+                return parent_project.get_alias_uris(parent_alias, first_pass=first_pass, tracking=tracking)
+            elif parent_project.disallow_subproject_uris:
+                return [None]
+
         uri_list: List[Union[SourceMirror, str]] = []
         policy = self._context.track_source if tracking else self._context.fetch_source
 
@@ -814,7 +863,19 @@ class Project:
 
         # Junction configuration
         junctions_node = pre_config_node.get_mapping("junctions", default={})
-        junctions_node.validate_keys(["duplicates", "internal"])
+        junctions_node.validate_keys(["duplicates", "internal", "disallow-subproject-uris"])
+
+        if self.junction and self.junction._get_project().disallow_subproject_uris:
+            # If the parent project doesn't allow subproject URIs, this must
+            # be enforced for nested subprojects as well.
+            self.disallow_subproject_uris = True
+
+            # The `disallow-subproject-uris` flag also implies fatal `unaliased-url` in subprojects
+            # to ensure no subproject URIs escape the parent project's control.
+            if CoreWarnings.UNALIASED_URL not in self._fatal_warnings:
+                self._fatal_warnings.append(CoreWarnings.UNALIASED_URL)
+        else:
+            self.disallow_subproject_uris = junctions_node.get_bool("disallow-subproject-uris", default=False)
 
         # Parse duplicates
         junction_duplicates = junctions_node.get_mapping("duplicates", default={})

{BuildStream-2.2.0.dev2 → buildstream-2.2.0.dev3}/src/buildstream/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2024-04-04T11:11:47+0200",
+ "date": "2024-04-16T20:26:00+0100",
  "dirty": false,
  "error": null,
- "full-revisionid": "23147f2e467207afa68059c1201c2cf1d7ecd5a1",
- "version": "2.2.0.dev2"
+ "full-revisionid": "0bd7059b909f2b6b0f56ac41561bd8c2b0c2ec2d",
+ "version": "2.2.0.dev3"
 }
 '''  # END VERSION_JSON
 

{BuildStream-2.2.0.dev2 → buildstream-2.2.0.dev3}/src/buildstream/downloadablefilesource.py

@@ -25,6 +25,62 @@ Any derived classes must write their own stage() and get_unique_key()
 implementation.
 
 
+SourceMirror extra data "http-auth"
+--------------------------------------------
+The DownloadableFileSource, and consequently any :class:`Source <buildstream.source.Source>`
+implementations which derive from DownloadableFileSource, support the "http-auth"
+extra data returned by :class:`SourceMirror <buildstream.sourcemirror.SourceMirror>` plugins
+through :func:`Source.translate_url() <buildstream.source.Source.translate_url>`.
+
+This functionality is available **Since: 2.2**.
+
+This allows one to use :class:`SourceMirror <buildstream.sourcemirror.SourceMirror>` plugins
+to add an authorization header to the ``GET`` requests.
+
+
+**Example:**
+
+.. code:: python
+
+   class MySourceMirror(SourceMirror):
+
+        def translate_url(
+            self,
+            *,
+            alias: str,
+            alias_url: str,
+            source_url: str,
+            extra_data: Optional[Dict[str, Any]],
+        ) -> str:
+
+            #
+            # Set the "http-auth" extra data
+            #
+            if extra_data is not None:
+                extra_data["http-auth"] = "bearer"
+
+            # ...
+
+Only the "http-auth" value ``bearer`` is supported.
+
+
+**Example:**
+
+If the URL reported by :func:`SourceMirror.translate_url() <buildstream.sourcemirror.SourceMirror.translate_url>`
+is ``http://flying-ponies.com/downloads/pony.tgz``, then a corresponding entry will be expected in the
+user's ``~/.netrc``:
+
+.. code::
+
+   flying-ponies.com
+       password 1234
+
+DownloadableFileSource will add the following header to the ``GET`` request to download the file:
+
+.. code::
+
+   Authorization: Bearer 1234
+
 """
 
 
@@ -88,13 +144,21 @@ class _NetrcPasswordManager:
             return login, password
 
 
-def _download_file(opener_creator, url, etag, directory):
-    opener = opener_creator.get_url_opener()
+def _download_file(opener_creator, url, etag, directory, bearer_auth):
+    opener = opener_creator.get_url_opener(bearer_auth)
     default_name = os.path.basename(url)
     request = urllib.request.Request(url)
     request.add_header("Accept", "*/*")
     request.add_header("User-Agent", "BuildStream/2")
 
+    if opener_creator.netrc_config and bearer_auth:
+        parts = urllib.parse.urlsplit(url)
+        entry = opener_creator.netrc_config.authenticators(parts.hostname)
+        if entry:
+            _, _, password = entry
+            auth_header = "Bearer " + password
+            request.add_header("Authorization", auth_header)
+
     if etag is not None:
         request.add_header("If-None-Match", etag)
 
@@ -145,7 +209,11 @@ class DownloadableFileSource(Source):
     def configure(self, node):
         self.original_url = node.get_str("url")
         self.ref = node.get_str("ref", None)
-        self.url = self.translate_url(self.original_url)
+
+        extra_data = {}
+        self.url = self.translate_url(self.original_url, extra_data=extra_data)
+        self.bearer_auth = extra_data.get("http-auth") == "bearer"
+
         self._mirror_dir = os.path.join(self.get_mirror_directory(), utils.url_directory_name(self.original_url))
 
     def preflight(self):
@@ -230,7 +298,7 @@ class DownloadableFileSource(Source):
             url_opener_creator = _UrlOpenerCreator(self._parse_netrc())
 
             local_file, new_etag, error = self.blocking_activity(
-                _download_file, (url_opener_creator, self.url, etag, td), activity_name
+                _download_file, (url_opener_creator, self.url, etag, td, self.bearer_auth), activity_name
             )
 
             if error:
@@ -286,8 +354,8 @@ class _UrlOpenerCreator:
     def __init__(self, netrc_config):
         self.netrc_config = netrc_config
 
-    def get_url_opener(self):
-        if self.netrc_config:
+    def get_url_opener(self, bearer_auth):
+        if self.netrc_config and not bearer_auth:
             netrc_pw_mgr = _NetrcPasswordManager(self.netrc_config)
             http_auth = urllib.request.HTTPBasicAuthHandler(netrc_pw_mgr)
             ftp_handler = _NetrcFTPOpener(self.netrc_config)

{BuildStream-2.2.0.dev2 → buildstream-2.2.0.dev3}/src/buildstream/plugins/elements/junction.py

@@ -50,6 +50,14 @@ Overview
      overrides:
        subproject-junction.bst: local-junction.bst
 
+     # Optionally override aliases in subprojects, to allow using mirrors
+     # defined in the parent project.
+     aliases:
+       subproject-alias: local-alias
+
+     # A default mapping can be set (defaults to none)
+     map-aliases: identity
+
 With a junction element in place, local elements can depend on elements in
 the other BuildStream project using :ref:`element paths <format_element_names>`.
 For example, if you have a ``toolchain.bst`` junction element referring to
@@ -324,6 +332,12 @@ in the same build pipeline, please refer to the
 
 from buildstream import Element, ElementError
 from buildstream._pipeline import PipelineError
+from buildstream.types import FastEnum
+
+
+class _AliasMappingStrategy(FastEnum):
+    NONE = "none"
+    IDENTITY = "identity"
 
 
 # Element implementation for the 'junction' kind.
@@ -338,7 +352,7 @@ class JunctionElement(Element):
 
     def configure(self, node):
 
-        node.validate_keys(["path", "options", "overrides"])
+        node.validate_keys(["path", "options", "overrides", "aliases", "map-aliases"])
 
         self.path = node.get_str("path", default="")
         self.options = node.get_mapping("options", default={})
@@ -361,6 +375,16 @@ class JunctionElement(Element):
                 )
             self.overrides[key] = junction_name
 
+        # Map from subproject alias to local alias
+        self.aliases = node.get_mapping("aliases", default={})
+        self.map_aliases = node.get_enum("map-aliases", _AliasMappingStrategy, default=_AliasMappingStrategy.NONE)
+
+    def get_parent_alias(self, alias):
+        parent_alias = self.aliases.get_str(alias, default=None)
+        if parent_alias is None and self.map_aliases == _AliasMappingStrategy.IDENTITY:
+            parent_alias = alias
+        return parent_alias
+
     def preflight(self):
         pass
 

{BuildStream-2.2.0.dev2 → buildstream-2.2.0.dev3}/src/buildstream/sandbox/_sandboxremote.py

@@ -99,7 +99,6 @@ class SandboxRemote(SandboxREAPI):
                     grpc.StatusCode.RESOURCE_EXHAUSTED,
                     grpc.StatusCode.INTERNAL,
                     grpc.StatusCode.DEADLINE_EXCEEDED,
-                    grpc.StatusCode.UNAVAILABLE,
                 ):
                     raise SandboxError(
                         "Failed contacting remote execution server at {}."

{BuildStream-2.2.0.dev2 → buildstream-2.2.0.dev3}/src/buildstream/source.py

@@ -768,7 +768,7 @@ class Source(Plugin):
                 alias=url_alias, alias_url=project_alias_url, source_url=url_body, extra_data=extra_data
             )
         else:
-            return project.translate_url(url, first_pass=self.__first_pass)
+            return project.translate_url(url, source=self, first_pass=self.__first_pass)
 
     def mark_download_url(self, url: str, *, primary: bool = True) -> None:
         """Identifies the URL that this Source uses to download
@@ -830,7 +830,7 @@ class Source(Plugin):
         # If there is an alias in use, ensure that it exists in the project
         if alias:
             project = self._get_project()
-            if not project.alias_exists(alias, first_pass=self.__first_pass):
+            if not project.alias_exists(alias, first_pass=self.__first_pass, source=self):
                 raise SourceError(
                     "{}: Invalid alias '{}' specified in URL: {}".format(self, alias, url),
                     reason="invalid-source-alias",
@@ -1287,7 +1287,7 @@ class Source(Plugin):
     def _get_alias(self):
         alias = self.__expected_alias
         project = self._get_project()
-        if project.alias_exists(alias, first_pass=self.__first_pass):
+        if project.alias_exists(alias, first_pass=self.__first_pass, source=self):
             # The alias must already be defined in the project's aliases
             # otherwise http://foo gets treated like it contains an alias
             return alias

{BuildStream-2.2.0.dev2 → buildstream-2.2.0.dev3}/src/buildstream/utils.py

@@ -731,7 +731,7 @@ def _get_host_tool_internal(
 ) -> str:
     search_path = os.environ.get("PATH", "").split(os.pathsep)
     if search_subprojects_dir:
-        search_path.insert(0, os.path.join(_site.subprojects, search_subprojects_dir))
+        search_path.append(os.path.join(_site.subprojects, search_subprojects_dir))
 
     program_path = shutil.which(name, path=os.pathsep.join(search_path))
 

{BuildStream-2.2.0.dev2 → buildstream-2.2.0.dev3}/tests/conftest.py

@@ -47,36 +47,37 @@ def pytest_addoption(parser):
     parser.addoption("--remote-cache", action="store_true", default=False, help="Run remote-cache tests only")
 
 
-def pytest_runtest_setup(item):
-    # Without --integration: skip tests not marked with 'integration'
-    if not item.config.getvalue("integration"):
-        if item.get_closest_marker("integration"):
-            pytest.skip("skipping integration test")
-
-    # With --remote-execution: only run tests marked with 'remoteexecution'
-    if item.config.getvalue("remote_execution"):
-        if not item.get_closest_marker("remoteexecution"):
-            pytest.skip("skipping non remote-execution test")
-
-    # Without --remote-execution: skip tests marked with 'remoteexecution'
-    else:
-        if item.get_closest_marker("remoteexecution"):
-            pytest.skip("skipping remote-execution test")
-
-    # With --remote-cache: only run tests marked with 'remotecache'
-    if item.config.getvalue("remote_cache"):
-        if not item.get_closest_marker("remotecache"):
-            pytest.skip("skipping non remote-cache test")
-
-    # Without --remote-cache: skip tests marked with 'remotecache'
-    else:
-        if item.get_closest_marker("remotecache"):
-            pytest.skip("skipping remote-cache test")
-
-    # With --plugins only run plugins tests
-    if item.config.getvalue("plugins"):
-        if not item.get_closest_marker("generic_source_test"):
-            pytest.skip("Skipping not generic source test")
+def pytest_collection_modifyitems(session, config, items):
+    for item in items:
+        # Without --integration: skip tests not marked with 'integration'
+        if not config.getvalue("integration"):
+            if item.get_closest_marker("integration"):
+                item.add_marker(pytest.mark.skip("skipping integration test"))
+
+        # With --remote-execution: only run tests marked with 'remoteexecution'
+        if config.getvalue("remote_execution"):
+            if not item.get_closest_marker("remoteexecution"):
+                item.add_marker(pytest.mark.skip("skipping non remote-execution test"))
+
+        # Without --remote-execution: skip tests marked with 'remoteexecution'
+        else:
+            if item.get_closest_marker("remoteexecution"):
+                item.add_marker(pytest.mark.skip("skipping remote-execution test"))
+
+        # With --remote-cache: only run tests marked with 'remotecache'
+        if config.getvalue("remote_cache"):
+            if not item.get_closest_marker("remotecache"):
+                item.add_marker(pytest.mark.skip("skipping non remote-cache test"))
+
+        # Without --remote-cache: skip tests marked with 'remotecache'
+        else:
+            if item.get_closest_marker("remotecache"):
+                item.add_marker(pytest.mark.skip("skipping remote-cache test"))
+
+        # With --plugins only run plugins tests
+        if config.getvalue("plugins"):
+            if not item.get_closest_marker("generic_source_test"):
+                item.add_marker(pytest.mark.skip("Skipping not generic source test"))
 
 
 #################################################