buildstream-plugins 2.3.0.dev1__tar.gz → 2.5.0.dev0__tar.gz

{buildstream_plugins-2.3.0.dev1 → buildstream-plugins-2.5.0.dev0}/NEWS

@@ -1,3 +1,26 @@
+=========================
+buildstream-plugins 2.5.0
+=========================
+
+ o Drop support for python 3.8 and add support for python 3.13 (#84)
+
+ o Implement Source.collect_source_info() or SourceFetcher.get_source_info() on
+   all source plugins (#87)
+
+=========================
+buildstream-plugins 2.4.0
+=========================
+
+ o docker: Support storing ref in project.refs (#66)
+
+ o simple_mirror: Added SourceMirror plugin supporting enhanced URL formatting (#80)
+
+ o gitlab_lfs_mirror: Added SourceMirror intended for use with GitLab LFS repos (#80)
+
+ o Enhanced testing and CI enhancements (#78)
+
+ o Resurrecting CI (#81)
+
 =========================
 buildstream-plugins 2.3.0
 =========================

{buildstream_plugins-2.3.0.dev1 → buildstream-plugins-2.5.0.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: buildstream-plugins
-Version: 2.3.0.dev1
+Version: 2.5.0.dev0
 Summary: A collection of plugins for BuildStream.
 Home-page: https://buildstream.build
 Author: The Apache Software Foundation
@@ -15,11 +15,11 @@ Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: POSIX
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Software Development :: Build Tools
 Description-Content-Type: text/x-rst; charset=UTF-8
 License-File: LICENSE

{buildstream_plugins-2.3.0.dev1 → buildstream-plugins-2.5.0.dev0}/project.conf

@@ -17,7 +17,7 @@
 # that plugins can be loaded via junctions.
 #
 name: buildstream-plugins
-min-version: 2.0
+min-version: 2.5
 
 plugins:
 - origin: local
@@ -39,3 +39,9 @@ plugins:
   - git
   - patch
   - pip
+
+- origin: local
+  path: src/buildstream_plugins/sourcemirrors
+  source-mirrors:
+  - gitlab_lfs_mirror
+  - simple_mirror

{buildstream_plugins-2.3.0.dev1 → buildstream-plugins-2.5.0.dev0}/requirements/test-requirements.txt

@@ -8,4 +8,5 @@ pylint
 pycodestyle
 pyftpdlib
 responses
+setuptools
 wheel

{buildstream_plugins-2.3.0.dev1 → buildstream-plugins-2.5.0.dev0}/setup.py

@@ -53,11 +53,11 @@ setup(
         "License :: OSI Approved :: Apache Software License",
         "Operating System :: POSIX",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
         "Programming Language :: Python :: 3.12",
+        "Programming Language :: Python :: 3.13",
         "Topic :: Software Development :: Build Tools",
     ],
     description="A collection of plugins for BuildStream.",
@@ -92,6 +92,10 @@ setup(
             "pip = buildstream_plugins.sources.pip",
             "zip = buildstream_plugins.sources.zip",
         ],
+        "buildstream.plugins.sourcemirrors": [
+            "gitlab_lfs_mirror = buildstream_plugins.sourcemirrors.gitlab_lfs_mirror",
+            "simple_mirror = buildstream_plugins.sourcemirrors.simple_mirror",
+        ],
     },
     extras_require={
         "cargo": ['tomli; python_version < "3.11"'],

{buildstream_plugins-2.3.0.dev1 → buildstream-plugins-2.5.0.dev0}/src/buildstream_plugins/__init__.py

@@ -15,4 +15,4 @@
 #
 # Remember to adjust this version number before tagging releases
 #
-__version__ = "2.3.0.dev1"
+__version__ = "2.5.0.dev0"

{buildstream_plugins-2.3.0.dev1 → buildstream-plugins-2.5.0.dev0}/src/buildstream_plugins/elements/make.py

@@ -26,11 +26,6 @@ This is a `BuildElement
 <https://docs.buildstream.build/master/buildstream.scriptelement.html#module-buildstream.scriptelement>`_
 implementation for using GNU make based build.
 
-.. note::
-
-   The ``make`` element is available since `format version 9
-   <https://docs.buildstream.build/master/format_project.html#project-format-version>`_
-
 Here is the default configuration for the ``make`` element in full:
 
   .. literalinclude:: ../../../src/buildstream_plugins/elements/make.yaml

buildstream-plugins-2.5.0.dev0/src/buildstream_plugins/sourcemirrors/gitlab_lfs_mirror.py

@@ -0,0 +1,89 @@
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+gitlab_lfs_mirror - plugin for accessing files stored in git-lfs on GitLab
+==========================================================================
+
+.. note::
+
+   The ``gitlab_lfs_mirror`` plugin is available *Since 2.4.0*
+
+If you store files in a git-lfs repository on gitlab.com, you can access
+them using a URL like
+
+https://gitlab.com/path/to/repo/-/raw/master/path/to/file
+
+which you can then use as a mirror for buildstream. However this only works for
+public repositories. For internal (on self-hosted GitLab instances) and private
+repositories, the above doesn't work since buildstream cannot authenticate with
+the GitLab web UI.
+
+This plugin solves this by going through the GitLab REST API. You need an
+access token that access the Repository Files API (i.e. with read_api or
+read_repository scope). As of this writing, the GitLab CI/CD job token doesn't
+allow access to this API endpoint, so you need a dedicated access token.
+
+**Usage:**
+
+.. code:: yaml
+
+   - name: my-mirror
+     kind: gitlab_lfs_mirror
+     config:
+       url: https://gitlab.example.com/
+       project: mirrors/{alias}
+       ref: main # optional, defaults to master
+       aliases:
+       - my-alias
+       - another-alias
+"""
+
+from posixpath import join
+from urllib.parse import quote
+
+from buildstream import SourceMirror
+
+
+class GitlabLFSMirror(SourceMirror):
+    BST_MIN_VERSION = "2.2"
+
+    def configure(self, node):
+        node.validate_keys(["aliases", "url", "project", "ref"])
+        self.set_supported_aliases(node.get_str_list("aliases"))
+
+        self.url = node.get_str("url")
+        self.project = node.get_str("project")
+        self.ref = node.get_str("ref", "master")
+
+    def translate_url(self, *, alias, alias_url, source_url, extra_data):
+        project_id = quote(self.project.format(alias=alias), safe="")
+        filename = quote(source_url, safe="")
+
+        translated_url = join(
+            self.url,
+            "api/v4/projects",
+            project_id,
+            "repository/files",
+            filename,
+            f"raw?ref={self.ref}&lfs=true",
+        )
+
+        if extra_data is not None:
+            extra_data["http-auth"] = "bearer"
+
+        return translated_url
+
+
+def setup():
+    return GitlabLFSMirror

buildstream-plugins-2.5.0.dev0/src/buildstream_plugins/sourcemirrors/simple_mirror.py

@@ -0,0 +1,59 @@
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+simple_mirror - plugin for simplifying mirror definitions
+=========================================================
+
+.. note::
+
+   The ``simple_mirror`` plugin is available *Since 2.4.0*
+
+**Usage:**
+
+.. code:: yaml
+
+   - name: my-mirror
+     kind: simple_mirror
+     config:
+       url: https://example.com/mirrors/{alias}/
+       aliases:
+       - my-alias
+       - another-alias
+
+This plugin simplifies defining mirrors for projects where the mirrors follow
+a predictable URL format that only varies with the alias name.
+"""
+
+from posixpath import join
+from buildstream import SourceMirror
+
+
+class SimpleMirror(SourceMirror):
+    BST_MIN_VERSION = "2.2"
+
+    def configure(self, node):
+        node.validate_keys(["url", "aliases"])
+        self.set_supported_aliases(node.get_str_list("aliases"))
+
+        self.url = node.get_str("url")
+
+    def translate_url(self, *, alias, alias_url, source_url, extra_data):
+        base_url = self.url.format(alias=alias)
+        translated_url = join(base_url, source_url)
+
+        return translated_url
+
+
+def setup():
+    return SimpleMirror

{buildstream_plugins-2.3.0.dev1 → buildstream-plugins-2.5.0.dev0}/src/buildstream_plugins/sources/bzr.py

@@ -53,9 +53,31 @@ bzr - stage files from a bazaar repository
    # revision number to the one on the tip of the branch specified in 'track'.
    ref: 6622
 
+   # Specify the version to be reported as the *guess_version* when reporting
+   # SourceInfo
+   #
+   # Since 2.5
+   #
+   version: 1.2
+
 See `built-in functionality doumentation
 <https://docs.buildstream.build/master/buildstream.source.html#core-source-builtins>`_ for
 details on common configuration options for sources.
+
+
+Reporting `SourceInfo <https://docs.buildstream.build/master/buildstream.source.html#buildstream.source.SourceInfo>`_
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The bzr source reports the URL of the bzr repository as the *url*.
+
+Further, the bzr source reports the `SourceInfoMedium.BZR
+<https://docs.buildstream.build/master/buildstream.source.html#buildstream.source.SourceInfoMedium.BZR>`_
+*medium* and the` `SourceVersionType.COMMIT
+<https://docs.buildstream.build/master/buildstream.source.html#buildstream.source.SourceVersionType.COMMIT>`_
+*version_type*, for which it reports the bzr revision number as the *version*.
+
+Since the bzr source does not have a way to know what the release version
+corresponds to the revision number, the bzr source exposes the ``version`` configuration
+attribute to allow explicit specification of the *guess_version*.
 """
 
 import os
@@ -66,26 +88,44 @@ from contextlib import contextmanager
 from buildstream import Source, SourceError
 from buildstream import utils
 
+#
+# Soft import of buildstream symbols only available in newer versions
+#
+# The BST_MIN_VERSION will provide a better user experience.
+#
+try:
+    from buildstream import SourceInfoMedium, SourceVersionType
+except ImportError:
+    pass
+
 
 class BzrSource(Source):
     # pylint: disable=attribute-defined-outside-init
 
-    BST_MIN_VERSION = "2.0"
+    BST_MIN_VERSION = "2.5"
 
     def configure(self, node):
-        node.validate_keys(["url", "track", "ref", *Source.COMMON_CONFIG_KEYS])
+        node.validate_keys(["url", "track", "ref", "version", *Source.COMMON_CONFIG_KEYS])
 
         self.original_url = node.get_str("url")
         self.tracking = node.get_str("track")
         self.ref = node.get_str("ref", None)
         self.url = self.translate_url(self.original_url)
+        self.version = node.get_str("version", None)
 
     def preflight(self):
         # Check if bzr is installed, get the binary at the same time.
         self.host_bzr = utils.get_host_tool("bzr")
 
     def get_unique_key(self):
-        return [self.original_url, self.tracking, self.ref]
+        unique_key = [self.original_url, self.tracking, self.ref]
+
+        # Backwards compatible method of supporting configuration
+        # attributes which affect SourceInfo generation.
+        if self.version is not None:
+            unique_key.append(self.version)
+
+        return unique_key
 
     def is_cached(self):
         with self._locked():
@@ -167,6 +207,13 @@ class BzrSource(Source):
                 fail="Failed to switch workspace's parent branch to {}".format(url),
             )
 
+    def collect_source_info(self):
+        return [
+            self.create_source_info(
+                self.url, SourceInfoMedium.BAZAAR, SourceVersionType.COMMIT, self.ref, version_guess=self.version
+            )
+        ]
+
     # _locked()
     #
     # This context manager ensures exclusive access to the

{buildstream_plugins-2.3.0.dev1 → buildstream-plugins-2.5.0.dev0}/src/buildstream_plugins/sources/cargo.py

@@ -61,6 +61,19 @@ Set the `ref` field to an empty list like so: `ref: []`, and then run
 See `built-in functionality doumentation
 <https://docs.buildstream.build/master/buildstream.source.html#core-source-builtins>`_ for
 details on common configuration options for sources.
+
+
+Reporting `SourceInfo <https://docs.buildstream.build/master/buildstream.source.html#buildstream.source.SourceInfo>`_
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The cargo source reports the URL of the archive crates as the *url* for each crate.
+
+Further, the cargo source reports the `SourceInfoMedium.REMOTE_FILE
+<https://docs.buildstream.build/master/buildstream.source.html#buildstream.source.SourceInfoMedium.REMOTE_FILE>`_
+*medium* and the `SourceVersionType.SHA256
+<https://docs.buildstream.build/master/buildstream.source.html#buildstream.source.SourceVersionType.SHA256>`_
+*version_type*, for which it reports the checksum of the archive as the *version*.
+
+The versions extracted from the Cargo.lock at tracking time are used to report the *guess_version*.
 """
 
 import json
@@ -79,6 +92,16 @@ from buildstream import utils
 
 from ._utils import download_file
 
+#
+# Soft import of buildstream symbols only available in newer versions
+#
+# The BST_MIN_VERSION will provide a better user experience.
+#
+try:
+    from buildstream import SourceInfoMedium, SourceVersionType
+except:
+    pass
+
 
 # This automatically goes into .cargo/config
 #
@@ -133,6 +156,12 @@ class Crate(SourceFetcher):
                     "File downloaded from {} has sha256sum '{}', not '{}'!".format(crate_url, sha256, self.sha)
                 )
 
+    def get_source_info(self):
+        url, _ = self._get_url()
+        return self.cargo.create_source_info(
+            url, SourceInfoMedium.REMOTE_FILE, SourceVersionType.SHA256, self.sha, version_guess=self.version
+        )
+
     ########################################################
     #        Helper APIs for the Cargo Source to use       #
     ########################################################
@@ -334,7 +363,7 @@ class Crate(SourceFetcher):
 
 
 class CargoSource(Source):
-    BST_MIN_VERSION = "2.0"
+    BST_MIN_VERSION = "2.5"
 
     # We need the Cargo.lock file to construct our ref at track time
     BST_REQUIRES_PREVIOUS_SOURCES_TRACK = True

{buildstream_plugins-2.3.0.dev1 → buildstream-plugins-2.5.0.dev0}/src/buildstream_plugins/sources/docker.py

@@ -57,6 +57,13 @@ docker - stage files from Docker images
    #
    # **Since**: 2.0.1
 
+   # Specify the version to be reported as the *guess_version* when reporting
+   # SourceInfo
+   #
+   # Since: 2.5
+   #
+   version: 1.2
+
 Note that Docker images may contain device nodes. BuildStream elements cannot
 contain device nodes so those will be dropped. Any regular files in the /dev
 directory will also be dropped.
@@ -64,6 +71,32 @@ directory will also be dropped.
 See `built-in functionality doumentation
 <https://docs.buildstream.build/master/buildstream.source.html#core-source-builtins>`_ for
 details on common configuration options for sources.
+
+
+Reporting `SourceInfo <https://docs.buildstream.build/master/buildstream.source.html#buildstream.source.SourceInfo>`_
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The docker source reports the URL of the docker registry as the *url*.
+
+Further, the docker source reports the `SourceInfoMedium.OCI_IMAGE
+<https://docs.buildstream.build/master/buildstream.source.html#buildstream.source.SourceInfoMedium.OCI_IMAGE>`_
+*medium* and the `SourceVersionType.OCI_DIGEST
+<https://docs.buildstream.build/master/buildstream.source.html#buildstream.source.SourceVersionType.OCI_DIGEST>`_
+*version_type*, for which it reports the content digest of the docker image as the *version*.
+
+Additionally, the docker source reports the docker image name through
+the ``image-name`` key of the *extra_data*.
+
+As such, after removing the scheme from the URL (i.e. remove ``https://``) the same docker
+image can be obtained by calling:
+
+.. code:: bash
+
+   docker pull <url-without-scheme>/<image-name>@<version>
+
+Since the docker source does not have any way to guess what tag is associated
+to the digest, or what release version that would mean; the docker source exposes
+the ``version`` configuration attribute to allow explicit specification of the
+*guess_version*.
 """
 
 import hashlib
@@ -84,6 +117,16 @@ from buildstream.utils import (
     move_atomic,
 )
 
+#
+# Soft import of buildstream symbols only available in newer versions
+#
+# The BST_MIN_VERSION will provide a better user experience.
+#
+try:
+    from buildstream import SourceInfoMedium, SourceVersionType
+except ImportError:
+    pass
+
 _DOCKER_HUB_URL = "https://registry.hub.docker.com"
 
 
@@ -342,7 +385,7 @@ class ReadableTarInfo(tarfile.TarInfo):
 class DockerSource(Source):
     # pylint: disable=too-many-instance-attributes
 
-    BST_MIN_VERSION = "2.0"
+    BST_MIN_VERSION = "2.5"
 
     # Docker identifies images by a content digest calculated from the image's
     # manifest. This corresponds well with the concept of a 'ref' in
@@ -365,7 +408,8 @@ class DockerSource(Source):
         # url is deprecated, but accept it as a valid key so that we can raise
         # a nicer warning.
         node.validate_keys(
-            Source.COMMON_CONFIG_KEYS + ["architecture", "registry-url", "image", "os", "ref", "track", "url"]
+            Source.COMMON_CONFIG_KEYS
+            + ["architecture", "registry-url", "image", "os", "ref", "track", "url", "version"]
         )
 
         if "url" in node and ("image" in node or "registry-url" in node):
@@ -387,30 +431,39 @@ class DockerSource(Source):
             self.original_registry_url = node.get_str("registry-url", _DOCKER_HUB_URL)
             self.registry_url = self.translate_url(self.original_registry_url)
 
-        if "ref" in node:
-            self.digest = self._ref_to_digest(node.get_str("ref"))
-        else:
-            self.digest = None
         self.tag = node.get_str("track", "") or None
 
         self.architecture = node.get_str("architecture", "") or default_architecture()
         self.os = node.get_str("os", "") or default_os()
+        self.version = node.get_str("version", None)
 
-        if not (self.digest or self.tag):
-            raise SourceError("{}: Must specify either 'ref' or 'track' parameters".format(self))
+        self.digest = None
+        self.load_ref(node)
 
         self.client = DockerRegistryV2Client(self.registry_url)
 
         self.manifest = None
 
+    def load_ref(self, node):
+        ref = node.get_str("ref", None)
+        if ref is not None:
+            self.digest = self._ref_to_digest(ref)
+
     def preflight(self):
         return
 
     def get_unique_key(self):
         if self.url is not None:
-            return [self.url, self.digest]
+            unique_key = [self.url, self.digest]
         else:
-            return [self.original_registry_url, self.image, self.digest]
+            unique_key = [self.original_registry_url, self.image, self.digest]
+
+        # Backwards compatible method of supporting configuration
+        # attributes which affect SourceInfo generation.
+        if self.version is not None:
+            unique_key.append(self.version)
+
+        return unique_key
 
     def get_ref(self):
         return None if self.digest is None else self._digest_to_ref(self.digest)
@@ -582,6 +635,28 @@ class DockerSource(Source):
         except (OSError, SourceError, tarfile.TarError) as e:
             raise SourceError("{}: Error staging source: {}".format(self, e)) from e
 
+    def collect_source_info(self):
+
+        # If the image was configured with "url" only rather
+        # than "registry-url" and "image", then self.image
+        # at this point will have a leading forward slash "/".
+        #
+        # Lets just normalize that problem here, as changing
+        # the self.image itself can break cache keys.
+        #
+        image_name = self.image.lstrip("/")
+
+        return [
+            self.create_source_info(
+                self.registry_url,
+                SourceInfoMedium.OCI_IMAGE,
+                SourceVersionType.OCI_DIGEST,
+                self.digest,
+                version_guess=self.version,
+                extra_data={"image-name": image_name},
+            )
+        ]
+
     @staticmethod
     def _get_extract_and_remove_files(layer_tar_path):
         """Return the set of files to remove and extract for a given layer