buildstream-plugins 2.5.0.dev0__tar.gz → 2.8.0.dev0__tar.gz

{buildstream-plugins-2.5.0.dev0 → buildstream_plugins-2.8.0.dev0}/.asf.yaml

@@ -1,4 +1,3 @@
-
 #
 # Avoid diluting the dev mailing list with meta stuff,
 # redirect it all to the commit list.
@@ -40,3 +39,16 @@ github:
   # Enable pages publishing
   ghp_branch: gh-pages
   ghp_path: /docs
+  rulesets:
+    - name: "Default Branch Protection"
+      type: branch
+      branches:
+        includes:
+          - "~DEFAULT_BRANCH"
+          - "release/*"
+          - "rel/*"
+        excludes: []
+      bypass_teams:
+        - root
+      restrict_deletion: true
+      restrict_force_push: true

{buildstream-plugins-2.5.0.dev0 → buildstream_plugins-2.8.0.dev0}/NEWS

@@ -1,3 +1,19 @@
+=========================
+buildstream-plugins 2.8.0
+=========================
+
+ o docker: support OCI images (#105)
+
+=========================
+buildstream-plugins 2.7.0
+=========================
+
+ o cargo: Support source provenance (#98)
+
+ o Fix multiple issues in the Docker source plugin (#96)
+
+ o Support Python 3.14 and drop support for Python 3.9 (#94, #97)
+
 =========================
 buildstream-plugins 2.5.0
 =========================

{buildstream-plugins-2.5.0.dev0/src/buildstream_plugins.egg-info → buildstream_plugins-2.8.0.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: buildstream-plugins
-Version: 2.5.0.dev0
+Version: 2.8.0.dev0
 Summary: A collection of plugins for BuildStream.
 Home-page: https://buildstream.build
 Author: The Apache Software Foundation
@@ -15,17 +15,28 @@ Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: POSIX
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Software Development :: Build Tools
 Description-Content-Type: text/x-rst; charset=UTF-8
 License-File: LICENSE
 License-File: NOTICE
 Provides-Extra: cargo
 Requires-Dist: tomli; python_version < "3.11" and extra == "cargo"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: project-url
+Dynamic: provides-extra
+Dynamic: summary
 
 BuildStream Plugins
 ===================

{buildstream-plugins-2.5.0.dev0 → buildstream_plugins-2.8.0.dev0}/pyproject.toml

@@ -12,13 +12,8 @@
 #  limitations under the License.
 #
 [build-system]
-requires = [
-    # We need at least version 36.6.0 that introduced "build_meta"
-    "setuptools>=36.6.0",
-    # In order to build wheels, and as required by PEP 517
-    "wheel",
-    "Cython"
-]
+# We need at least version 36.6.0 that introduced "build_meta"
+requires = ["setuptools>=36.6.0"]
 build-backend = "setuptools.build_meta"
 
 [tool.black]

{buildstream-plugins-2.5.0.dev0 → buildstream_plugins-2.8.0.dev0}/setup.py

@@ -53,11 +53,11 @@ setup(
         "License :: OSI Approved :: Apache Software License",
         "Operating System :: POSIX",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
         "Programming Language :: Python :: 3.12",
         "Programming Language :: Python :: 3.13",
+        "Programming Language :: Python :: 3.14",
         "Topic :: Software Development :: Build Tools",
     ],
     description="A collection of plugins for BuildStream.",

{buildstream-plugins-2.5.0.dev0 → buildstream_plugins-2.8.0.dev0}/src/buildstream_plugins/__init__.py

@@ -15,4 +15,4 @@
 #
 # Remember to adjust this version number before tagging releases
 #
-__version__ = "2.5.0.dev0"
+__version__ = "2.8.0.dev0"

{buildstream-plugins-2.5.0.dev0 → buildstream_plugins-2.8.0.dev0}/src/buildstream_plugins/sources/cargo.py

@@ -74,6 +74,10 @@ Further, the cargo source reports the `SourceInfoMedium.REMOTE_FILE
 *version_type*, for which it reports the checksum of the archive as the *version*.
 
 The versions extracted from the Cargo.lock at tracking time are used to report the *guess_version*.
+
+Additional source info can be specified using the `provenance` field for each referenced crate as per
+`built-in source provenance documentation
+<https://docs.buildstream.build/master/buildstream.source.html#built-in-functionality>`_.
 """
 
 import json
@@ -99,7 +103,7 @@ from ._utils import download_file
 #
 try:
     from buildstream import SourceInfoMedium, SourceVersionType
-except:
+except ImportError:
     pass
 
 
@@ -125,13 +129,14 @@ _default_vendor_config_template = (
 #    sha (str|None): The sha256 checksum of the downloaded crate
 #
 class Crate(SourceFetcher):
-    def __init__(self, cargo, name, version, sha=None):
+    def __init__(self, cargo, name, version, sha=None, provenance=None):
         super().__init__()
 
         self.cargo = cargo
         self.name = name
         self.version = str(version)
         self.sha = sha
+        self.provenance = provenance
         self.mark_download_url(cargo.url)
 
     ########################################################
@@ -159,7 +164,12 @@ class Crate(SourceFetcher):
     def get_source_info(self):
         url, _ = self._get_url()
         return self.cargo.create_source_info(
-            url, SourceInfoMedium.REMOTE_FILE, SourceVersionType.SHA256, self.sha, version_guess=self.version
+            url,
+            SourceInfoMedium.REMOTE_FILE,
+            SourceVersionType.SHA256,
+            self.sha,
+            version_guess=self.version,
+            provenance_node=self.provenance,
         )
 
     ########################################################
@@ -368,16 +378,29 @@ class CargoSource(Source):
     # We need the Cargo.lock file to construct our ref at track time
     BST_REQUIRES_PREVIOUS_SOURCES_TRACK = True
 
+    BST_CUSTOM_SOURCE_PROVENANCE = True
+
     ########################################################
     #       Plugin/Source API method implementations       #
     ########################################################
     def configure(self, node):
-
         # The url before any aliasing
         #
         self.original_url = node.get_str("url", "https://static.crates.io/crates")
         self.cargo_lock = node.get_str("cargo-lock", "Cargo.lock")
         self.vendor_dir = node.get_str("vendor-dir", "crates")
+        self.provenance = {}
+
+        # Because source provenance info lives in the ref it would be
+        # removed upon source track, create a backup here so it is
+        # available to be added back in when setting the ref
+        ref = node.get_sequence("ref", None)
+        if ref is not None:
+            for crate in ref:
+                if "provenance" in crate:
+                    crate_name = crate.get_str("name")
+                    provenance = crate.get_mapping("provenance")
+                    self.provenance[crate_name] = provenance.strip_node_info()
 
         # If the specified URL is just an alias, require the alias to resolve
         # to a URL with a trailing slash. Otherwise, append a trailing slash if
@@ -413,6 +436,14 @@ class CargoSource(Source):
         return self.ref
 
     def set_ref(self, ref, node):
+        def add_provenance(crate):
+            if crate["name"] in self.provenance:
+                crate["provenance"] = self.provenance[crate["name"]]
+            return crate
+
+        # add provenance back in
+        ref = list(map(add_provenance, ref))
+
         node["ref"] = ref
         self._recompute_crates(node.get_sequence("ref"))
 
@@ -513,6 +544,7 @@ class CargoSource(Source):
                 crate.get_str("name"),
                 crate.get_str("version"),
                 sha=crate.get_str("sha", None),
+                provenance=crate.get_mapping("provenance", None),
             )
             for crate in refs
         ]

{buildstream-plugins-2.5.0.dev0 → buildstream_plugins-2.8.0.dev0}/src/buildstream_plugins/sources/docker.py

@@ -21,8 +21,8 @@
 #
 
 """
-docker - stage files from Docker images
-=======================================
+docker - stage files from Docker/OCI images
+===========================================
 
 **Usage:**
 
@@ -107,6 +107,8 @@ import shutil
 import tarfile
 import urllib.parse
 
+from collections import OrderedDict
+
 import requests
 
 from buildstream import Source, SourceError
@@ -115,6 +117,7 @@ from buildstream.utils import (
     sha256sum,
     link_files,
     move_atomic,
+    get_umask,
 )
 
 #
@@ -279,6 +282,8 @@ class DockerRegistryV2Client:
         accept_types = [
             "application/vnd.docker.distribution.manifest.v2+json",
             "application/vnd.docker.distribution.manifest.list.v2+json",
+            "application/vnd.oci.image.index.v1+json",
+            "application/vnd.oci.image.manifest.v1+json",
         ]
 
         manifest_url = urljoin(image_path, "manifests", urllib.parse.quote(reference))
@@ -316,7 +321,10 @@ class DockerRegistryV2Client:
                 manifest=response.text,
             )
 
-        if manifest["mediaType"] == "application/vnd.docker.distribution.manifest.list.v2+json":
+        if manifest["mediaType"] in (
+            "application/vnd.docker.distribution.manifest.list.v2+json",
+            "application/vnd.oci.image.index.v1+json",
+        ):
             # This is a "fat manifest", we need to narrow down to a specific
             # architecture.
             for sub in manifest["manifests"]:
@@ -332,7 +340,10 @@ class DockerRegistryV2Client:
                 "No images found for architecture {}, OS {}".format(architecture, os_),
                 manifest=response.text,
             )
-        if manifest["mediaType"] == "application/vnd.docker.distribution.manifest.v2+json":
+        if manifest["mediaType"] in (
+            "application/vnd.docker.distribution.manifest.v2+json",
+            "application/vnd.oci.image.manifest.v1+json",
+        ):
             return response.text, our_digest
         else:
             raise DockerManifestError(
@@ -363,23 +374,26 @@ class DockerRegistryV2Client:
 
 class ReadableTarInfo(tarfile.TarInfo):
     """
-    The goal is to override`TarFile`'s `extractall` semantics by ensuring that on extraction, the
-    files are readable by the owner of the file. This is done by over-riding the accessor for the
-    mode` attribute in `TarInfo`, class that encapsulates the internal meta-data of the tarball,
+    The goal is to override `TarFile`'s `extractall` semantics by ensuring that on extraction, the
+    files are readable by the owner of the file. This is done by overriding the accessor for the
+    `mode` attribute in `TarInfo`, the class that encapsulates the internal meta-data of the tarball,
     so that the owner-read bit is always set.
     """
 
-    # The mode attribute is not declared as a property and so
-    # this trips up the static type checker, mark this as "ignore"
-    #
+    # https://github.com/python/mypy/issues/4125
     @property  # type: ignore
     def mode(self):
-        # ensure file is readable by owner
-        return self.__permission | 0o400
+        # Respect umask instead of the file mode stored in the archive.
+        # The only bit used from the embedded mode is the executable bit for files.
+        umask = get_umask()
+        if self.isdir() or bool(self.__permission & 0o100):
+            return 0o777 & ~umask
+        else:
+            return 0o666 & ~umask
 
     @mode.setter
     def mode(self, permission):
-        self.__permission = permission
+        self.__permission = permission  # pylint: disable=attribute-defined-outside-init
 
 
 class DockerSource(Source):
@@ -570,7 +584,10 @@ class DockerSource(Source):
                     raise SourceError(e) from e
 
                 for layer in manifest["layers"]:
-                    if layer["mediaType"] != "application/vnd.docker.image.rootfs.diff.tar.gzip":
+                    if layer["mediaType"] not in (
+                        "application/vnd.docker.image.rootfs.diff.tar.gzip",
+                        "application/vnd.oci.image.layer.v1.tar+gzip",
+                    ):
                         raise SourceError("Unsupported layer type: {}".format(layer["mediaType"]))
 
                     layer_digest = layer["digest"]
@@ -605,31 +622,54 @@ class DockerSource(Source):
             raise SourceError("Unable to load manifest: {}".format(e)) from e
 
         try:
+            layer_members = []
+
+            # Create a list of members to extract from each layer
             for layer in manifest["layers"]:
                 layer_digest = layer["digest"]
                 blob_path = os.path.join(mirror_dir, layer_digest + ".tar.gz")
 
                 self._verify_blob(blob_path, expected_digest=layer_digest)
-                (
-                    extract_fileset,
-                    white_out_fileset,
-                ) = self._get_extract_and_remove_files(blob_path)
-
-                # remove files associated with whiteouts
-                for white_out_file in white_out_fileset:
-                    white_out_file = os.path.join(directory, white_out_file)
-                    os.remove(white_out_file)
 
-                # extract files for the current layer
+                members_dict, whiteout_paths, opaque_whiteout_dirs = self._get_members_and_whiteout_paths(blob_path)
+
+                # Process opaque whiteouts: remove members under the corresponding directories from previous layers
+                for _, prev_members_dict in layer_members:
+                    members_to_remove = []
+                    for member_path in prev_members_dict:
+                        # Check if this member should be removed by any opaque whiteout
+                        for opaque_whiteout_dir in opaque_whiteout_dirs:
+                            if member_path.startswith(opaque_whiteout_dir):
+                                members_to_remove.append(member_path)
+                                break
+                    for member_path in members_to_remove:
+                        del prev_members_dict[member_path]
+
+                # Process whiteouts: remove corresponding members from previous layers
+                for _, prev_members_dict in layer_members:
+                    for whiteout_path in whiteout_paths:
+                        prev_members_dict.pop(whiteout_path, None)
+
+                layer_members.append((blob_path, members_dict))
+
+            # Extract files for each layer
+            for blob_path, members_dict in layer_members:
+                if not members_dict:
+                    # No files to extract from this layer
+                    continue
+
+                # Extract files for the current layer
                 with tarfile.open(blob_path, tarinfo=ReadableTarInfo) as tar:
                     with self.tempdir() as td:
+                        members = list(members_dict.values())
+
                         if hasattr(tarfile, "tar_filter"):
                             # Python 3.12+ (and older versions with backports)
                             tar.extraction_filter = tarfile.tar_filter
                         else:
-                            for member in extract_fileset:
+                            for member in members:
                                 self._assert_tarinfo_safe(member, td)
-                        tar.extractall(path=td, members=extract_fileset)
+                        tar.extractall(path=td, members=members)
                         link_files(td, directory)
 
         except (OSError, SourceError, tarfile.TarError) as e:
@@ -658,14 +698,14 @@ class DockerSource(Source):
         ]
 
     @staticmethod
-    def _get_extract_and_remove_files(layer_tar_path):
+    def _get_members_and_whiteout_paths(layer_tar_path):
         """Return the set of files to remove and extract for a given layer
 
-        :param layer_tar_path: The path where a layer has been extracted
-        :return: Tuple of filesets
-          - extract_fileset: files to extract into staging directory
-          - delete_fileset: files to remove from staging directory as the current layer
-            contains a whiteout corresponding to a staged file in the previous layers
+        :param layer_tar_path: The path to the layer tar.gz file
+        :return: Tuple of (members_dict, whiteout_paths, opaque_whiteout_dirs)
+          - members_dict: OrderedDict of TarInfo members to extract into staging directory
+          - whiteout_paths: list of file paths that should be removed from previous layers
+          - opaque_whiteout_dirs: list of directory paths whose contents should be removed from previous layers
 
         """
 
@@ -677,7 +717,7 @@ class DockerSource(Source):
             """
             # whiteout files have the syntax of `*/.wh.*`
             file_name = os.path.basename(white_out_file)
-            path = os.path.join(os.path.dirname(white_out_file), file_name.split(".wh.")[1])
+            path = os.path.join(os.path.dirname(white_out_file), file_name.split(".wh.", maxsplit=1)[1])
             return path
 
         def is_regular_file(info):
@@ -688,16 +728,27 @@ class DockerSource(Source):
             """
             return not (info.name.startswith("dev/") or info.isdev())
 
-        with tarfile.open(layer_tar_path) as tar:
-            extract_fileset = []
-            delete_fileset = []
+        with tarfile.open(layer_tar_path, tarinfo=ReadableTarInfo) as tar:
+            members_dict = OrderedDict()
+            whiteout_paths = []
+            opaque_whiteout_dirs = []
+
             for member in tar.getmembers():
-                if os.path.basename(member.name).startswith(".wh."):
-                    delete_fileset.append(strip_wh(member.name))
+                member_basename = os.path.basename(member.name)
+                if member_basename == ".wh..wh..opq":
+                    # Opaque whiteout file
+                    opaque_whiteout_dir = os.path.dirname(member.name)
+                    if opaque_whiteout_dir != "":
+                        # Add "/" to facilitate path prefix matching
+                        opaque_whiteout_dir += "/"
+                    opaque_whiteout_dirs.append(opaque_whiteout_dir)
+                elif member_basename.startswith(".wh."):
+                    # Whiteout file
+                    whiteout_paths.append(strip_wh(member.name))
                 elif is_regular_file(member):
-                    extract_fileset.append(member)
+                    members_dict[member.name] = member
 
-        return extract_fileset, delete_fileset
+        return members_dict, whiteout_paths, opaque_whiteout_dirs
 
     # Assert that a tarfile is safe to extract; specifically, make
     # sure that we don't do anything outside of the target

{buildstream-plugins-2.5.0.dev0 → buildstream_plugins-2.8.0.dev0/src/buildstream_plugins.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: buildstream-plugins
-Version: 2.5.0.dev0
+Version: 2.8.0.dev0
 Summary: A collection of plugins for BuildStream.
 Home-page: https://buildstream.build
 Author: The Apache Software Foundation
@@ -15,17 +15,28 @@ Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: POSIX
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Software Development :: Build Tools
 Description-Content-Type: text/x-rst; charset=UTF-8
 License-File: LICENSE
 License-File: NOTICE
 Provides-Extra: cargo
 Requires-Dist: tomli; python_version < "3.11" and extra == "cargo"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: project-url
+Dynamic: provides-extra
+Dynamic: summary
 
 BuildStream Plugins
 ===================