taskcluster-taskgraph 9.2.1__tar.gz → 10.0.0__tar.gz

{taskcluster-taskgraph-9.2.1/src/taskcluster_taskgraph.egg-info → taskcluster-taskgraph-10.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: taskcluster-taskgraph
-Version: 9.2.1
+Version: 10.0.0
 Summary: Build taskcluster taskgraphs
 Home-page: https://github.com/taskcluster/taskgraph
 Classifier: Development Status :: 5 - Production/Stable

{taskcluster-taskgraph-9.2.1 → taskcluster-taskgraph-10.0.0/src/taskcluster_taskgraph.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: taskcluster-taskgraph
-Version: 9.2.1
+Version: 10.0.0
 Summary: Build taskcluster taskgraphs
 Home-page: https://github.com/taskcluster/taskgraph
 Classifier: Development Status :: 5 - Production/Stable

{taskcluster-taskgraph-9.2.1 → taskcluster-taskgraph-10.0.0}/src/taskgraph/__init__.py

@@ -2,7 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-__version__ = "9.2.1"
+__version__ = "10.0.0"
 
 # Maximum number of dependencies a single task can have
 # https://docs.taskcluster.net/reference/platform/taskcluster-queue/references/api#createTask

{taskcluster-taskgraph-9.2.1 → taskcluster-taskgraph-10.0.0}/src/taskgraph/run-task/fetch-content

@@ -10,6 +10,7 @@ import contextlib
 import datetime
 import gzip
 import hashlib
+import io
 import json
 import lzma
 import multiprocessing
@@ -332,21 +333,41 @@ def gpg_verify_path(path: pathlib.Path, public_key_data: bytes, signature_data:
             subprocess.run(["gpgconf", "--kill", "gpg-agent"], env=env)
 
 
-def open_tar_stream(path: pathlib.Path):
-    """"""
-    if path.suffix == ".bz2":
-        return bz2.open(str(path), "rb")
-    elif path.suffix in (".gz", ".tgz") :
-        return gzip.open(str(path), "rb")
-    elif path.suffix == ".xz":
-        return lzma.open(str(path), "rb")
-    elif path.suffix == ".zst":
-        dctx = ZstdDecompressor()
-        return dctx.stream_reader(path.open("rb"))
-    elif path.suffix == ".tar":
-        return path.open("rb")
-    else:
-        raise ValueError("unknown archive format for tar file: %s" % path)
+class ArchiveTypeNotSupported(Exception):
+    def __init__(self, path: pathlib.Path):
+        super(Exception, self).__init__("Archive type not supported for %s" % path)
+
+
+def open_stream(path: pathlib.Path):
+    """Attempt to identify a path as an extractable archive by looking at its
+    content."""
+    fh = path.open(mode="rb")
+    magic = fh.read(6)
+    fh.seek(0)
+    if magic[:2] == b"PK":
+        return "zip", fh
+    if magic[:2] == b"\x1f\x8b":
+        fh = gzip.GzipFile(fileobj=fh)
+    elif magic[:3] == b"BZh":
+        fh = bz2.BZ2File(fh)
+    elif magic == b"\xfd7zXZ\x00":
+        fh = lzma.LZMAFile(fh)
+    elif magic[:4] == b"\x28\xb5\x2f\xfd":
+        fh = ZstdDecompressor().stream_reader(fh)
+    fh = io.BufferedReader(fh)
+    try:
+        # A full tar info header is 512 bytes.
+        headers = fh.peek(512)
+        # 257 is the offset of the ustar magic.
+        magic = headers[257 : 257 + 8]
+        # For older unix tar, rely on TarInfo.frombuf's checksum check
+        if magic in (b"ustar\x0000", b"ustar  \x00") or tarfile.TarInfo.frombuf(
+            headers[:512], tarfile.ENCODING, "surrogateescape"
+        ):
+            return "tar", fh
+    except Exception as e:
+        pass
+    raise ArchiveTypeNotSupported(path)
 
 
 def archive_type(path: pathlib.Path):
@@ -359,7 +380,7 @@ def archive_type(path: pathlib.Path):
         return None
 
 
-def extract_archive(path, dest_dir, typ):
+def extract_archive(path, dest_dir):
     """Extract an archive to a destination directory."""
 
     # Resolve paths to absolute variants.
@@ -371,8 +392,8 @@ def extract_archive(path, dest_dir, typ):
 
     # We pipe input to the decompressor program so that we can apply
     # custom decompressors that the program may not know about.
+    typ, ifh = open_stream(path)
     if typ == "tar":
-        ifh = open_tar_stream(path)
         # On Windows, the tar program doesn't support things like symbolic
         # links, while Windows actually support them. The tarfile module in
         # python does. So use that. But since it's significantly slower than
@@ -419,10 +440,8 @@ def repack_archive(
 ):
     assert orig != dest
     log("Repacking as %s" % dest)
-    orig_typ = archive_type(orig)
+    orig_typ, ifh = open_stream(orig)
     typ = archive_type(dest)
-    if not orig_typ:
-        raise Exception("Archive type not supported for %s" % orig.name)
     if not typ:
         raise Exception("Archive type not supported for %s" % dest.name)
 
@@ -448,7 +467,7 @@ def repack_archive(
         ctx = ZstdCompressor()
         if orig_typ == "zip":
             assert typ == "tar"
-            zip = zipfile.ZipFile(orig)
+            zip = zipfile.ZipFile(ifh)
             # Convert the zip stream to a tar on the fly.
             with ctx.stream_writer(fh) as compressor, tarfile.open(
                 fileobj=compressor, mode="w:"
@@ -490,7 +509,6 @@ def repack_archive(
                 raise Exception("Repacking a tar to zip is not supported")
             assert typ == "tar"
 
-            ifh = open_tar_stream(orig)
             if filter:
                 # To apply the filter, we need to open the tar stream and
                 # tweak it.
@@ -533,11 +551,12 @@ def fetch_and_extract(url, dest_dir, extract=True, sha256=None, size=None):
     if not extract:
         return
 
-    typ = archive_type(dest_path)
-    if typ:
-        extract_archive(dest_path, dest_dir, typ)
+    try:
+        extract_archive(dest_path, dest_dir)
         log("Removing %s" % dest_path)
         dest_path.unlink()
+    except ArchiveTypeNotSupported:
+        pass
 
 
 def fetch_urls(downloads):

{taskcluster-taskgraph-9.2.1 → taskcluster-taskgraph-10.0.0}/src/taskgraph/transforms/docker_image.py

@@ -24,9 +24,9 @@ CONTEXTS_DIR = "docker-contexts"
 DIGEST_RE = re.compile("^[0-9a-f]{64}$")
 
 IMAGE_BUILDER_IMAGE = (
-    "mozillareleases/image_builder:5.0.0"
+    "mozillareleases/image_builder:5.1.0"
     "@sha256:"
-    "e510a9a9b80385f71c112d61b2f2053da625aff2b6d430411ac42e424c58953f"
+    "7fe70dcedefffffa03237ba5d456d42e0d7461de066db3f7a7c280a104869cd5"
 )
 
 transforms = TransformSequence()

{taskcluster-taskgraph-9.2.1 → taskcluster-taskgraph-10.0.0}/src/taskgraph/transforms/run/__init__.py

@@ -14,7 +14,7 @@ import copy
 import json
 import logging
 
-from voluptuous import Any, Exclusive, Extra, Optional, Required
+from voluptuous import Exclusive, Extra, Optional, Required
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.transforms.cached_tasks import order_tasks
@@ -85,7 +85,6 @@ run_description_schema = Schema(
         },
         # A list of artifacts to install from 'fetch' tasks.
         Optional("fetches"): {
-            Any("toolchain", "fetch"): [str],
             str: [
                 str,
                 fetches_schema,
@@ -252,7 +251,15 @@ def use_fetches(config, tasks):
         for kind in sorted(fetches):
             artifacts = fetches[kind]
             if kind in ("fetch", "toolchain"):
-                for fetch_name in sorted(artifacts):
+                for artifact in sorted(artifacts):
+                    # Convert name only fetch entries to full fledged ones for
+                    # easier processing.
+                    if isinstance(artifact, str):
+                        artifact = {
+                            "artifact": artifact,
+                        }
+
+                    fetch_name = artifact["artifact"]
                     label = f"{kind}-{fetch_name}"
                     label = aliases.get(label, label)
                     if label not in artifact_names:
@@ -263,15 +270,21 @@ def use_fetches(config, tasks):
                         env.update(extra_env[label])
 
                     path = artifact_names[label]
+                    dest = artifact.get("dest", None)
+                    extract = artifact.get("extract", True)
+                    verify_hash = artifact.get("verify-hash", False)
 
                     dependencies[label] = label
-                    task_fetches.append(
-                        {
-                            "artifact": path,
-                            "task": f"<{label}>",
-                            "extract": True,
-                        }
-                    )
+                    fetch = {
+                        "artifact": path,
+                        "task": f"<{label}>",
+                        "extract": extract,
+                    }
+                    if dest is not None:
+                        fetch["dest"] = dest
+                    if verify_hash:
+                        fetch["verify-hash"] = verify_hash
+                    task_fetches.append(fetch)
             else:
                 if kind not in dependencies:
                     raise Exception(

{taskcluster-taskgraph-9.2.1 → taskcluster-taskgraph-10.0.0}/test/test_transforms_run.py

@@ -6,6 +6,7 @@
 Tests for the 'run' transform subsystem.
 """
 
+import json
 import os
 from copy import deepcopy
 from pprint import pprint
@@ -89,6 +90,12 @@ def assert_use_fetches_toolchain_env(task):
     assert task["worker"]["env"]["FOO"] == "1"
 
 
+def assert_use_fetches_toolchain_extract_false(task):
+    fetches = json.loads(task["worker"]["env"]["MOZ_FETCHES"]["task-reference"])
+    assert len(fetches) == 1
+    assert fetches[0]["extract"] is False
+
+
 @pytest.mark.parametrize(
     "task,kind_dependencies_tasks",
     (
@@ -107,6 +114,20 @@ def assert_use_fetches_toolchain_env(task):
             ],
             id="toolchain_env",
         ),
+        pytest.param(
+            {"fetches": {"toolchain": [{"artifact": "foo", "extract": False}]}},
+            [
+                Task(
+                    kind="toolchain",
+                    label="toolchain-foo",
+                    attributes={
+                        "toolchain-artifact": "target.whl",
+                    },
+                    task={},
+                )
+            ],
+            id="toolchain_extract_false",
+        ),
     ),
 )
 def test_use_fetches(