taskcluster-taskgraph 11.2.2__tar.gz → 11.2.4__tar.gz

{taskcluster_taskgraph-11.2.2 → taskcluster_taskgraph-11.2.4}/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Change Log
 
+## [11.2.4] - 2024-10-16
+
+### Fixed
+
+- Regression to `static-url` fetches when using the `strip_components` or `add_prefix` features
+
+## [11.2.3] - 2024-10-16
+
+### Fixed
+
+- A couple more regressions with the Decision image
+  - Cleaned up temporary `uv` lockfile owend by root
+  - Added another `--no-cache` flag to a `uv` command
+- `run-task` now installs packages to user site dir when installing requirements with `uv`
+- Fetch tasks using `static-url` will now properly rename non-archive files
+
 ## [11.2.2] - 2024-10-15
 
 ### Fixed

{taskcluster_taskgraph-11.2.2 → taskcluster_taskgraph-11.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: taskcluster-taskgraph
-Version: 11.2.2
+Version: 11.2.4
 Summary: Build taskcluster taskgraphs
 Project-URL: Repository, https://github.com/taskcluster/taskgraph
 Project-URL: Issues, https://github.com/taskcluster/taskgraph/issues

{taskcluster_taskgraph-11.2.2 → taskcluster_taskgraph-11.2.4}/src/taskgraph/__init__.py

@@ -2,7 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-__version__ = "11.2.2"
+__version__ = "11.2.4"
 
 # Maximum number of dependencies a single task can have
 # https://docs.taskcluster.net/docs/reference/platform/queue/api#createTask

{taskcluster_taskgraph-11.2.2 → taskcluster_taskgraph-11.2.4}/src/taskgraph/run-task/fetch-content

@@ -435,11 +435,49 @@ def extract_archive(path, dest_dir):
     log("%s extracted in %.3fs" % (path, time.time() - t0))
 
 
+def should_repack_archive(orig: pathlib.Path, dest: pathlib.Path, strip_components=0, add_prefix="") -> bool:
+    """
+    Determines whether we should attempt to repack an archive based on the naming conventions used
+    in the original file path, the destination file path, and any modifications such as stripping
+    components or adding a prefix.
+    """
+
+    if strip_components or add_prefix:
+        # If strip_components or add_prefix is specified, we should always repack.
+        return True
+
+    if orig.suffixes == dest.suffixes:
+        # If all suffixes are exactly the same, then a rename will suffice.
+        return False
+
+    if dest.suffixes[-2:] == [".tar", ".zst"]:
+        # If the destination is a ".tar.zst" file then we will always try to repack it ourselves.
+        return True
+
+    if orig.suffix == dest.suffix:
+        # If the final suffix is the same, this is likely just a rename.
+        #
+        # This may be a case where multiple suffixes were detected unnecessarily due to the file name
+        # containing a semantic version. For example the file "python-3.8.10-amd64.exe" would be detected
+        # to have three suffixes [".8", ".10-amd64", ".exe"]. Changing this to "python.exe" should be a
+        # simple rename, rather than a repack.
+        #
+        # In this case, we will try to determine if the original path has a supported archive suffix.
+        # If the original path is detected to be an archive, we will try to repack, otherwise rename.
+        return archive_type(orig) is not None
+
+    # Otherwise, if the paths aren't the same, assume it's an archive and try to repack.
+    #
+    # It would be best to fail early if the repack fails than to fail during a test because a renamed
+    # file was incorrect type after all.
+    return True
+
+
 def repack_archive(
     orig: pathlib.Path, dest: pathlib.Path, strip_components=0, prefix=""
 ):
     assert orig != dest
-    log("Repacking as %s" % dest)
+    log(f"Repacking {orig} as {dest}")
     orig_typ, ifh = open_stream(orig)
     typ = archive_type(dest)
     if not typ:
@@ -534,6 +572,9 @@ def repack_archive(
                 # We only change compression here. The tar stream is unchanged.
                 ctx.copy_stream(ifh, fh)
 
+        else:
+            raise Exception(f"Attempt to repack an archive of unsupported type {orig_typ}")
+
 
 def fetch_and_extract(url, dest_dir, extract=True, sha256=None, size=None):
     """Fetch a URL and extract it to a destination path.
@@ -773,8 +814,12 @@ def command_static_url(args):
         if gpg_sig_url:
             gpg_verify_path(dl_dest, gpg_key, gpg_signature)
 
-        if dl_dest != dest or args.strip_components or args.add_prefix:
+        if should_repack_archive(dl_dest, dest, args.strip_components, args.add_prefix):
             repack_archive(dl_dest, dest, args.strip_components, args.add_prefix)
+        elif dl_dest != dest:
+            log(f"Renaming {dl_dest} to {dest}")
+            dl_dest.rename(dest)
+
     except Exception:
         try:
             dl_dest.unlink()
@@ -783,7 +828,7 @@ def command_static_url(args):
 
         raise
 
-    if dl_dest != dest:
+    if dl_dest != dest and dl_dest.exists():
         log("Removing %s" % dl_dest)
         dl_dest.unlink()
 

{taskcluster_taskgraph-11.2.2 → taskcluster_taskgraph-11.2.4}/src/taskgraph/run-task/run-task

@@ -1012,9 +1012,10 @@ def install_pip_requirements(repositories):
 
     # TODO: Stop using system Python (#381)
     if shutil.which("uv"):
-        cmd = ["uv", "pip", "install", "--python", sys.executable, "--break-system-packages"]
+        user_site_dir = subprocess.run([sys.executable, "-msite", "--user-site"], capture_output=True, text=True).stdout.strip()
+        cmd = ["uv", "pip", "install", "--python", sys.executable, "--target", user_site_dir]
     else:
-        cmd = [sys.executable, "-mpip", "install", "--break-system-packages"]
+        cmd = [sys.executable, "-mpip", "install", "--user", "--break-system-packages"]
 
     if os.environ.get("PIP_DISABLE_REQUIRE_HASHES") != "1":
         cmd.append("--require-hashes")

{taskcluster_taskgraph-11.2.2 → taskcluster_taskgraph-11.2.4}/taskcluster/docker/decision/system-setup.sh

@@ -9,7 +9,7 @@ apt-get install -y --force-yes --no-install-recommends \
     python3-pip
 
 pushd /setup/taskgraph
-uv export --no-dev > /setup/requirements.txt
+uv export --no-cache --no-dev > /setup/requirements.txt
 uv pip install --no-cache --system --break-system-packages -r /setup/requirements.txt
 uv pip install --no-cache --system --break-system-packages --no-deps .
 popd
@@ -18,3 +18,4 @@ apt-get clean
 apt-get autoclean
 rm -rf /var/lib/apt/lists/
 rm -rf /setup
+rm /tmp/uv-*.lock

taskcluster_taskgraph-11.2.4/test/test_scripts_fetch_content.py

@@ -0,0 +1,178 @@
+import os
+import pathlib
+import urllib.request
+from importlib.machinery import SourceFileLoader
+from importlib.util import module_from_spec, spec_from_loader
+from unittest.mock import MagicMock
+
+import pytest
+
+import taskgraph
+
+
+@pytest.fixture(scope="module")
+def fetch_content_mod():
+    spec = spec_from_loader(
+        "fetch-content",
+        SourceFileLoader(
+            "fetch-content",
+            os.path.join(
+                os.path.dirname(taskgraph.__file__), "run-task", "fetch-content"
+            ),
+        ),
+    )
+    assert spec
+    assert spec.loader
+    mod = module_from_spec(spec)
+    spec.loader.exec_module(mod)
+    return mod
+
+
+@pytest.mark.parametrize(
+    "url,sha256,size,headers,raises",
+    (
+        pytest.param(
+            "https://example.com",
+            "c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2",
+            6,
+            ["User-Agent: foobar"],
+            False,
+            id="valid",
+        ),
+        pytest.param(
+            "https://example.com",
+            "abcdef",
+            6,
+            ["User-Agent: foobar"],
+            True,
+            id="invalid sha256",
+        ),
+        pytest.param(
+            "https://example.com",
+            "c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2",
+            123,
+            ["User-Agent: foobar"],
+            True,
+            id="invalid size",
+        ),
+    ),
+)
+def test_stream_download(
+    monkeypatch, fetch_content_mod, url, sha256, size, headers, raises
+):
+    def mock_urlopen(req, timeout=None, *, cafile=None):
+        assert req._full_url == url
+        assert timeout is not None
+        if headers:
+            assert len(req.headers) == len(headers)
+            for header in headers:
+                k, v = header.split(":")
+                k = k.lower().capitalize().strip()
+                assert k in req.headers
+                assert req.headers[k] == v.strip()
+
+        # create a mock context manager
+        cm = MagicMock()
+        cm.getcode.return_value = 200
+
+        def getheader(field):
+            if field.lower() == "content-length":
+                return size
+
+        # simulates chunking
+        cm.getheader = getheader
+        cm.read.side_effect = [b"foo", b"bar", None]
+        cm.__enter__.return_value = cm
+        return cm
+
+    monkeypatch.setattr(urllib.request, "urlopen", mock_urlopen)
+
+    result = b""
+    try:
+        for chunk in fetch_content_mod.stream_download(url, sha256, size, headers):
+            result += chunk
+        assert result == b"foobar"
+    except fetch_content_mod.IntegrityError:
+        if not raises:
+            raise
+
+
+@pytest.mark.parametrize(
+    "expected,orig,dest,strip_components,add_prefix",
+    [
+        # Archives to repack
+        (True, pathlib.Path("archive"), pathlib.Path("archive.tar.zst"), 0, ""),
+        (True, pathlib.Path("archive.tar"), pathlib.Path("archive.tar.zst"), 0, ""),
+        (True, pathlib.Path("archive.tgz"), pathlib.Path("archive.tar.zst"), 0, ""),
+        (True, pathlib.Path("archive.zip"), pathlib.Path("archive.tar.zst"), 0, ""),
+        (True, pathlib.Path("archive.tar.xz"), pathlib.Path("archive.tar.zst"), 0, ""),
+        (True, pathlib.Path("archive.zst"), pathlib.Path("archive.tar.zst"), 0, ""),
+        # Path is exactly the same
+        (False, pathlib.Path("archive"), pathlib.Path("archive"), 0, ""),
+        (False, pathlib.Path("file.txt"), pathlib.Path("file.txt"), 0, ""),
+        (False, pathlib.Path("archive.tar"), pathlib.Path("archive.tar"), 0, ""),
+        (False, pathlib.Path("archive.tgz"), pathlib.Path("archive.tgz"), 0, ""),
+        (False, pathlib.Path("archive.zip"), pathlib.Path("archive.zip"), 0, ""),
+        (
+            False,
+            pathlib.Path("archive.tar.zst"),
+            pathlib.Path("archive.tar.zst"),
+            0,
+            "",
+        ),
+        (
+            False,
+            pathlib.Path("archive-before.tar.zst"),
+            pathlib.Path("archive-after.tar.zst"),
+            0,
+            "",
+        ),
+        (
+            False,
+            pathlib.Path("before.foo.bar.baz"),
+            pathlib.Path("after.foo.bar.baz"),
+            0,
+            "",
+        ),
+        # Non-default values for strip_components and add_prefix parameters
+        (True, pathlib.Path("archive.tar.zst"), pathlib.Path("archive.tar.zst"), 1, ""),
+        (
+            True,
+            pathlib.Path("archive.tar.zst"),
+            pathlib.Path("archive.tar.zst"),
+            0,
+            "prefix",
+        ),
+        (
+            True,
+            pathlib.Path("archive.tar.zst"),
+            pathlib.Path("archive.tar.zst"),
+            1,
+            "prefix",
+        ),
+        # Real edge cases that should not be repacks
+        (
+            False,
+            pathlib.Path("python-3.8.10-amd64.exe"),
+            pathlib.Path("python.exe"),
+            0,
+            "",
+        ),
+        (
+            False,
+            pathlib.Path("9ee26e91-9b52-44ba-8d30-c0230dd587b2.bin"),
+            pathlib.Path("model.esen.intgemm.alphas.bin"),
+            0,
+            "",
+        ),
+    ],
+)
+def test_should_repack_archive(
+    fetch_content_mod, orig, dest, expected, strip_components, add_prefix
+):
+    assert (
+        fetch_content_mod.should_repack_archive(
+            orig, dest, strip_components, add_prefix
+        )
+        == expected
+    ), f"Failed for orig: {orig}, dest: {dest}, strip_components: {strip_components}, add_prefix: {add_prefix}, expected {expected} but received {not expected}"

{taskcluster_taskgraph-11.2.2 → taskcluster_taskgraph-11.2.4}/test/test_scripts_run_task.py

@@ -1,5 +1,6 @@
 import io
 import os
+import site
 import stat
 import subprocess
 import sys
@@ -85,6 +86,7 @@ def test_install_pip_requirements(
             sys.executable,
             "-mpip",
             "install",
+            "--user",
             "--break-system-packages",
             "--require-hashes",
             "-r",
@@ -105,6 +107,7 @@ def test_install_pip_requirements(
             sys.executable,
             "-mpip",
             "install",
+            "--user",
             "--break-system-packages",
             "--require-hashes",
             "-r",
@@ -137,7 +140,8 @@ def test_install_pip_requirements_with_uv(
             "install",
             "--python",
             sys.executable,
-            "--break-system-packages",
+            "--target",
+            site.getusersitepackages(),
             "--require-hashes",
             "-r",
             str(req),

taskcluster_taskgraph-11.2.2/test/test_scripts_fetch_content.py

@@ -1,96 +0,0 @@
-import os
-import urllib.request
-from importlib.machinery import SourceFileLoader
-from importlib.util import module_from_spec, spec_from_loader
-from unittest.mock import MagicMock
-
-import pytest
-
-import taskgraph
-
-
-@pytest.fixture(scope="module")
-def fetch_content_mod():
-    spec = spec_from_loader(
-        "fetch-content",
-        SourceFileLoader(
-            "fetch-content",
-            os.path.join(
-                os.path.dirname(taskgraph.__file__), "run-task", "fetch-content"
-            ),
-        ),
-    )
-    assert spec
-    assert spec.loader
-    mod = module_from_spec(spec)
-    spec.loader.exec_module(mod)
-    return mod
-
-
-@pytest.mark.parametrize(
-    "url,sha256,size,headers,raises",
-    (
-        pytest.param(
-            "https://example.com",
-            "c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2",
-            6,
-            ["User-Agent: foobar"],
-            False,
-            id="valid",
-        ),
-        pytest.param(
-            "https://example.com",
-            "abcdef",
-            6,
-            ["User-Agent: foobar"],
-            True,
-            id="invalid sha256",
-        ),
-        pytest.param(
-            "https://example.com",
-            "c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2",
-            123,
-            ["User-Agent: foobar"],
-            True,
-            id="invalid size",
-        ),
-    ),
-)
-def test_stream_download(
-    monkeypatch, fetch_content_mod, url, sha256, size, headers, raises
-):
-    def mock_urlopen(req, timeout=None, *, cafile=None):
-        assert req._full_url == url
-        assert timeout is not None
-        if headers:
-            assert len(req.headers) == len(headers)
-            for header in headers:
-                k, v = header.split(":")
-                k = k.lower().capitalize().strip()
-                assert k in req.headers
-                assert req.headers[k] == v.strip()
-
-        # create a mock context manager
-        cm = MagicMock()
-        cm.getcode.return_value = 200
-
-        def getheader(field):
-            if field.lower() == "content-length":
-                return size
-
-        # simulates chunking
-        cm.getheader = getheader
-        cm.read.side_effect = [b"foo", b"bar", None]
-        cm.__enter__.return_value = cm
-        return cm
-
-    monkeypatch.setattr(urllib.request, "urlopen", mock_urlopen)
-
-    result = b""
-    try:
-        for chunk in fetch_content_mod.stream_download(url, sha256, size, headers):
-            result += chunk
-        assert result == b"foobar"
-    except fetch_content_mod.IntegrityError:
-        if not raises:
-            raise