PyPI - taskcluster-taskgraph - Versions diffs - 11.2.1__tar.gz → 11.2.3__tar.gz - Mend

taskcluster-taskgraph 11.2.1tar.gz → 11.2.3tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (246) hide show

{taskcluster_taskgraph-11.2.1 → taskcluster_taskgraph-11.2.3}/.pre-commit-config.yaml RENAMED Viewed

@@ -7,7 +7,7 @@ ci:
   autoupdate_schedule: monthly
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
@@ -15,7 +15,7 @@ repos:
         exclude: template
       - id: check-added-large-files
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.3
+    rev: v0.6.9
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

{taskcluster_taskgraph-11.2.1 → taskcluster_taskgraph-11.2.3}/CHANGELOG.md RENAMED Viewed

@@ -1,5 +1,23 @@
 # Change Log
+## [11.2.3] - 2024-10-16
+### Fixed
+- A couple more regressions with the Decision image
+  - Cleaned up temporary `uv` lockfile owend by root
+  - Added another `--no-cache` flag to a `uv` command
+- `run-task` now installs packages to user site dir when installing requirements with `uv`
+- Fetch tasks using `static-url` will now properly rename non-archive files
+## [11.2.2] - 2024-10-15
+### Fixed
+- Regression in Decision image where `taskgraph` was no longer on the $PATH
+- Decision image no longer contains root owned `uv` cache
+- `uv` now included in the `run-task` image
 ## [11.2.1] - 2024-10-03
 ### Fixed

{taskcluster_taskgraph-11.2.1 → taskcluster_taskgraph-11.2.3}/PKG-INFO RENAMED Viewed

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: taskcluster-taskgraph
-Version: 11.2.1
+Version: 11.2.3
 Summary: Build taskcluster taskgraphs
 Project-URL: Repository, https://github.com/taskcluster/taskgraph
 Project-URL: Issues, https://github.com/taskcluster/taskgraph/issues
@@ -14,6 +14,7 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Software Development
 Requires-Python: >=3.8
 Requires-Dist: appdirs>=1.4

{taskcluster_taskgraph-11.2.1 → taskcluster_taskgraph-11.2.3}/pyproject.toml RENAMED Viewed

@@ -15,6 +15,7 @@ classifiers = [
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
   "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
   "Topic :: Software Development",
 ]
 requires-python = ">=3.8"

{taskcluster_taskgraph-11.2.1 → taskcluster_taskgraph-11.2.3}/src/taskgraph/__init__.py RENAMED Viewed

@@ -2,7 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
-__version__ = "11.2.1"
+__version__ = "11.2.3"
 # Maximum number of dependencies a single task can have
 # https://docs.taskcluster.net/docs/reference/platform/queue/api#createTask

{taskcluster_taskgraph-11.2.1 → taskcluster_taskgraph-11.2.3}/src/taskgraph/run-task/fetch-content RENAMED Viewed

@@ -435,11 +435,49 @@ def extract_archive(path, dest_dir):
     log("%s extracted in %.3fs" % (path, time.time() - t0))
+def should_repack_archive(orig: pathlib.Path, dest: pathlib.Path, strip_components=0, add_prefix="") -> bool:
+    """
+    Determines whether we should attempt to repack an archive based on the naming conventions used
+    in the original file path, the destination file path, and any modifications such as stripping
+    components or adding a prefix.
+    """
+    if strip_components or add_prefix:
+        # If strip_components or add_prefix is specified, we should always repack.
+        return True
+    if orig.suffixes == dest.suffixes:
+        # If all suffixes are exactly the same, then a rename will suffice.
+        return False
+    if dest.suffixes[-2:] == [".tar", ".zst"]:
+        # If the destination is a ".tar.zst" file then we will always try to repack it ourselves.
+        return True
+    if orig.suffix == dest.suffix:
+        # If the final suffix is the same, this is likely just a rename.
+        #
+        # This may be a case where multiple suffixes were detected unnecessarily due to the file name
+        # containing a semantic version. For example the file "python-3.8.10-amd64.exe" would be detected
+        # to have three suffixes [".8", ".10-amd64", ".exe"]. Changing this to "python.exe" should be a
+        # simple rename, rather than a repack.
+        #
+        # In this case, we will try to determine if the original path has a supported archive suffix.
+        # If the original path is detected to be an archive, we will try to repack, otherwise rename.
+        return archive_type(orig) is not None
+    # Otherwise, if the paths aren't the same, assume it's an archive and try to repack.
+    #
+    # It would be best to fail early if the repack fails than to fail during a test because a renamed
+    # file was incorrect type after all.
+    return True
 def repack_archive(
     orig: pathlib.Path, dest: pathlib.Path, strip_components=0, prefix=""
 ):
     assert orig != dest
-    log("Repacking as %s" % dest)
+    log(f"Repacking {orig} as {dest}")
     orig_typ, ifh = open_stream(orig)
     typ = archive_type(dest)
     if not typ:
@@ -534,6 +572,9 @@ def repack_archive(
                 # We only change compression here. The tar stream is unchanged.
                 ctx.copy_stream(ifh, fh)
+        else:
+            raise Exception(f"Attempt to repack an archive of unsupported type {orig_typ}")
 def fetch_and_extract(url, dest_dir, extract=True, sha256=None, size=None):
     """Fetch a URL and extract it to a destination path.
@@ -773,8 +814,13 @@ def command_static_url(args):
         if gpg_sig_url:
             gpg_verify_path(dl_dest, gpg_key, gpg_signature)
-        if dl_dest != dest or args.strip_components or args.add_prefix:
-            repack_archive(dl_dest, dest, args.strip_components, args.add_prefix)
+        if dl_dest != dest:
+            if should_repack_archive(dl_dest, dest, args.strip_components, args.add_prefix):
+                repack_archive(dl_dest, dest, args.strip_components, args.add_prefix)
+            else:
+                log(f"Renaming {dl_dest} to {dest}")
+                dl_dest.rename(dest)
     except Exception:
         try:
             dl_dest.unlink()
@@ -783,7 +829,7 @@ def command_static_url(args):
         raise
-    if dl_dest != dest:
+    if dl_dest != dest and dl_dest.exists():
         log("Removing %s" % dl_dest)
         dl_dest.unlink()

{taskcluster_taskgraph-11.2.1 → taskcluster_taskgraph-11.2.3}/src/taskgraph/run-task/run-task RENAMED Viewed

@@ -1012,9 +1012,10 @@ def install_pip_requirements(repositories):
     # TODO: Stop using system Python (#381)
     if shutil.which("uv"):
-        cmd = ["uv", "pip", "install", "--python", sys.executable, "--break-system-packages"]
+        user_site_dir = subprocess.run([sys.executable, "-msite", "--user-site"], capture_output=True, text=True).stdout.strip()
+        cmd = ["uv", "pip", "install", "--python", sys.executable, "--target", user_site_dir]
     else:
-        cmd = [sys.executable, "-mpip", "install", "--break-system-packages"]
+        cmd = [sys.executable, "-mpip", "install", "--user", "--break-system-packages"]
     if os.environ.get("PIP_DISABLE_REQUIRE_HASHES") != "1":
         cmd.append("--require-hashes")

{taskcluster_taskgraph-11.2.1 → taskcluster_taskgraph-11.2.3}/taskcluster/docker/decision/Dockerfile RENAMED Viewed

@@ -1,9 +1,6 @@
 FROM $DOCKER_IMAGE_PARENT
 LABEL maintainer="Release Engineering <release+taskgraph@mozilla.com>"
-# Keep uv version in sync with python image
-COPY --from=ghcr.io/astral-sh/uv:0.4.9 /uv /bin/uv
 ENV UV_PROJECT_ENVIRONMENT=/setup/taskgraph
 # %include src

{taskcluster_taskgraph-11.2.1 → taskcluster_taskgraph-11.2.3}/taskcluster/docker/decision/system-setup.sh RENAMED Viewed

@@ -9,11 +9,13 @@ apt-get install -y --force-yes --no-install-recommends \
     python3-pip
 pushd /setup/taskgraph
-uv export --no-dev > /setup/requirements.txt
-uv pip install --system --break-system-packages -r /setup/requirements.txt
+uv export --no-cache --no-dev > /setup/requirements.txt
+uv pip install --no-cache --system --break-system-packages -r /setup/requirements.txt
+uv pip install --no-cache --system --break-system-packages --no-deps .
 popd
 apt-get clean
 apt-get autoclean
 rm -rf /var/lib/apt/lists/
 rm -rf /setup
+rm /tmp/uv-*.lock

{taskcluster_taskgraph-11.2.1 → taskcluster_taskgraph-11.2.3}/taskcluster/docker/python/Dockerfile RENAMED Viewed

@@ -28,8 +28,8 @@ ENV SHELL=/bin/bash \
     HOME=/builds/worker \
     PATH=/builds/worker/.local/bin:$PATH
-# Keep uv version in sync with decision image
-COPY --from=ghcr.io/astral-sh/uv:0.4.9 /uv /bin/uv
+# %ARG UV_VERSION
+COPY --from=ghcr.io/astral-sh/uv:$UV_VERSION /uv /bin/uv
 # %ARG PYTHON_VERSIONS
 RUN uv python install $PYTHON_VERSIONS

{taskcluster_taskgraph-11.2.1 → taskcluster_taskgraph-11.2.3}/taskcluster/docker/run-task/Dockerfile RENAMED Viewed

@@ -16,5 +16,8 @@ ENV PATH=/builds/worker/bin:$PATH \
     SHELL=/bin/bash \
     HOME=/builds/worker
+# %ARG UV_VERSION
+COPY --from=ghcr.io/astral-sh/uv:$UV_VERSION /uv /bin/uv
 # Set a default command useful for debugging
 CMD ["/bin/bash", "--login"]

{taskcluster_taskgraph-11.2.1 → taskcluster_taskgraph-11.2.3}/taskcluster/kinds/docker-image/kind.yml RENAMED Viewed

@@ -10,6 +10,10 @@ transforms:
     - taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
+task-defaults:
+    args:
+        UV_VERSION: 0.4.9
 # make a task for each docker-image we might want.  For the moment, since we
 # write artifacts for each, these are whitelisted, but ideally that will change
 # (to use subdirectory clones of the proper directory), at which point we can
@@ -26,7 +30,7 @@ tasks:
     python:
         symbol: I(py)
         args:
-            PYTHON_VERSIONS: "3.12 3.11 3.10 3.9 3.8"
+            PYTHON_VERSIONS: "3.13 3.12 3.11 3.10 3.9 3.8"
     run-task:
         symbol: I(rt)
     skopeo:

{taskcluster_taskgraph-11.2.1 → taskcluster_taskgraph-11.2.3}/taskcluster/kinds/test/kind.yml RENAMED Viewed

@@ -40,7 +40,7 @@ tasks:
         description: "Run `unit tests` to validate the latest changes"
         attributes:
             artifact_prefix: public
-        python-versions: ["3.12", "3.11", "3.10", "3.9", "3.8"]
+        python-versions: ["3.13", "3.12", "3.11", "3.10", "3.9", "3.8"]
         run:
             command: >-
                 uv run coverage run --data-file $MOZ_ARTIFACT_DIR/coverage --context=$PYTHON_VERSION -m pytest -vv

taskcluster_taskgraph-11.2.3/test/test_scripts_fetch_content.py ADDED Viewed

@@ -0,0 +1,178 @@
+import os
+import pathlib
+import urllib.request
+from importlib.machinery import SourceFileLoader
+from importlib.util import module_from_spec, spec_from_loader
+from unittest.mock import MagicMock
+import pytest
+import taskgraph
+@pytest.fixture(scope="module")
+def fetch_content_mod():
+    spec = spec_from_loader(
+        "fetch-content",
+        SourceFileLoader(
+            "fetch-content",
+            os.path.join(
+                os.path.dirname(taskgraph.__file__), "run-task", "fetch-content"
+            ),
+        ),
+    )
+    assert spec
+    assert spec.loader
+    mod = module_from_spec(spec)
+    spec.loader.exec_module(mod)
+    return mod
+@pytest.mark.parametrize(
+    "url,sha256,size,headers,raises",
+    (
+        pytest.param(
+            "https://example.com",
+            "c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2",
+            6,
+            ["User-Agent: foobar"],
+            False,
+            id="valid",
+        ),
+        pytest.param(
+            "https://example.com",
+            "abcdef",
+            6,
+            ["User-Agent: foobar"],
+            True,
+            id="invalid sha256",
+        ),
+        pytest.param(
+            "https://example.com",
+            "c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2",
+            123,
+            ["User-Agent: foobar"],
+            True,
+            id="invalid size",
+        ),
+    ),
+)
+def test_stream_download(
+    monkeypatch, fetch_content_mod, url, sha256, size, headers, raises
+):
+    def mock_urlopen(req, timeout=None, *, cafile=None):
+        assert req._full_url == url
+        assert timeout is not None
+        if headers:
+            assert len(req.headers) == len(headers)
+            for header in headers:
+                k, v = header.split(":")
+                k = k.lower().capitalize().strip()
+                assert k in req.headers
+                assert req.headers[k] == v.strip()
+        # create a mock context manager
+        cm = MagicMock()
+        cm.getcode.return_value = 200
+        def getheader(field):
+            if field.lower() == "content-length":
+                return size
+        # simulates chunking
+        cm.getheader = getheader
+        cm.read.side_effect = [b"foo", b"bar", None]
+        cm.__enter__.return_value = cm
+        return cm
+    monkeypatch.setattr(urllib.request, "urlopen", mock_urlopen)
+    result = b""
+    try:
+        for chunk in fetch_content_mod.stream_download(url, sha256, size, headers):
+            result += chunk
+        assert result == b"foobar"
+    except fetch_content_mod.IntegrityError:
+        if not raises:
+            raise
+@pytest.mark.parametrize(
+    "expected,orig,dest,strip_components,add_prefix",
+    [
+        # Archives to repack
+        (True, pathlib.Path("archive"), pathlib.Path("archive.tar.zst"), 0, ""),
+        (True, pathlib.Path("archive.tar"), pathlib.Path("archive.tar.zst"), 0, ""),
+        (True, pathlib.Path("archive.tgz"), pathlib.Path("archive.tar.zst"), 0, ""),
+        (True, pathlib.Path("archive.zip"), pathlib.Path("archive.tar.zst"), 0, ""),
+        (True, pathlib.Path("archive.tar.xz"), pathlib.Path("archive.tar.zst"), 0, ""),
+        (True, pathlib.Path("archive.zst"), pathlib.Path("archive.tar.zst"), 0, ""),
+        # Path is exactly the same
+        (False, pathlib.Path("archive"), pathlib.Path("archive"), 0, ""),
+        (False, pathlib.Path("file.txt"), pathlib.Path("file.txt"), 0, ""),
+        (False, pathlib.Path("archive.tar"), pathlib.Path("archive.tar"), 0, ""),
+        (False, pathlib.Path("archive.tgz"), pathlib.Path("archive.tgz"), 0, ""),
+        (False, pathlib.Path("archive.zip"), pathlib.Path("archive.zip"), 0, ""),
+        (
+            False,
+            pathlib.Path("archive.tar.zst"),
+            pathlib.Path("archive.tar.zst"),
+            0,
+            "",
+        ),
+        (
+            False,
+            pathlib.Path("archive-before.tar.zst"),
+            pathlib.Path("archive-after.tar.zst"),
+            0,
+            "",
+        ),
+        (
+            False,
+            pathlib.Path("before.foo.bar.baz"),
+            pathlib.Path("after.foo.bar.baz"),
+            0,
+            "",
+        ),
+        # Non-default values for strip_components and add_prefix parameters
+        (True, pathlib.Path("archive.tar.zst"), pathlib.Path("archive.tar.zst"), 1, ""),
+        (
+            True,
+            pathlib.Path("archive.tar.zst"),
+            pathlib.Path("archive.tar.zst"),
+            0,
+            "prefix",
+        ),
+        (
+            True,
+            pathlib.Path("archive.tar.zst"),
+            pathlib.Path("archive.tar.zst"),
+            1,
+            "prefix",
+        ),
+        # Real edge cases that should not be repacks
+        (
+            False,
+            pathlib.Path("python-3.8.10-amd64.exe"),
+            pathlib.Path("python.exe"),
+            0,
+            "",
+        ),
+        (
+            False,
+            pathlib.Path("9ee26e91-9b52-44ba-8d30-c0230dd587b2.bin"),
+            pathlib.Path("model.esen.intgemm.alphas.bin"),
+            0,
+            "",
+        ),
+    ],
+)
+def test_should_repack_archive(
+    fetch_content_mod, orig, dest, expected, strip_components, add_prefix
+):
+    assert (
+        fetch_content_mod.should_repack_archive(
+            orig, dest, strip_components, add_prefix
+        )
+        == expected
+    ), f"Failed for orig: {orig}, dest: {dest}, strip_components: {strip_components}, add_prefix: {add_prefix}, expected {expected} but received {not expected}"

{taskcluster_taskgraph-11.2.1 → taskcluster_taskgraph-11.2.3}/test/test_scripts_run_task.py RENAMED Viewed

@@ -1,5 +1,6 @@
 import io
 import os
+import site
 import stat
 import subprocess
 import sys
@@ -85,6 +86,7 @@ def test_install_pip_requirements(
             sys.executable,
             "-mpip",
             "install",
+            "--user",
             "--break-system-packages",
             "--require-hashes",
             "-r",
@@ -105,6 +107,7 @@ def test_install_pip_requirements(
             sys.executable,
             "-mpip",
             "install",
+            "--user",
             "--break-system-packages",
             "--require-hashes",
             "-r",
@@ -137,7 +140,8 @@ def test_install_pip_requirements_with_uv(
             "install",
             "--python",
             sys.executable,
-            "--break-system-packages",
+            "--target",
+            site.getusersitepackages(),
             "--require-hashes",
             "-r",
             str(req),

taskcluster_taskgraph-11.2.1/setup.py DELETED Viewed

@@ -1,52 +0,0 @@
-import os
-from distutils.util import convert_path
-from setuptools import find_packages, setup
-project_dir = os.path.abspath(os.path.dirname(__file__))
-namespace = {}
-version_file = convert_path("src/taskgraph/__init__.py")
-with open(version_file) as fh:
-    exec(fh.read(), namespace)
-with open(os.path.join(project_dir, "requirements/base.in")) as fp:
-    requirements = fp.read().splitlines()
-with open(os.path.join(project_dir, "README.rst")) as fh:
-    long_description = fh.read()
-setup(
-    name="taskcluster-taskgraph",
-    version=namespace["__version__"],
-    description="Build taskcluster taskgraphs",
-    long_description=long_description,
-    url="https://github.com/taskcluster/taskgraph",
-    packages=find_packages("src"),
-    package_dir={"": "src"},
-    install_requires=requirements,
-    extras_require={
-        "load-image": ["zstandard"],
-    },
-    classifiers=[
-        "Development Status :: 5 - Production/Stable",
-        "Environment :: Console",
-        "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
-        "Programming Language :: Python :: 3.10",
-        "Programming Language :: Python :: 3.11",
-        "Programming Language :: Python :: 3.12",
-        "Topic :: Software Development",
-    ],
-    entry_points={"console_scripts": ["taskgraph = taskgraph.main:main"]},
-    package_data={
-        "taskgraph": [
-            "run-task/run-task",
-            "run-task/fetch-content",
-            "run-task/hgrc",
-            "run-task/robustcheckout.py",
-        ],
-        "taskgraph.test": ["automationrelevance.json"],
-    },
-)

taskcluster_taskgraph-11.2.1/test/test_scripts_fetch_content.py DELETED Viewed

@@ -1,96 +0,0 @@
-import os
-import urllib.request
-from importlib.machinery import SourceFileLoader
-from importlib.util import module_from_spec, spec_from_loader
-from unittest.mock import MagicMock
-import pytest
-import taskgraph
-@pytest.fixture(scope="module")
-def fetch_content_mod():
-    spec = spec_from_loader(
-        "fetch-content",
-        SourceFileLoader(
-            "fetch-content",
-            os.path.join(
-                os.path.dirname(taskgraph.__file__), "run-task", "fetch-content"
-            ),
-        ),
-    )
-    assert spec
-    assert spec.loader
-    mod = module_from_spec(spec)
-    spec.loader.exec_module(mod)
-    return mod
-@pytest.mark.parametrize(
-    "url,sha256,size,headers,raises",
-    (
-        pytest.param(
-            "https://example.com",
-            "c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2",
-            6,
-            ["User-Agent: foobar"],
-            False,
-            id="valid",
-        ),
-        pytest.param(
-            "https://example.com",
-            "abcdef",
-            6,
-            ["User-Agent: foobar"],
-            True,
-            id="invalid sha256",
-        ),
-        pytest.param(
-            "https://example.com",
-            "c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2",
-            123,
-            ["User-Agent: foobar"],
-            True,
-            id="invalid size",
-        ),
-    ),
-)
-def test_stream_download(
-    monkeypatch, fetch_content_mod, url, sha256, size, headers, raises
-):
-    def mock_urlopen(req, timeout=None, *, cafile=None):
-        assert req._full_url == url
-        assert timeout is not None
-        if headers:
-            assert len(req.headers) == len(headers)
-            for header in headers:
-                k, v = header.split(":")
-                k = k.lower().capitalize().strip()
-                assert k in req.headers
-                assert req.headers[k] == v.strip()
-        # create a mock context manager
-        cm = MagicMock()
-        cm.getcode.return_value = 200
-        def getheader(field):
-            if field.lower() == "content-length":
-                return size
-        # simulates chunking
-        cm.getheader = getheader
-        cm.read.side_effect = [b"foo", b"bar", None]
-        cm.__enter__.return_value = cm
-        return cm
-    monkeypatch.setattr(urllib.request, "urlopen", mock_urlopen)
-    result = b""
-    try:
-        for chunk in fetch_content_mod.stream_download(url, sha256, size, headers):
-            result += chunk
-        assert result == b"foobar"
-    except fetch_content_mod.IntegrityError:
-        if not raises:
-            raise