taskcluster-taskgraph 16.2.0__tar.gz → 16.2.1__tar.gz

{taskcluster_taskgraph-16.2.0 → taskcluster_taskgraph-16.2.1}/CHANGELOG.md

@@ -1,4 +1,13 @@
 # Change Log
+
+## [16.2.1] - 2025-10-14
+
+### Fixed
+
+- `run-task` script now cleans up zombie processes
+- Fixed regression to `taskgraph.util.taskcluster.list_artifacts`
+- Fixed regression to `taskgraph.util.taskcluster.get_artifact`
+
 ## [16.2.0] - 2025-09-26
 
 ### Added

{taskcluster_taskgraph-16.2.0 → taskcluster_taskgraph-16.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: taskcluster-taskgraph
-Version: 16.2.0
+Version: 16.2.1
 Summary: Build taskcluster taskgraphs
 Project-URL: Repository, https://github.com/taskcluster/taskgraph
 Project-URL: Issues, https://github.com/taskcluster/taskgraph/issues
@@ -26,7 +26,7 @@ Requires-Dist: redo>=2.0
 Requires-Dist: requests>=2.25
 Requires-Dist: slugid>=2.0
 Requires-Dist: taskcluster-urls>=11.0
-Requires-Dist: taskcluster>=55.0
+Requires-Dist: taskcluster>=91.0
 Requires-Dist: voluptuous>=0.12.1
 Provides-Extra: load-image
 Requires-Dist: zstandard; extra == 'load-image'

{taskcluster_taskgraph-16.2.0 → taskcluster_taskgraph-16.2.1}/pyproject.toml

@@ -1,7 +1,7 @@
 ### Project
 [project]
 name = "taskcluster-taskgraph"
-version = "16.2.0"
+version = "16.2.1"
 description = "Build taskcluster taskgraphs"
 readme = "README.rst"
 authors = [
@@ -29,7 +29,7 @@ dependencies = [
   "redo>=2.0",
   "requests>=2.25",
   "slugid>=2.0",
-  "taskcluster>=55.0",
+  "taskcluster>=91.0",
   "taskcluster-urls>=11.0",
   "voluptuous>=0.12.1",
 ]

{taskcluster_taskgraph-16.2.0 → taskcluster_taskgraph-16.2.1}/src/taskgraph/run-task/run-task

@@ -28,6 +28,7 @@ import socket
 import stat
 import subprocess
 import sys
+import threading
 import time
 import urllib.error
 import urllib.request
@@ -121,6 +122,17 @@ def print_line(prefix, m):
     sys.stdout.buffer.flush()
 
 
+def reap_zombies(main_subprocess):
+    """Wait for main_subprocess to exit, while awaiting any other child processes"""
+    while main_subprocess.poll() is None:
+        with main_subprocess._waitpid_lock:
+            if main_subprocess.returncode is not None:
+                return
+            pid, status = os.wait()
+            if pid == main_subprocess.pid:
+                main_subprocess._handle_exitstatus(status)
+
+
 def _call_windows_retry(func, args=(), retry_max=5, retry_delay=0.5):
     """
     It's possible to see spurious errors on Windows due to various things
@@ -278,6 +290,13 @@ def run_command(prefix, args, *, extra_env=None, cwd=None):
 
     stdout = io.TextIOWrapper(p.stdout, encoding="latin1")
 
+    if os.getpid() == 1:
+        # in docker we're init, so we get to adopt unawaited zombies
+        reaper_thread = threading.Thread(target=reap_zombies, args=(p,))
+        reaper_thread.start()
+    else:
+        reaper_thread = None
+
     while True:
         data = stdout.readline().encode("latin1")
 
@@ -286,6 +305,8 @@ def run_command(prefix, args, *, extra_env=None, cwd=None):
 
         print_line(prefix, data)
 
+    if reaper_thread:
+        reaper_thread.join()
     return p.wait()
 
 

{taskcluster_taskgraph-16.2.0 → taskcluster_taskgraph-16.2.1}/src/taskgraph/util/taskcluster.py

@@ -175,9 +175,8 @@ def get_artifact(task_id, path):
 
 def list_artifacts(task_id):
     queue = get_taskcluster_client("queue")
-    task = queue.task(task_id)
-    if task:
-        return task["artifacts"]
+    response = queue.listLatestArtifacts(task_id)
+    return response["artifacts"]
 
 
 def get_artifact_prefix(task):

taskcluster_taskgraph-16.2.1/test/test_util_archive.py

@@ -0,0 +1,181 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import hashlib
+import io
+import stat
+import tarfile
+
+import pytest
+
+from taskgraph.util.archive import (
+    DEFAULT_MTIME,
+    create_tar_from_files,
+    create_tar_gz_from_files,
+)
+
+MODE_STANDARD = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH
+
+
+def file_hash(path):
+    h = hashlib.sha1()
+    with open(path, "rb") as fh:
+        while True:
+            data = fh.read(8192)
+            if not data:
+                break
+            h.update(data)
+
+    return h.hexdigest()
+
+
+@pytest.fixture
+def create_files(tmp_path):
+    def inner():
+        files = {}
+        for i in range(10):
+            p = tmp_path / f"file{i:02d}"
+            p.write_bytes(b"file%02d" % i)
+            # Need to set permissions or umask may influence testing.
+            p.chmod(MODE_STANDARD)
+            files[f"file{i:02d}"] = str(p)
+
+        for i in range(10):
+            files[f"file{i + 10:02d}"] = io.BytesIO(b"file%02d" % (i + 10))
+
+        return files
+
+    return inner
+
+
+def verify_basic_tarfile(tf):
+    assert len(tf.getmembers()) == 20
+
+    names = [f"file{i:02d}" for i in range(20)]
+    assert tf.getnames() == names
+
+    for ti in tf.getmembers():
+        assert ti.uid == 0
+        assert ti.gid == 0
+        assert ti.uname == ""
+        assert ti.gname == ""
+        assert ti.mode == MODE_STANDARD
+        assert ti.mtime == DEFAULT_MTIME
+
+
+@pytest.mark.xfail(reason="ValueError is not thrown despite being provided directory.")
+def test_dirs_refused(tmp_path):
+    tp = tmp_path / "test.tar"
+    with open(tp, "wb") as fh:
+        with pytest.raises(ValueError, match="not a regular"):
+            create_tar_from_files(fh, {"test": str(tmp_path)})
+
+
+def test_setuid_setgid_refused(tmp_path):
+    uid = tmp_path / "setuid"
+    uid.touch()
+    uid.chmod(MODE_STANDARD | stat.S_ISUID)
+
+    gid = tmp_path / "setgid"
+    gid.touch()
+    gid.chmod(MODE_STANDARD | stat.S_ISGID)
+
+    tp = tmp_path / "test.tar"
+    with open(tp, "wb") as fh:
+        with pytest.raises(ValueError, match="cannot add file with setuid"):
+            create_tar_from_files(fh, {"test": str(uid)})
+        with pytest.raises(ValueError, match="cannot add file with setuid"):
+            create_tar_from_files(fh, {"test": str(gid)})
+
+
+def test_create_tar_basic(tmp_path, create_files):
+    files = create_files()
+
+    tp = tmp_path / "test.tar"
+    with open(tp, "wb") as fh:
+        create_tar_from_files(fh, files)
+
+    # Output should be deterministic.
+    assert file_hash(tp) == "01cd314e277f060e98c7de6c8ea57f96b3a2065c"
+
+    with tarfile.open(tp, "r") as tf:
+        verify_basic_tarfile(tf)
+
+
+def test_executable_preserved(tmp_path):
+    p = tmp_path / "exec"
+    p.write_bytes(b"#!/bin/bash\n")
+    p.chmod(MODE_STANDARD | stat.S_IXUSR)
+
+    tp = tmp_path / "test.tar"
+    with open(tp, "wb") as fh:
+        create_tar_from_files(fh, {"exec": str(p)})
+
+    # Test determinism by creating the same file again
+    tp2 = tmp_path / "test2.tar"
+    with open(tp2, "wb") as fh:
+        create_tar_from_files(fh, {"exec": str(p)})
+
+    assert file_hash(str(tp)) == file_hash(str(tp2))
+
+    # Verify executable permissions are preserved in tar
+    with tarfile.open(tp, "r") as tf:
+        m = tf.getmember("exec")
+        assert m.mode == MODE_STANDARD | stat.S_IXUSR
+
+        # Verify file content is correct
+        extracted_content = tf.extractfile(m).read()
+        assert extracted_content == b"#!/bin/bash\n"
+
+
+def test_create_tar_gz_basic(tmp_path, create_files):
+    gp = tmp_path / "test.tar.gz"
+    with open(gp, "wb") as fh:
+        create_tar_gz_from_files(fh, create_files())
+
+    # Test determinism by creating the same file again with fresh BytesIO objects
+    gp2 = tmp_path / "test2.tar.gz"
+    with open(gp2, "wb") as fh:
+        create_tar_gz_from_files(fh, create_files())
+
+    assert file_hash(str(gp)) == file_hash(str(gp2))
+
+    # Create uncompressed version for size comparison
+    tp = tmp_path / "test.tar"
+    with open(tp, "wb") as fh:
+        create_tar_from_files(fh, create_files())
+    uncompressed_size = tp.stat().st_size
+    compressed_size = gp.stat().st_size
+
+    # Compressed should be smaller than uncompressed
+    assert compressed_size < uncompressed_size
+
+    # Verify the contents are correct
+    with tarfile.open(gp, "r:gz") as tf:
+        verify_basic_tarfile(tf)
+
+
+def test_tar_gz_name(tmp_path, create_files):
+    gp = tmp_path / "test.tar.gz"
+    with open(gp, "wb") as fh:
+        create_tar_gz_from_files(fh, create_files(), filename="foobar")
+
+    # Test determinism by creating the same file again with fresh BytesIO objects
+    gp2 = tmp_path / "test2.tar.gz"
+    with open(gp2, "wb") as fh:
+        create_tar_gz_from_files(fh, create_files(), filename="foobar")
+
+    assert file_hash(str(gp)) == file_hash(str(gp2))
+
+    # Create version without filename for comparison
+    gp_no_name = tmp_path / "test_no_name.tar.gz"
+    with open(gp_no_name, "wb") as fh:
+        create_tar_gz_from_files(fh, create_files())
+
+    # Files should be different (different filename in gzip header)
+    assert file_hash(str(gp)) != file_hash(str(gp_no_name))
+
+    # Verify the contents are correct
+    with tarfile.open(gp, "r:gz") as tf:
+        verify_basic_tarfile(tf)

{taskcluster_taskgraph-16.2.0 → taskcluster_taskgraph-16.2.1}/test/test_util_taskcluster.py

@@ -110,16 +110,25 @@ def test_get_artifact(responses, root_url):
 
     # Test text artifact
     responses.get(
-        f"{root_url}/api/queue/v1/task/{tid}/artifacts/artifact.txt",
+        "http://foo.bar/artifact.txt",
         body=b"foobar",
     )
+    responses.get(
+        f"{root_url}/api/queue/v1/task/{tid}/artifacts/artifact.txt",
+        body=b'{"type": "s3", "url": "http://foo.bar/artifact.txt"}',
+        status=303,
+        headers={"Location": "http://foo.bar/artifact.txt"},
+    )
     raw = tc.get_artifact(tid, "artifact.txt")
     assert raw.read() == b"foobar"
 
     # Test JSON artifact
+    responses.get("http://foo.bar/artifact.json", json={"foo": "bar"})
     responses.get(
         f"{root_url}/api/queue/v1/task/{tid}/artifacts/artifact.json",
-        json={"foo": "bar"},
+        body=b'{"type": "s3", "url": "http://foo.bar/artifact.json"}',
+        status=303,
+        headers={"Location": "http://foo.bar/artifact.json"},
     )
     result = tc.get_artifact(tid, "artifact.json")
     assert result == {"foo": "bar"}
@@ -127,9 +136,15 @@ def test_get_artifact(responses, root_url):
     # Test YAML artifact
     expected_result = {"foo": b"\xe2\x81\x83".decode()}
     responses.get(
-        f"{root_url}/api/queue/v1/task/{tid}/artifacts/artifact.yml",
+        "http://foo.bar/artifact.yml",
         body=b'foo: "\xe2\x81\x83"',
     )
+    responses.get(
+        f"{root_url}/api/queue/v1/task/{tid}/artifacts/artifact.yml",
+        body=b'{"type": "s3", "url": "http://foo.bar/artifact.yml"}',
+        status=303,
+        headers={"Location": "http://foo.bar/artifact.yml"},
+    )
     result = tc.get_artifact(tid, "artifact.yml")
     assert result == expected_result
 
@@ -139,7 +154,7 @@ def test_list_artifact(responses, root_url):
     tc.get_taskcluster_client.cache_clear()
 
     responses.get(
-        f"{root_url}/api/queue/v1/task/{tid}",
+        f"{root_url}/api/queue/v1/task/{tid}/artifacts",
         json={"artifacts": ["file1.txt", "file2.json"]},
     )
 

{taskcluster_taskgraph-16.2.0 → taskcluster_taskgraph-16.2.1}/uv.lock

@@ -1,5 +1,5 @@
 version = 1
-revision = 2
+revision = 3
 requires-python = ">=3.8"
 resolution-markers = [
     "python_full_version >= '3.11'",
@@ -998,7 +998,7 @@ version = "1.3.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "typing-extensions", version = "4.13.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" },
-    { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9' and python_full_version < '3.13'" },
+    { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9' and python_full_version < '3.11'" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" }
 wheels = [
@@ -3320,7 +3320,7 @@ wheels = [
 
 [[package]]
 name = "taskcluster"
-version = "90.0.5"
+version = "91.0.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "aiohttp", version = "3.10.11", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" },
@@ -3333,14 +3333,14 @@ dependencies = [
     { name = "slugid" },
     { name = "taskcluster-urls" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/7a/9a/f28142083e024bce75ebbc6ac5de0b3ce8a0b5989c83493a172718fde986/taskcluster-90.0.5.tar.gz", hash = "sha256:3529b62d05b3a869669e7c128cf46849de6641d50d598e8ce81cd4ed7ba756f6", size = 129199, upload-time = "2025-10-08T13:55:23.916Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/64/b9/96fc43251b42348b2e422a45bd7c6795311e32e322b141e97be83ba28e64/taskcluster-91.0.0.tar.gz", hash = "sha256:2e21feac068750503a32f4ca4273c904d8455d6b1db8e7295c3aa119af65abf4", size = 129244, upload-time = "2025-10-14T17:13:33.602Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/31/4c/7c4473080e9e74666e49fa3477f601d7188bb6bd1db68ec5e4c875829505/taskcluster-90.0.5-py3-none-any.whl", hash = "sha256:d4ca28ffa391fbd2f6553b5c390a606a01748ba12791ad90675efd65b734a3c7", size = 147341, upload-time = "2025-10-08T13:55:21.866Z" },
+    { url = "https://files.pythonhosted.org/packages/2c/16/92be916f30d90ae41ee406a843f42bbedbe95109d61cb188187ab72bad75/taskcluster-91.0.0-py3-none-any.whl", hash = "sha256:c6301a5f730bb65feb2811b1190bef8124af0aeddd931eb877e066202348f5bd", size = 147340, upload-time = "2025-10-14T17:13:31.658Z" },
 ]
 
 [[package]]
 name = "taskcluster-taskgraph"
-version = "16.2"
+version = "16.2.1"
 source = { editable = "." }
 dependencies = [
     { name = "appdirs" },
@@ -3411,7 +3411,7 @@ requires-dist = [
     { name = "redo", specifier = ">=2.0" },
     { name = "requests", specifier = ">=2.25" },
     { name = "slugid", specifier = ">=2.0" },
-    { name = "taskcluster", specifier = ">=55.0" },
+    { name = "taskcluster", specifier = ">=91.0" },
     { name = "taskcluster-urls", specifier = ">=11.0" },
     { name = "voluptuous", specifier = ">=0.12.1" },
     { name = "zstandard", marker = "extra == 'load-image'" },

taskcluster_taskgraph-16.2.0/test/test_util_archive.py

@@ -1,174 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import hashlib
-import io
-import os
-import shutil
-import stat
-import tarfile
-import tempfile
-import unittest
-
-import pytest
-
-from taskgraph.util.archive import (
-    DEFAULT_MTIME,
-    create_tar_from_files,
-    create_tar_gz_from_files,
-)
-
-MODE_STANDARD = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH
-
-
-def file_hash(path):
-    h = hashlib.sha1()
-    with open(path, "rb") as fh:
-        while True:
-            data = fh.read(8192)
-            if not data:
-                break
-            h.update(data)
-
-    return h.hexdigest()
-
-
-class TestArchive(unittest.TestCase):
-    def _create_files(self, root):
-        files = {}
-        for i in range(10):
-            p = os.path.join(root, f"file{i:02d}")
-            with open(p, "wb") as fh:
-                fh.write(b"file%02d" % i)
-            # Need to set permissions or umask may influence testing.
-            os.chmod(p, MODE_STANDARD)
-            files[f"file{i:02d}"] = p
-
-        for i in range(10):
-            files[f"file{i + 10:02d}"] = io.BytesIO(b"file%02d" % (i + 10))
-
-        return files
-
-    def _verify_basic_tarfile(self, tf):
-        self.assertEqual(len(tf.getmembers()), 20)
-
-        names = [f"file{i:02d}" for i in range(20)]
-        self.assertEqual(tf.getnames(), names)
-
-        for ti in tf.getmembers():
-            self.assertEqual(ti.uid, 0)
-            self.assertEqual(ti.gid, 0)
-            self.assertEqual(ti.uname, "")
-            self.assertEqual(ti.gname, "")
-            self.assertEqual(ti.mode, MODE_STANDARD)
-            self.assertEqual(ti.mtime, DEFAULT_MTIME)
-
-    @pytest.mark.xfail(
-        reason="ValueError is not thrown despite being provided directory."
-    )
-    def test_dirs_refused(self):
-        d = tempfile.mkdtemp()
-        try:
-            tp = os.path.join(d, "test.tar")
-            with open(tp, "wb") as fh:
-                with self.assertRaisesRegex(ValueError, "not a regular"):
-                    create_tar_from_files(fh, {"test": d})
-        finally:
-            shutil.rmtree(d)
-
-    def test_setuid_setgid_refused(self):
-        d = tempfile.mkdtemp()
-        try:
-            uid = os.path.join(d, "setuid")
-            gid = os.path.join(d, "setgid")
-            with open(uid, "a"):
-                pass
-            with open(gid, "a"):
-                pass
-
-            os.chmod(uid, MODE_STANDARD | stat.S_ISUID)
-            os.chmod(gid, MODE_STANDARD | stat.S_ISGID)
-
-            tp = os.path.join(d, "test.tar")
-            with open(tp, "wb") as fh:
-                with self.assertRaisesRegex(ValueError, "cannot add file with setuid"):
-                    create_tar_from_files(fh, {"test": uid})
-                with self.assertRaisesRegex(ValueError, "cannot add file with setuid"):
-                    create_tar_from_files(fh, {"test": gid})
-        finally:
-            shutil.rmtree(d)
-
-    def test_create_tar_basic(self):
-        d = tempfile.mkdtemp()
-        try:
-            files = self._create_files(d)
-
-            tp = os.path.join(d, "test.tar")
-            with open(tp, "wb") as fh:
-                create_tar_from_files(fh, files)
-
-            # Output should be deterministic.
-            self.assertEqual(file_hash(tp), "01cd314e277f060e98c7de6c8ea57f96b3a2065c")
-
-            with tarfile.open(tp, "r") as tf:
-                self._verify_basic_tarfile(tf)
-
-        finally:
-            shutil.rmtree(d)
-
-    @pytest.mark.xfail(reason="hash mismatch")
-    def test_executable_preserved(self):
-        d = tempfile.mkdtemp()
-        try:
-            p = os.path.join(d, "exec")
-            with open(p, "wb") as fh:
-                fh.write("#!/bin/bash\n")
-            os.chmod(p, MODE_STANDARD | stat.S_IXUSR)
-
-            tp = os.path.join(d, "test.tar")
-            with open(tp, "wb") as fh:
-                create_tar_from_files(fh, {"exec": p})
-
-            self.assertEqual(file_hash(tp), "357e1b81c0b6cfdfa5d2d118d420025c3c76ee93")
-
-            with tarfile.open(tp, "r") as tf:
-                m = tf.getmember("exec")
-                self.assertEqual(m.mode, MODE_STANDARD | stat.S_IXUSR)
-
-        finally:
-            shutil.rmtree(d)
-
-    def test_create_tar_gz_basic(self):
-        d = tempfile.mkdtemp()
-        try:
-            files = self._create_files(d)
-
-            gp = os.path.join(d, "test.tar.gz")
-            with open(gp, "wb") as fh:
-                create_tar_gz_from_files(fh, files)
-
-            self.assertEqual(file_hash(gp), "7c4da5adc5088cdf00911d5daf9a67b15de714b7")
-
-            with tarfile.open(gp, "r:gz") as tf:
-                self._verify_basic_tarfile(tf)
-
-        finally:
-            shutil.rmtree(d)
-
-    def test_tar_gz_name(self):
-        d = tempfile.mkdtemp()
-        try:
-            files = self._create_files(d)
-
-            gp = os.path.join(d, "test.tar.gz")
-            with open(gp, "wb") as fh:
-                create_tar_gz_from_files(fh, files, filename="foobar")
-
-            self.assertEqual(file_hash(gp), "721e00083c17d16df2edbddf40136298c06d0c49")
-
-            with tarfile.open(gp, "r:gz") as tf:
-                self._verify_basic_tarfile(tf)
-
-        finally:
-            shutil.rmtree(d)