taskcluster-taskgraph 24.1.1__tar.gz → 24.2.1__tar.gz

{taskcluster_taskgraph-24.1.1 → taskcluster_taskgraph-24.2.1}/.github/workflows/codspeed.yml

@@ -25,7 +25,7 @@ jobs:
       - name: Install dependencies
         run: uv sync
       - name: Run benchmarks
-        uses: CodSpeedHQ/action@9d332c4d90b43981c3e55ae8e38e68709996240f  # v4.17.0
+        uses: CodSpeedHQ/action@a4a36bb07c0638b0b4ca52bf1f3dad1b4289e52f  # v4.18.1
         with:
           mode: simulation
           run: uv run pytest test/ --codspeed

{taskcluster_taskgraph-24.1.1 → taskcluster_taskgraph-24.2.1}/.github/workflows/pypi-publish.yml

@@ -13,9 +13,9 @@ jobs:
     steps:
       - name: Checkout sources
         uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10  # v6.0.3
-      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1  # v6.3.0
         with:
-          python-version: '3.11'
+          python-version: '3.14'
           cache: 'pip'
       - name: Build taskcluster-taskgraph package distributions
         run: |
@@ -39,9 +39,9 @@ jobs:
     steps:
       - name: Checkout sources
         uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10  # v6.0.3
-      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1  # v6.3.0
         with:
-          python-version: '3.11'
+          python-version: '3.14'
           cache: 'pip'
       - name: Build pytest-taskgraph package distributions
         working-directory: packages/pytest-taskgraph
@@ -62,9 +62,9 @@ jobs:
     steps:
       - name: Checkout sources
         uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10  # v6.0.3
-      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1  # v6.3.0
         with:
-          python-version: '3.11'
+          python-version: '3.14'
           cache: 'pip'
       - name: Build sphinx-taskgraph package distributions
         working-directory: packages/sphinx-taskgraph

{taskcluster_taskgraph-24.1.1 → taskcluster_taskgraph-24.2.1}/.pre-commit-config.yaml

@@ -13,13 +13,13 @@ repos:
         exclude: template
       - id: check-added-large-files
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.15.15
+    rev: v0.15.20
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
       - id: ruff-format
   - repo: https://github.com/astral-sh/uv-pre-commit
-    rev: 0.11.17
+    rev: 0.11.26
     hooks:
       - id: uv-lock
   - repo: https://github.com/adrienverge/yamllint.git

{taskcluster_taskgraph-24.1.1 → taskcluster_taskgraph-24.2.1}/CHANGELOG.md

@@ -1,5 +1,24 @@
 # Change Log
 
+## [24.2.1] - 2026-07-27
+
+### Fixed
+
+- Don't run nested ProcessPoolExecutors (#1008)
+
+## [24.2.0] - 2026-07-15
+
+### Added
+
+- Included profile data for `run-task` VCS operations in the log
+
+### Fixed
+
+- Ensure `INDEX_RANK` env is cast to a string for index tasks
+- Properly capture all exceptions in `taskgraph.create.create_tasks`
+- Send `Accept-Encoding` header in `fetch-content` script
+- Release graphs from memory upon completion when using `taskgraph <phase> --diff`
+
 ## [24.1.1] - 2026-06-11
 
 ### Fixed

{taskcluster_taskgraph-24.1.1 → taskcluster_taskgraph-24.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: taskcluster-taskgraph
-Version: 24.1.1
+Version: 24.2.1
 Summary: Build taskcluster taskgraphs
 Project-URL: Repository, https://github.com/taskcluster/taskgraph
 Project-URL: Issues, https://github.com/taskcluster/taskgraph/issues

{taskcluster_taskgraph-24.1.1 → taskcluster_taskgraph-24.2.1}/docs/howto/bootstrap-taskgraph.rst

@@ -73,22 +73,15 @@ This section outlines the recommended way to do that.
 Define Requirements
 ~~~~~~~~~~~~~~~~~~~
 
-We'll use a tool called `pip-compile`_ to help generate the
-``requirements.txt`` that will contain our Decision task dependencies,
-including Taskgraph itself.
+We'll use `uv`_ to help generate the ``requirements.txt`` that will contain our
+Decision task dependencies, including Taskgraph itself.
 
 .. note::
 
    You may use other tools to create the ``requirements.txt``, such as
-   `hashin`_. As long as package hashes are included.
+   `pip-compile`_ or `hashin`_. As long as package hashes are included.
 
-First make sure you have ``pip-compile`` installed:
-
-.. code-block:: bash
-
-   pip install pip-tools
-
-Next, create the ``requirements.in`` file. For this example, let's pretend that
+First make sure you `install uv`_. Next, create the ``requirements.in`` file. For this example, let's pretend that
 in addition to ``taskcluster-taskgraph`` we also want to depend on
 ``requests``:
 
@@ -100,9 +93,7 @@ in addition to ``taskcluster-taskgraph`` we also want to depend on
    requests
    EOF
 
-   # This works best if you use the same Python as the one used in the Decision
-   # image (currently 3.13).
-   pip-compile --generate-hashes --output-file requirements.txt requirements.in
+   uv pip compile --universal --generate-hashes --output-file requirements.txt requirements.in
 
 .. note::
 
@@ -113,6 +104,8 @@ in addition to ``taskcluster-taskgraph`` we also want to depend on
 If you end up creating transforms that require additional packages, add them to
 ``requirements.in`` and re-generate the lock file.
 
+.. _uv: https://docs.astral.sh/uv/
+.. _install uv: https://docs.astral.sh/uv/getting-started/installation/
 .. _pip-compile: https://github.com/jazzband/pip-tools
 .. _hashin: https://github.com/peterbe/hashin
 .. _version specifiers: https://pip.pypa.io/en/stable/cli/pip_install/#requirement-specifiers
@@ -186,7 +179,7 @@ file:
 .. code-block:: shell
 
    cd taskcluster
-   pip-compile --generate-hashes --output-file requirements.txt requirements.in
+   uv pip compile --universal --generate-hashes --output-file requirements.txt requirements.in
 
 If you pinned the package to a specific version don't forget to update
 ``requirements.in`` first.
@@ -209,7 +202,7 @@ accomplished using `pip's version control support`_:
 
    cd taskcluster
    echo "taskcluster-taskgraph@git+https://github.com/taskcluster/taskgraph@refs/pull/123/head" > requirements.in
-   pip-compile --output-file requirements.txt requirements.in
+   uv pip compile --universal --output-file requirements.txt requirements.in
 
 Next edit your ``.taskcluster.yml`` to disable hashing since pip does not
 support `hashes with url requirements`_:
@@ -222,7 +215,7 @@ support `hashes with url requirements`_:
 
 .. note::
 
-   Be sure to omit the ``--generate-hashes`` argument to ``pip-compile``
+   Be sure to omit the ``--generate-hashes`` argument to ``uv pip compile``
    otherwise ``pip`` will implicitly turn hashing back on.
 
 This way you can push an experimental change to a PR and then install it in

{taskcluster_taskgraph-24.1.1 → taskcluster_taskgraph-24.2.1}/pyproject.toml

@@ -1,7 +1,7 @@
 ### Project
 [project]
 name = "taskcluster-taskgraph"
-version = "24.1.1"
+version = "24.2.1"
 description = "Build taskcluster taskgraphs"
 readme = "README.rst"
 authors = [

{taskcluster_taskgraph-24.1.1 → taskcluster_taskgraph-24.2.1}/src/taskgraph/create.py

@@ -64,7 +64,6 @@ def create_tasks(graph_config, taskgraph, label_to_taskid, params, decision_task
         fs = {}
         fs_to_task = {}
         skipped = set()
-        errors = {}
 
         # We can't submit a task until its dependencies have been submitted.
         # So our strategy is to walk the graph and submit tasks once all
@@ -72,12 +71,6 @@ def create_tasks(graph_config, taskgraph, label_to_taskid, params, decision_task
         tasklist = set(taskgraph.graph.visit_postorder())
         alltasks = tasklist.copy()
 
-        def handle_exception(fut):
-            if exc := fut.exception():
-                task_id, label = fs_to_task[fut]
-                skipped.add(task_id)
-                errors[label] = exc
-
         def schedule_tasks():
             to_remove = set()
             new = set()
@@ -87,7 +80,13 @@ def create_tasks(graph_config, taskgraph, label_to_taskid, params, decision_task
                 new.add(fut)
                 fs[task_id] = fut
                 fs_to_task[fut] = (task_id, label)
-                fut.add_done_callback(handle_exception)
+
+                def mark_failed_as_skipped(fut):
+                    if fut.exception():
+                        task_id, _ = fs_to_task[fut]
+                        skipped.add(task_id)
+
+                fut.add_done_callback(mark_failed_as_skipped)
 
             for task_id in tasklist:
                 task_def = taskgraph.tasks[task_id].task
@@ -127,6 +126,12 @@ def create_tasks(graph_config, taskgraph, label_to_taskid, params, decision_task
         # Wait for all futures to complete.
         futures.wait(fs.values())
 
+        # Collect errors.
+        errors = {}
+        for fut, (task_id, label) in fs_to_task.items():
+            if exc := fut.exception():
+                errors[label] = exc
+
         if errors:
             raise CreateTasksException(errors)
 

{taskcluster_taskgraph-24.1.1 → taskcluster_taskgraph-24.2.1}/src/taskgraph/main.py

@@ -179,6 +179,16 @@ def format_taskgraph(options, parameters, overrides, logfile=None):
     return format_method(tg)
 
 
+def format_taskgraph_multiple(*args, **kwargs):
+    """Thin wrapper around format_taskgraph to avoid the taskgraph generator
+    that it runs using multiple processes. (This function is intended to be
+    called by something that is already using a ProcessPoolExecutor, so
+    there's no advantage to spawning additional processes in the subprocesses.)"""
+
+    os.environ["TASKGRAPH_SERIAL"] = "1"
+    return format_taskgraph(*args, **kwargs)
+
+
 def dump_output(out, path=None, params_spec=None):
     from taskgraph.parameters import Parameters  # noqa: PLC0415
 
@@ -222,32 +232,32 @@ def generate_taskgraph(options, parameters, overrides, logdir):
         return 0
 
     futures = {}
+    returncode = 0
     with ProcessPoolExecutor(max_workers=options["max_workers"]) as executor:
         for spec in parameters:
             f = executor.submit(
-                format_taskgraph, options, spec, overrides, logfile(spec)
+                format_taskgraph_multiple, options, spec, overrides, logfile(spec)
             )
             futures[f] = spec
 
-    returncode = 0
-    for future in as_completed(futures):
-        output_file = options["output_file"]
-        spec = futures[future]
-        e = future.exception()
-        if e:
-            returncode = 1
-            out = "".join(traceback.format_exception(type(e), e, e.__traceback__))
-            if options["diff"]:
-                # Dump to console so we don't accidentally diff the tracebacks.
-                output_file = None
-        else:
-            out = future.result()
+        for future in as_completed(futures):
+            output_file = options["output_file"]
+            spec = futures.pop(future)
+            e = future.exception()
+            if e:
+                returncode = 1
+                out = "".join(traceback.format_exception(type(e), e, e.__traceback__))
+                if options["diff"]:
+                    # Dump to console so we don't accidentally diff the tracebacks.
+                    output_file = None
+            else:
+                out = future.result()
 
-        dump_output(
-            out,
-            path=output_file,
-            params_spec=spec if len(parameters) > 1 else None,
-        )
+            dump_output(
+                out,
+                path=output_file,
+                params_spec=spec if len(parameters) > 1 else None,
+            )
 
     return returncode
 

{taskcluster_taskgraph-24.1.1 → taskcluster_taskgraph-24.2.1}/src/taskgraph/morph.py

@@ -147,7 +147,9 @@ def make_index_task(parent_task, taskgraph, label_to_taskid, parameters, graph_c
     task.task["payload"]["command"] = ["insert-indexes.js"] + index_paths
     task.task["payload"]["env"] = {
         "TARGET_TASKID": parent_task.task_id,
-        "INDEX_RANK": parent_task.task.get("extra", {}).get("index", {}).get("rank", 0),
+        "INDEX_RANK": str(
+            parent_task.task.get("extra", {}).get("index", {}).get("rank", 0)
+        ),
     }
     return task, taskgraph, label_to_taskid
 

{taskcluster_taskgraph-24.1.1 → taskcluster_taskgraph-24.2.1}/src/taskgraph/run-task/fetch-content

@@ -181,7 +181,7 @@ def stream_download(url, sha256=None, size=None, headers=None):
     length = 0
 
     t0 = time.time()
-    req_headers = {}
+    req_headers = {"Accept-Encoding": "gzip"}
     for header in headers:
         key, val = header.split(":")
         req_headers[key.strip()] = val.strip()

{taskcluster_taskgraph-24.1.1 → taskcluster_taskgraph-24.2.1}/src/taskgraph/run-task/run-task

@@ -14,6 +14,7 @@ current time to improve log usefulness.
 """
 
 import argparse
+import contextlib
 import datetime
 import errno
 import io
@@ -567,6 +568,61 @@ def configure_volume_posix(volume, user, group, running_as_root):
         set_dir_permissions(volume, user.pw_uid, group.gr_gid)
 
 
+class PerfRecorder:
+    """Collects operation timings and emits them as PERFHERDER_DATA."""
+
+    def __init__(self, framework: str):
+        self.framework = framework
+        self.optimes = []
+        self._stack = []
+
+    def start(self, *names: str):
+        self._stack.append((time.time(), names))
+
+    def stop(self, errored: bool = False):
+        start, names = self._stack.pop()
+        elapsed = time.time() - start
+        suffix = "_errored" if errored else ""
+        for name in names:
+            self.optimes.append((name + suffix, elapsed))
+
+    @contextlib.contextmanager
+    def record(self, *names: str):
+        self.start(*names)
+        try:
+            yield
+        except BaseException:
+            self.stop(errored=True)
+            raise
+        else:
+            self.stop()
+
+    def emit(self):
+        instance_type = os.environ.get("TASKCLUSTER_INSTANCE_TYPE")
+        if not instance_type or not self.optimes:
+            return
+
+        suites = []
+        for op, duration in self.optimes:
+            suites.append(
+                {
+                    "name": op,
+                    "value": duration,
+                    "lowerIsBetter": True,
+                    "shouldAlert": False,
+                    "extraOptions": [instance_type],
+                    "subtests": [],
+                }
+            )
+
+        perfherder = {"framework": {"name": self.framework}, "suites": suites}
+        sys.stdout.write(
+            "PERFHERDER_DATA: {}\n".format(json.dumps(perfherder, sort_keys=True))
+        )
+        sys.stdout.flush()
+        self.optimes = []
+
+
 def git_fetch(
     destination_path: str,
     *targets: str,
@@ -668,105 +724,114 @@ def git_checkout(
             "Must specify both ssh_key_file and ssh_known_hosts_file, if either are specified",
         )
 
-    # Bypass Git's "safe directory" feature as the destination could be
-    # coming from a cache and therefore cloned by a different user.
-    args = [
-        "git",
-        "config",
-        "--global",
-        "--add",
-        "safe.directory",
-        Path(destination_path).as_posix(),
-    ]
-    retry_required_command(b"vcs", args, extra_env=env)
-
-    if not os.path.exists(destination_path):
-        # Repository doesn't already exist, needs to be cloned
+    has_repo = os.path.exists(destination_path)
+    perf = PerfRecorder("vcs")
+    with perf.record("overall", "overall_pull" if has_repo else "overall_clone"):
+        # Bypass Git's "safe directory" feature as the destination could be
+        # coming from a cache and therefore cloned by a different user.
         args = [
             "git",
-            "clone",
+            "config",
+            "--global",
+            "--add",
+            "safe.directory",
+            Path(destination_path).as_posix(),
         ]
-
-        if shallow:
-            args.extend(["--depth=1", "--no-checkout"])
-
-        args.extend(
-            [
-                base_repo if base_repo else head_repo,
-                destination_path,
-            ]
-        )
-
         retry_required_command(b"vcs", args, extra_env=env)
 
-    # For Github based repos, base_rev often doesn't refer to an ancestor of
-    # head_rev simply due to Github not providing that information in their
-    # webhook events. Therefore we fetch it independently from `head_rev` so
-    # that consumers can compute the merge-base or files modified between the
-    # two as needed.
-    if base_rev and base_rev != NULL_REVISION:
-        git_fetch(destination_path, base_rev, shallow=shallow, env=env)
-
-    # If a head_ref was provided, it might be tag, so we need to make sure we fetch
-    # those. This is explicitly only done when base and head repo match,
-    # because it is the only scenario where tags could be present. (PRs, for
-    # example, always include an explicit rev.) Failure to do this could result
-    # in not having a tag, or worse: having an outdated version of one.
-    tags = False
-    if head_ref and not head_ref.startswith("refs/heads/") and base_repo == head_repo:
-        tags = True
-
-    # Fetch head_ref and/or head_rev
-    targets = []
-    if head_ref:
-        targets.append(head_ref)
-    if not head_ref or (shallow and head_rev):
-        # If head_ref wasn't provided, we fallback to head_rev. If we have a
-        # shallow clone, head_rev needs to be fetched independently regardless.
-        targets.append(head_rev)
-
-    git_fetch(
-        destination_path,
-        *targets,
-        remote=head_repo,
-        tags=tags,
-        shallow=shallow,
-        env=env,
-    )
-
-    args = [
-        "git",
-        "checkout",
-        "-f",
-    ]
+        if not has_repo:
+            # Repository doesn't already exist, needs to be cloned
+            args = [
+                "git",
+                "clone",
+            ]
 
-    if head_ref:
-        args.extend(["-B", shortref(head_ref)])
+            if shallow:
+                args.extend(["--depth=1", "--no-checkout"])
 
-    # `git fetch` set `FETCH_HEAD` reference to the last commit of the desired branch
-    args.append(head_rev if head_rev else "FETCH_HEAD")
+            args.extend(
+                [
+                    base_repo if base_repo else head_repo,
+                    destination_path,
+                ]
+            )
 
-    run_required_command(b"vcs", args, cwd=destination_path)
+            with perf.record("clone"):
+                retry_required_command(b"vcs", args, extra_env=env)
+
+        # For Github based repos, base_rev often doesn't refer to an ancestor of
+        # head_rev simply due to Github not providing that information in their
+        # webhook events. Therefore we fetch it independently from `head_rev` so
+        # that consumers can compute the merge-base or files modified between the
+        # two as needed.
+        if base_rev and base_rev != NULL_REVISION:
+            with perf.record("fetch_base"):
+                git_fetch(destination_path, base_rev, shallow=shallow, env=env)
+
+        # If a head_ref was provided, it might be tag, so we need to make sure we fetch
+        # those. This is explicitly only done when base and head repo match,
+        # because it is the only scenario where tags could be present. (PRs, for
+        # example, always include an explicit rev.) Failure to do this could result
+        # in not having a tag, or worse: having an outdated version of one.
+        tags = False
+        if head_ref and not head_ref.startswith("refs/heads/") and base_repo == head_repo:
+            tags = True
+
+        # Fetch head_ref and/or head_rev
+        targets = []
+        if head_ref:
+            targets.append(head_ref)
+        if not head_ref or (shallow and head_rev):
+            # If head_ref wasn't provided, we fallback to head_rev. If we have a
+            # shallow clone, head_rev needs to be fetched independently regardless.
+            targets.append(head_rev)
+
+        with perf.record("pull"):
+            git_fetch(
+                destination_path,
+                *targets,
+                remote=head_repo,
+                tags=tags,
+                shallow=shallow,
+                env=env,
+            )
 
-    if os.path.exists(os.path.join(destination_path, ".gitmodules")):
         args = [
             "git",
-            "submodule",
-            "init",
+            "checkout",
+            "-f",
         ]
 
-        run_required_command(b"vcs", args, cwd=destination_path)
+        if head_ref:
+            args.extend(["-B", shortref(head_ref)])
 
-        args = [
-            "git",
-            "submodule",
-            "update",
-            "--force",  # Overrides any potential local changes
-        ]
+        # `git fetch` set `FETCH_HEAD` reference to the last commit of the desired branch
+        args.append(head_rev if head_rev else "FETCH_HEAD")
+
+        with perf.record("update"):
+            run_required_command(b"vcs", args, cwd=destination_path)
 
-        run_required_command(b"vcs", args, cwd=destination_path)
+        if os.path.exists(os.path.join(destination_path, ".gitmodules")):
+            args = [
+                "git",
+                "submodule",
+                "init",
+            ]
 
-    _clean_git_checkout(destination_path)
+            run_required_command(b"vcs", args, cwd=destination_path)
+
+            args = [
+                "git",
+                "submodule",
+                "update",
+                "--force",  # Overrides any potential local changes
+            ]
+
+            with perf.record("submodule_update"):
+                run_required_command(b"vcs", args, cwd=destination_path)
+
+        with perf.record("purge"):
+            _clean_git_checkout(destination_path)
 
     args = ["git", "rev-parse", "--verify", "HEAD"]
 
@@ -793,6 +858,8 @@ def git_checkout(
 
     print_line(b"vcs", msg.encode("utf-8"))
 
+    perf.emit()
+
     return commit_hash
 
 

{taskcluster_taskgraph-24.1.1 → taskcluster_taskgraph-24.2.1}/src/taskgraph/transforms/docker_image.py

@@ -20,9 +20,9 @@ CONTEXTS_DIR = "docker-contexts"
 DIGEST_RE = re.compile("^[0-9a-f]{64}$")
 
 IMAGE_BUILDER_IMAGE = (
-    "mozillareleases/image_builder:6.1.0"
+    "mozillareleases/image_builder:6.2.0"
     "@sha256:"
-    "10c614e928500a7d5138773fb58941a24a17948e1a8193768e9ba762ba3e9c7e"
+    "f7df722770d6caadea79b2de458e072d8b3d93e59675b34dffb32c3e7e30f5b9"
 )
 
 transforms = TransformSequence()

{taskcluster_taskgraph-24.1.1 → taskcluster_taskgraph-24.2.1}/taskcluster/docker/fetch/Dockerfile

@@ -1,4 +1,4 @@
-FROM debian:12-slim
+FROM debian:12-slim@sha256:60eac759739651111db372c07be67863818726f754804b8707c90979bda511df
 LABEL maintainer="Release Engineering <release@mozilla.com>"
 
 ### Add worker user and setup its workspace.

{taskcluster_taskgraph-24.1.1 → taskcluster_taskgraph-24.2.1}/taskcluster/docker/index-task/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:18-alpine
+FROM node:24-alpine@sha256:a0b9bf06e4e6193cf7a0f58816cc935ff8c2a908f81e6f1a95432d679c54fbfd
 
 ENV       NODE_ENV        production
 RUN       mkdir /app

{taskcluster_taskgraph-24.1.1 → taskcluster_taskgraph-24.2.1}/taskcluster/docker/index-task/insert-indexes.js

@@ -1,4 +1,4 @@
-let taskcluster = require("taskcluster-client");
+import taskcluster from "@taskcluster/client";
 
 // Create instance of index client
 let index = new taskcluster.Index({

{taskcluster_taskgraph-24.1.1 → taskcluster_taskgraph-24.2.1}/taskcluster/docker/index-task/package.json

@@ -1,12 +1,13 @@
 {
   "private": true,
+  "type": "module",
   "scripts": {
     "start": "node index.js"
   },
   "dependencies": {
-    "taskcluster-client": "55.3.2"
+    "@taskcluster/client": "101.0.0"
   },
   "engines": {
-    "node": "18"
+    "node": "^24"
   }
 }