taskcluster-taskgraph 20.0.0__tar.gz → 21.0.0__tar.gz

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-21.0.0}/.pre-commit-config.yaml

@@ -13,13 +13,13 @@ repos:
         exclude: template
       - id: check-added-large-files
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.15.4
+    rev: v0.15.8
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
       - id: ruff-format
   - repo: https://github.com/astral-sh/uv-pre-commit
-    rev: 0.10.7
+    rev: 0.11.2
     hooks:
       - id: uv-lock
   - repo: https://github.com/adrienverge/yamllint.git
@@ -28,7 +28,7 @@ repos:
       - id: yamllint
         exclude: template
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.4.1
+    rev: v2.4.2
     hooks:
       - id: codespell
         entry: codespell -I .codespell-ignore-words.txt

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-21.0.0}/.taskcluster.yml

@@ -226,6 +226,7 @@ tasks:
                               - $if: 'isPullRequest'
                                 then:
                                     TASKGRAPH_PULL_REQUEST_NUMBER: '${event.pull_request.number}'
+                                    TASKGRAPH_EXTRA_REFS: {$json: ["refs/notes/decision-parameters"]}
                               - $if: 'tasks_for == "action" || tasks_for == "pr-action"'
                                 then:
                                     ACTION_TASK_GROUP_ID: '${action.taskGroupId}'  # taskGroupId of the target task
@@ -251,7 +252,13 @@ tasks:
                           - bash
                           - -cx
                           - $let:
-                                extraArgs: {$if: 'tasks_for == "cron"', then: '${cron.quoted_args}', else: ''}
+                                extraArgs:
+                                    $if: 'tasks_for == "cron"'
+                                    then: '${cron.quoted_args}'
+                                    else:
+                                        $if: 'tasks_for == "github-pull-request"'
+                                        then: '--allow-parameter-override'
+                                        else: ''
                             in:
                                 $if: 'tasks_for == "action" || tasks_for == "pr-action"'
                                 then: >

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-21.0.0}/CHANGELOG.md

@@ -1,5 +1,24 @@
 # Change Log
 
+## [21.0.0] - 2026-04-13
+
+### Added
+
+- Ability for `run-task` to fetch arbitrary extra refs
+- Ability for Decision task to override params via Git notes
+
+### Fixed
+
+- Improved index route verification
+
+### Changed
+
+- BREAKING CHANGE: run-task and decision images now use Debian 13
+
+### Removed
+
+- BREAKING CHANGE: Removed docker-worker features: dind, privileged, loopback-audio
+
 ## [20.0.0] - 2026-03-17
 
 ### Added

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-21.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: taskcluster-taskgraph
-Version: 20.0.0
+Version: 21.0.0
 Summary: Build taskcluster taskgraphs
 Project-URL: Repository, https://github.com/taskcluster/taskgraph
 Project-URL: Issues, https://github.com/taskcluster/taskgraph/issues

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-21.0.0}/docs/howto/bootstrap-taskgraph.rst

@@ -101,7 +101,7 @@ in addition to ``taskcluster-taskgraph`` we also want to depend on
    EOF
 
    # This works best if you use the same Python as the one used in the Decision
-   # image (currently 3.11).
+   # image (currently 3.13).
    pip-compile --generate-hashes --output-file requirements.txt requirements.in
 
 .. note::

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-21.0.0}/docs/reference/migrations.rst

@@ -3,6 +3,15 @@ Migration Guide
 
 This page can help when migrating Taskgraph across major versions.
 
+20.x -> 21.x
+------------
+
+* If you have tasks using `docker-worker`'s `dind`, `privileged` or `loopback-audio`
+  features, they are no longer supported. Either avoid using these features or migrate
+  to a `generic-worker` pool with D2G payloads enabled.
+* Ensure any task's using the `run-task` or `decision` images work with the new
+  Debian 13 base.
+
 19.x -> 20.x
 ------------
 

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-21.0.0}/pyproject.toml

@@ -1,7 +1,7 @@
 ### Project
 [project]
 name = "taskcluster-taskgraph"
-version = "20.0.0"
+version = "21.0.0"
 description = "Build taskcluster taskgraphs"
 readme = "README.rst"
 authors = [

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-21.0.0}/src/taskgraph/decision.py

@@ -264,6 +264,20 @@ def get_decision_parameters(graph_config, options):
     ] == "github-pull-request":
         set_try_config(parameters, task_config_file)
 
+    # load extra parameters from vcs note if able
+    note_ref = "refs/notes/decision-parameters"
+    if options.get("allow_parameter_override") and (
+        note_params := repo.get_note(note_ref)
+    ):
+        try:
+            note_params = json.loads(note_params)
+            logger.info(
+                f"Overriding parameters from {note_ref}:\n{json.dumps(note_params, indent=2)}"
+            )
+            parameters.update(note_params)
+        except ValueError as e:
+            raise Exception(f"Failed to parse {note_ref} as JSON: {e}") from e
+
     result = Parameters(**parameters)
     result.check()
     return result

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-21.0.0}/src/taskgraph/main.py

@@ -933,6 +933,12 @@ def load_task(args):
 @argument(
     "--verbose", "-v", action="store_true", help="include debug-level logging output"
 )
+@argument(
+    "--allow-parameter-override",
+    default=False,
+    action="store_true",
+    help="Allow user to override computed decision task parameters.",
+)
 def decision(options):
     from taskgraph.decision import taskgraph_decision  # noqa: PLC0415
 

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-21.0.0}/src/taskgraph/run-task/run-task

@@ -31,7 +31,7 @@ import time
 import urllib.error
 import urllib.request
 from pathlib import Path
-from typing import Dict, Optional
+from typing import Dict, List, Optional
 
 SECRET_BASEURL_TPL = "{}/secrets/v1/secret/{{}}".format(
     os.environ.get("TASKCLUSTER_PROXY_URL", "http://taskcluster").rstrip("/")
@@ -640,6 +640,7 @@ def git_checkout(
     ssh_key_file: Optional[Path],
     ssh_known_hosts_file: Optional[Path],
     shallow: bool = False,
+    extra_refs: Optional[List[str]] = None,
 ):
     assert head_ref or head_rev
 
@@ -734,6 +735,14 @@ def git_checkout(
         env=env,
     )
 
+    if extra_refs:
+        git_fetch(
+            destination_path,
+            *[f"{ref}:{ref}" for ref in extra_refs],
+            remote=head_repo,
+            env=env,
+        )
+
     args = [
         "git",
         "checkout",
@@ -932,6 +941,8 @@ def collect_vcs_options(args, project, name):
     head_rev = os.environ.get(f"{env_prefix}_HEAD_REV")
     pip_requirements = os.environ.get(f"{env_prefix}_PIP_REQUIREMENTS")
     private_key_secret = os.environ.get(f"{env_prefix}_SSH_SECRET_NAME")
+    if extra_refs := os.environ.get(f"{env_prefix}_EXTRA_REFS"):
+        extra_refs = json.loads(extra_refs)
 
     store_path = os.environ.get("HG_STORE_PATH")
 
@@ -965,6 +976,7 @@ def collect_vcs_options(args, project, name):
         "ssh-secret-name": private_key_secret,
         "pip-requirements": pip_requirements,
         "shallow-clone": shallow_clone,
+        "extra-refs": extra_refs,
     }
 
 
@@ -1014,6 +1026,7 @@ def vcs_checkout_from_args(options):
                 ssh_key_file,
                 ssh_known_hosts_file,
                 shallow=options.get("shallow-clone", False),
+                extra_refs=options.get("extra-refs"),
             )
         elif options["repo-type"] == "hg":
             revision = hg_checkout(

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-21.0.0}/src/taskgraph/transforms/task.py

@@ -273,9 +273,6 @@ class DockerWorkerPayloadSchema(Schema, forbid_unknown_fields=False, kw_only=Tru
     taskcluster_proxy: bool
     allow_ptrace: bool
     loopback_video: bool
-    loopback_audio: bool
-    docker_in_docker: bool  # (aka 'dind')
-    privileged: bool
     # environment variables
     env: dict[str, taskref_or_string_msgspec]
     # the maximum time to run, in seconds
@@ -356,9 +353,6 @@ def build_docker_worker_payload(config, task, task_def):
     if worker.get("chain-of-trust"):
         features["chainOfTrust"] = True
 
-    if worker.get("docker-in-docker"):
-        features["dind"] = True
-
     if task.get("needs-sccache"):
         features["taskclusterProxy"] = True
         task_def["scopes"].append(
@@ -375,16 +369,10 @@ def build_docker_worker_payload(config, task, task_def):
 
     capabilities = {}
 
-    for lo in "audio", "video":
-        if worker.get("loopback-" + lo):
-            capitalized = "loopback" + lo.capitalize()
-            devices = capabilities.setdefault("devices", {})
-            devices[capitalized] = True
-            task_def["scopes"].append("docker-worker:capability:device:" + capitalized)
-
-    if worker.get("privileged"):
-        capabilities["privileged"] = True
-        task_def["scopes"].append("docker-worker:capability:privileged")
+    if worker.get("loopback-video"):
+        devices = capabilities.setdefault("devices", {})
+        devices["loopbackVideo"] = True
+        task_def["scopes"].append("docker-worker:capability:device:loopbackVideo")
 
     task_def["payload"] = payload = {
         "image": image,
@@ -815,9 +803,6 @@ def set_defaults(config, tasks):
             worker.setdefault("taskcluster-proxy", False)
             worker.setdefault("allow-ptrace", False)
             worker.setdefault("loopback-video", False)
-            worker.setdefault("loopback-audio", False)
-            worker.setdefault("docker-in-docker", False)
-            worker.setdefault("privileged", False)
             worker.setdefault("volumes", [])
             worker.setdefault("env", {})
             if "caches" in worker:

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-21.0.0}/src/taskgraph/util/vcs.py

@@ -215,6 +215,14 @@ class Repository(ABC):
         If this function returns an unexpected value, then make sure
         the revision was fetched from the remote repository."""
 
+    def get_note(self, note: str, revision: Optional[str] = None) -> Optional[str]:
+        """Read a note attached to the given revision (defaults to HEAD).
+
+        Returns the note content as a string, or ``None`` if no note exists.
+        Only supported by Git; returns ``None`` for all other VCS types.
+        """
+        return None
+
 
 class HgRepository(Repository):
     @property
@@ -586,6 +594,18 @@ class GitRepository(Repository):
                 return False
             raise
 
+    def get_note(self, note: str, revision: Optional[str] = None) -> Optional[str]:
+        if not note.startswith("refs/notes/"):
+            note = f"refs/notes/{note}"
+
+        revision = revision or "HEAD"
+        try:
+            return self.run("notes", f"--ref={note}", "show", revision).strip()
+        except subprocess.CalledProcessError as e:
+            if e.returncode == 1:
+                return None
+            raise
+
 
 def get_repository(path: str):
     """Get a repository object for the repository at `path`.

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-21.0.0}/src/taskgraph/util/verify.py

@@ -235,10 +235,14 @@ def verify_routes_notification_filters(
                 )
 
 
+# https://docs.taskcluster.net/docs/reference/core/index#valid-characters
+INDEX_NAMESPACE_RE = re.compile(r"^([a-zA-Z0-9_!~*'()%-]+\.)*[a-zA-Z0-9_!~*'()%-]+$")
+
+
 @verifications.add("full_task_graph")
 def verify_index_route(task, taskgraph, scratch_pad, graph_config, parameters):
     """
-    This function ensures that routes do not contain forward slashes.
+    This function ensures that index routes would create valid taskcluster index paths
     """
     if task is None:
         return
@@ -247,11 +251,13 @@ def verify_index_route(task, taskgraph, scratch_pad, graph_config, parameters):
     route_prefix = "index."
 
     for route in routes:
-        # Check for invalid / in the index route
-        if route.startswith(route_prefix) and "/" in route:
-            raise Exception(
-                f"{task.label} has invalid route with forward slash: {route}"
-            )
+        if route.startswith(route_prefix):
+            namespace = route[len(route_prefix) :]
+            if not INDEX_NAMESPACE_RE.match(namespace):
+                raise Exception(
+                    f"{task.label} has invalid index route namespace: {route}. "
+                    f"Namespace must match {INDEX_NAMESPACE_RE.pattern}"
+                )
 
 
 @verifications.add("full_task_graph")

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-21.0.0}/taskcluster/docker/run-task/Dockerfile

@@ -1,4 +1,4 @@
-FROM debian:12-slim
+FROM debian:13-slim
 LABEL maintainer="Release Engineering <release+taskgraph-ci@mozilla.com>"
 
 # Add worker user

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-21.0.0}/test/test_decision.py

@@ -10,6 +10,8 @@ import tempfile
 import unittest
 from pathlib import Path
 
+import pytest
+
 from taskgraph import decision
 from taskgraph.util.vcs import GitRepository, HgRepository
 from taskgraph.util.yaml import load_yaml
@@ -137,3 +139,60 @@ class TestGetDecisionParameters(unittest.TestCase):
         self.options["tasks_for"] = "hg-push"
         params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options)
         self.assertEqual(params["target_tasks_method"], "nothing")
+
+
+_BASE_OPTIONS = {
+    "base_repository": "https://hg.mozilla.org/mozilla-unified",
+    "base_rev": "aaaa",
+    "head_repository": "https://hg.mozilla.org/mozilla-central",
+    "head_rev": "bbbb",
+    "head_ref": "default",
+    "head_tag": "",
+    "project": "mozilla-central",
+    "pushlog_id": "1",
+    "pushdate": 0,
+    "repository_type": "git",
+    "owner": "nobody@mozilla.com",
+    "tasks_for": "github-push",
+    "level": "1",
+}
+
+
+@unittest.mock.patch.object(
+    GitRepository,
+    "get_note",
+    return_value=json.dumps({"build_number": 99}),
+)
+@unittest.mock.patch.object(GitRepository, "get_changed_files", return_value=[])
+def test_decision_parameters_note(mock_files_changed, mock_get_note):
+    options = {**_BASE_OPTIONS, "allow_parameter_override": True}
+    params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, options)
+    mock_get_note.assert_called_once_with("refs/notes/decision-parameters")
+    assert params["build_number"] == 99
+
+
+@unittest.mock.patch.object(
+    GitRepository,
+    "get_note",
+    return_value=json.dumps({"build_number": 99}),
+)
+@unittest.mock.patch.object(GitRepository, "get_changed_files", return_value=[])
+def test_decision_parameters_note_disallow_override(mock_files_changed, mock_get_note):
+    options = {**_BASE_OPTIONS, "allow_parameter_override": False}
+    params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, options)
+    mock_get_note.assert_not_called()
+    assert params["build_number"] == 1
+
+
+@unittest.mock.patch.object(
+    GitRepository,
+    "get_note",
+    return_value="not valid json {",
+)
+@unittest.mock.patch.object(GitRepository, "get_changed_files", return_value=[])
+def test_decision_parameters_note_invalid_json(mock_files_changed, mock_get_note):
+    options = {**_BASE_OPTIONS, "allow_parameter_override": True}
+    with pytest.raises(
+        Exception, match="Failed to parse refs/notes/decision-parameters as JSON"
+    ):
+        decision.get_decision_parameters(FAKE_GRAPH_CONFIG, options)

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-21.0.0}/test/test_scripts_run_task.py

@@ -1,4 +1,5 @@
 import functools
+import json
 import os
 import site
 import stat
@@ -179,6 +180,17 @@ def test_install_pip_requirements_with_uv(
             {"shallow-clone": True},
             id="git_with_shallow_clone",
         ),
+        pytest.param(
+            {},
+            {
+                "REPOSITORY_TYPE": "git",
+                "HEAD_REPOSITORY": "https://github.com/example/repo",
+                "HEAD_REV": "abc123",
+                "EXTRA_REFS": json.dumps(["refs/notes/taskgraph", "refs/notes/other"]),
+            },
+            {"extra-refs": ["refs/notes/taskgraph", "refs/notes/other"]},
+            id="git_with_extra_refs",
+        ),
     ],
 )
 def test_collect_vcs_options(
@@ -216,6 +228,7 @@ def test_collect_vcs_options(
         "shallow-clone": False,
         "ssh-secret-name": env.get("SSH_SECRET_NAME"),
         "store-path": env.get("HG_STORE_PATH"),
+        "extra-refs": None,
     }
     if "PIP_REQUIREMENTS" in env:
         expected["pip-requirements"] = os.path.join(
@@ -638,3 +651,36 @@ def test_main_abspath_environment(mocker, run_main):
     assert env.get("MOZ_UV_HOME") == "/builds/worker/dir/uv"
     for key in envvars:
         assert env[key] == "/builds/worker/file"
+
+
+def test_git_checkout_extra_refs(mock_stdin, run_task_mod, mock_git_repo, tmp_path):
+    """extra_refs are fetched into the local repo during checkout."""
+    # Add a notes ref to the source repo
+    rev = mock_git_repo["main"][-1]
+    subprocess.check_call(
+        ["git", "notes", "--ref=refs/notes/taskgraph", "add", "-m", "test", rev],
+        cwd=mock_git_repo["path"],
+    )
+
+    destination = tmp_path / "destination"
+    run_task_mod.git_checkout(
+        destination_path=str(destination),
+        head_repo=mock_git_repo["path"],
+        base_repo=mock_git_repo["path"],
+        base_rev=None,
+        head_ref="main",
+        head_rev=None,
+        ssh_key_file=None,
+        ssh_known_hosts_file=None,
+        extra_refs=["refs/notes/taskgraph"],
+    )
+
+    # Verify the notes ref is available locally
+    result = subprocess.run(
+        ["git", "notes", "--ref=refs/notes/taskgraph", "show", rev],
+        cwd=str(destination),
+        capture_output=True,
+        text=True,
+    )
+    assert result.returncode == 0
+    assert "test" in result.stdout

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-21.0.0}/test/test_util_vcs.py

@@ -3,6 +3,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os
+import shutil
 import subprocess
 from pathlib import Path
 from textwrap import dedent
@@ -648,3 +649,23 @@ def test_get_changed_files_with_null_base_revision_shallow_clone(
     assert "first_file" in changed_files
     assert "file1.txt" in changed_files
     assert "file2.txt" in changed_files
+
+
+def test_get_note_git(git_repo, tmpdir):
+    """get_note returns note content when present, None otherwise."""
+    repo_path = tmpdir.join("git")
+    shutil.copytree(git_repo, repo_path)
+    repo = get_repository(str(repo_path))
+
+    # No note yet
+    assert repo.get_note("try-config") is None
+
+    rev = repo.head_rev
+    subprocess.check_call(
+        ["git", "notes", "--ref=refs/notes/try-config", "add", "-m", "test note", rev],
+        cwd=repo.path,
+    )
+
+    assert repo.get_note("try-config") == "test note"
+    assert repo.get_note("try-config", rev) == "test note"
+    assert repo.get_note("other") is None

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-21.0.0}/test/test_util_verify.py

@@ -211,6 +211,17 @@ def make_task_treeherder(label, symbol, platform="linux/opt"):
             pytest.raises(DeprecationWarning),
             id="routes_notfication_filter: deprecated",
         ),
+        pytest.param(
+            "verify_index_route",
+            make_graph(
+                make_task(
+                    "valid_route",
+                    task_def={"routes": ["index.example.v2.latest.taskgraph.decision"]},
+                ),
+            ),
+            does_not_raise(),
+            id="verify_index_route: valid route",
+        ),
         pytest.param(
             "verify_index_route",
             make_graph(
@@ -226,6 +237,28 @@ def make_task_treeherder(label, symbol, platform="linux/opt"):
             pytest.raises(Exception),
             id="verify_index_route: invalid slash in route",
         ),
+        pytest.param(
+            "verify_index_route",
+            make_graph(
+                make_task(
+                    "invalid_plus",
+                    task_def={"routes": ["index.example.1.0+hotfix1.latest"]},
+                ),
+            ),
+            pytest.raises(Exception),
+            id="verify_index_route: invalid plus in route",
+        ),
+        pytest.param(
+            "verify_index_route",
+            make_graph(
+                make_task(
+                    "invalid_space",
+                    task_def={"routes": ["index.example.some namespace.latest"]},
+                ),
+            ),
+            pytest.raises(Exception),
+            id="verify_index_route: invalid space in route",
+        ),
         pytest.param(
             "verify_task_dependencies",
             make_graph(

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-21.0.0}/uv.lock

@@ -2587,7 +2587,7 @@ wheels = [
 
 [[package]]
 name = "taskcluster-taskgraph"
-version = "20.0.0"
+version = "21.0.0"
 source = { editable = "." }
 dependencies = [
     { name = "appdirs" },