taskcluster-taskgraph 20.0.0__tar.gz → 22.0.0__tar.gz

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-22.0.0}/.pre-commit-config.yaml

@@ -13,13 +13,13 @@ repos:
         exclude: template
       - id: check-added-large-files
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.15.4
+    rev: v0.15.8
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
       - id: ruff-format
   - repo: https://github.com/astral-sh/uv-pre-commit
-    rev: 0.10.7
+    rev: 0.11.2
     hooks:
       - id: uv-lock
   - repo: https://github.com/adrienverge/yamllint.git
@@ -28,7 +28,7 @@ repos:
       - id: yamllint
         exclude: template
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.4.1
+    rev: v2.4.2
     hooks:
       - id: codespell
         entry: codespell -I .codespell-ignore-words.txt
@@ -48,6 +48,5 @@ exclude: |
   (?x)^(
     src/taskgraph/run-task/fetch-content |
     src/taskgraph/run-task/robustcheckout.py |
-    src/taskgraph/run-task/run-task |
-    taskcluster/scripts/external_tools
+    src/taskgraph/run-task/run-task
   )

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-22.0.0}/.taskcluster.yml

@@ -251,7 +251,13 @@ tasks:
                           - bash
                           - -cx
                           - $let:
-                                extraArgs: {$if: 'tasks_for == "cron"', then: '${cron.quoted_args}', else: ''}
+                                extraArgs:
+                                    $if: 'tasks_for == "cron"'
+                                    then: '${cron.quoted_args}'
+                                    else:
+                                        $if: 'tasks_for == "github-pull-request"'
+                                        then: '--allow-parameter-override'
+                                        else: ''
                             in:
                                 $if: 'tasks_for == "action" || tasks_for == "pr-action"'
                                 then: >

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-22.0.0}/CHANGELOG.md

@@ -1,5 +1,35 @@
 # Change Log
 
+## [22.0.0] - 2026-04-16
+
+### Fixed
+
+- Decision task now fetches `refs/notes/decision-parameters` itself
+
+### Removed
+
+- BREAKING CHANGE: Removed ability for `run-task` to fetch arbitrary extra refs
+- BREAKING CHANGE: Removed docker-worker feature: relengapi-proxy
+
+## [21.0.0] - 2026-04-13
+
+### Added
+
+- Ability for `run-task` to fetch arbitrary extra refs
+- Ability for Decision task to override params via Git notes
+
+### Fixed
+
+- Improved index route verification
+
+### Changed
+
+- BREAKING CHANGE: run-task and decision images now use Debian 13
+
+### Removed
+
+- BREAKING CHANGE: Removed docker-worker features: dind, privileged, loopback-audio
+
 ## [20.0.0] - 2026-03-17
 
 ### Added

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-22.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: taskcluster-taskgraph
-Version: 20.0.0
+Version: 22.0.0
 Summary: Build taskcluster taskgraphs
 Project-URL: Repository, https://github.com/taskcluster/taskgraph
 Project-URL: Issues, https://github.com/taskcluster/taskgraph/issues

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-22.0.0}/docs/howto/bootstrap-taskgraph.rst

@@ -101,7 +101,7 @@ in addition to ``taskcluster-taskgraph`` we also want to depend on
    EOF
 
    # This works best if you use the same Python as the one used in the Decision
-   # image (currently 3.11).
+   # image (currently 3.13).
    pip-compile --generate-hashes --output-file requirements.txt requirements.in
 
 .. note::

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-22.0.0}/docs/reference/migrations.rst

@@ -3,6 +3,23 @@ Migration Guide
 
 This page can help when migrating Taskgraph across major versions.
 
+21.x -> 22.x
+------------
+
+* If you have tasks using `docker-worker`'s `relengapi-proxy` feature, it is no
+  longer supported. Remove all references to it in your task definitions.
+* Remove references to `<REPO>_EXTRA_REFS` in all `run-task` based tasks. These
+  refs must now be fetched as part of the task payload.
+
+20.x -> 21.x
+------------
+
+* If you have tasks using `docker-worker`'s `dind`, `privileged` or `loopback-audio`
+  features, they are no longer supported. Either avoid using these features or migrate
+  to a `generic-worker` pool with D2G payloads enabled.
+* Ensure any task's using the `run-task` or `decision` images work with the new
+  Debian 13 base.
+
 19.x -> 20.x
 ------------
 

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-22.0.0}/pyproject.toml

@@ -1,7 +1,7 @@
 ### Project
 [project]
 name = "taskcluster-taskgraph"
-version = "20.0.0"
+version = "22.0.0"
 description = "Build taskcluster taskgraphs"
 readme = "README.rst"
 authors = [
@@ -94,7 +94,6 @@ source = ["src/taskgraph/", "src/taskgraph/run-task/", "packages", "test"]
 ### Lint and Format
 [tool.ruff]
 extend-exclude = [
-  "taskcluster/scripts/external_tools",
   "src/taskgraph/run-task/robustcheckout.py"
 ]
 line-length = 88

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-22.0.0}/src/taskgraph/decision.py

@@ -264,6 +264,20 @@ def get_decision_parameters(graph_config, options):
     ] == "github-pull-request":
         set_try_config(parameters, task_config_file)
 
+    # load extra parameters from vcs note if able
+    note_ref = "refs/notes/decision-parameters"
+    if options.get("allow_parameter_override") and (
+        note_params := repo.get_note(note_ref, parameters["head_repository"])
+    ):
+        try:
+            note_params = json.loads(note_params)
+            logger.info(
+                f"Overriding parameters from {note_ref}:\n{json.dumps(note_params, indent=2)}"
+            )
+            parameters.update(note_params)
+        except ValueError as e:
+            raise Exception(f"Failed to parse {note_ref} as JSON: {e}") from e
+
     result = Parameters(**parameters)
     result.check()
     return result

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-22.0.0}/src/taskgraph/main.py

@@ -933,6 +933,12 @@ def load_task(args):
 @argument(
     "--verbose", "-v", action="store_true", help="include debug-level logging output"
 )
+@argument(
+    "--allow-parameter-override",
+    default=False,
+    action="store_true",
+    help="Allow user to override computed decision task parameters.",
+)
 def decision(options):
     from taskgraph.decision import taskgraph_decision  # noqa: PLC0415
 

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-22.0.0}/src/taskgraph/transforms/task.py

@@ -268,14 +268,10 @@ class DockerWorkerPayloadSchema(Schema, forbid_unknown_fields=False, kw_only=Tru
     # image or in-tree docker image to run the task in.
     docker_image: DockerImage
     # worker features that should be enabled
-    relengapi_proxy: bool
     chain_of_trust: bool
     taskcluster_proxy: bool
     allow_ptrace: bool
     loopback_video: bool
-    loopback_audio: bool
-    docker_in_docker: bool  # (aka 'dind')
-    privileged: bool
     # environment variables
     env: dict[str, taskref_or_string_msgspec]
     # the maximum time to run, in seconds
@@ -343,9 +339,6 @@ def build_docker_worker_payload(config, task, task_def):
 
     features = {}
 
-    if worker.get("relengapi-proxy"):
-        features["relengAPIProxy"] = True
-
     if worker.get("taskcluster-proxy"):
         features["taskclusterProxy"] = True
 
@@ -356,9 +349,6 @@ def build_docker_worker_payload(config, task, task_def):
     if worker.get("chain-of-trust"):
         features["chainOfTrust"] = True
 
-    if worker.get("docker-in-docker"):
-        features["dind"] = True
-
     if task.get("needs-sccache"):
         features["taskclusterProxy"] = True
         task_def["scopes"].append(
@@ -375,16 +365,10 @@ def build_docker_worker_payload(config, task, task_def):
 
     capabilities = {}
 
-    for lo in "audio", "video":
-        if worker.get("loopback-" + lo):
-            capitalized = "loopback" + lo.capitalize()
-            devices = capabilities.setdefault("devices", {})
-            devices[capitalized] = True
-            task_def["scopes"].append("docker-worker:capability:device:" + capitalized)
-
-    if worker.get("privileged"):
-        capabilities["privileged"] = True
-        task_def["scopes"].append("docker-worker:capability:privileged")
+    if worker.get("loopback-video"):
+        devices = capabilities.setdefault("devices", {})
+        devices["loopbackVideo"] = True
+        task_def["scopes"].append("docker-worker:capability:device:loopbackVideo")
 
     task_def["payload"] = payload = {
         "image": image,
@@ -810,14 +794,10 @@ def set_defaults(config, tasks):
 
         worker = task["worker"]
         if worker["implementation"] in ("docker-worker",):
-            worker.setdefault("relengapi-proxy", False)
             worker.setdefault("chain-of-trust", False)
             worker.setdefault("taskcluster-proxy", False)
             worker.setdefault("allow-ptrace", False)
             worker.setdefault("loopback-video", False)
-            worker.setdefault("loopback-audio", False)
-            worker.setdefault("docker-in-docker", False)
-            worker.setdefault("privileged", False)
             worker.setdefault("volumes", [])
             worker.setdefault("env", {})
             if "caches" in worker:

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-22.0.0}/src/taskgraph/util/vcs.py

@@ -215,6 +215,22 @@ class Repository(ABC):
         If this function returns an unexpected value, then make sure
         the revision was fetched from the remote repository."""
 
+    def get_note(
+        self,
+        note: str,
+        remote: str,
+        revision: Optional[str] = None,
+    ) -> Optional[str]:
+        """Read a note attached to the given revision (defaults to HEAD).
+
+        The notes ref is fetched from ``remote`` first; a missing ref on the
+        remote is tolerated while other fetch failures propagate.
+
+        Returns the note content as a string, or ``None`` if no note exists.
+        Only supported by Git; returns ``None`` for all other VCS types.
+        """
+        return None
+
 
 class HgRepository(Repository):
     @property
@@ -586,6 +602,42 @@ class GitRepository(Repository):
                 return False
             raise
 
+    def get_note(
+        self,
+        note: str,
+        remote: str,
+        revision: Optional[str] = None,
+    ) -> Optional[str]:
+        if not note.startswith("refs/notes/"):
+            note = f"refs/notes/{note}"
+
+        try:
+            self.run("fetch", remote, f"{note}:{note}", stderr=subprocess.PIPE)
+        except subprocess.CalledProcessError:
+            ls_output = self.run(
+                "ls-remote",
+                "--exit-code",
+                remote,
+                note,
+                return_codes=(2,),
+                stderr=subprocess.PIPE,
+            )
+            if not ls_output:
+                logger.info(
+                    "Failed to fetch %s but the remote doesn't have it. Ignoring.", note
+                )
+                return None
+            raise
+
+        try:
+            return self.run(
+                "notes", f"--ref={note}", "show", revision or "HEAD"
+            ).strip()
+        except subprocess.CalledProcessError as e:
+            if e.returncode == 1:
+                return None
+            raise
+
 
 def get_repository(path: str):
     """Get a repository object for the repository at `path`.

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-22.0.0}/src/taskgraph/util/verify.py

@@ -235,10 +235,14 @@ def verify_routes_notification_filters(
                 )
 
 
+# https://docs.taskcluster.net/docs/reference/core/index#valid-characters
+INDEX_NAMESPACE_RE = re.compile(r"^([a-zA-Z0-9_!~*'()%-]+\.)*[a-zA-Z0-9_!~*'()%-]+$")
+
+
 @verifications.add("full_task_graph")
 def verify_index_route(task, taskgraph, scratch_pad, graph_config, parameters):
     """
-    This function ensures that routes do not contain forward slashes.
+    This function ensures that index routes would create valid taskcluster index paths
     """
     if task is None:
         return
@@ -247,11 +251,13 @@ def verify_index_route(task, taskgraph, scratch_pad, graph_config, parameters):
     route_prefix = "index."
 
     for route in routes:
-        # Check for invalid / in the index route
-        if route.startswith(route_prefix) and "/" in route:
-            raise Exception(
-                f"{task.label} has invalid route with forward slash: {route}"
-            )
+        if route.startswith(route_prefix):
+            namespace = route[len(route_prefix) :]
+            if not INDEX_NAMESPACE_RE.match(namespace):
+                raise Exception(
+                    f"{task.label} has invalid index route namespace: {route}. "
+                    f"Namespace must match {INDEX_NAMESPACE_RE.pattern}"
+                )
 
 
 @verifications.add("full_task_graph")

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-22.0.0}/taskcluster/docker/run-task/Dockerfile

@@ -1,4 +1,4 @@
-FROM debian:12-slim
+FROM debian:13-slim
 LABEL maintainer="Release Engineering <release+taskgraph-ci@mozilla.com>"
 
 # Add worker user

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-22.0.0}/test/conftest.py

@@ -1,5 +1,6 @@
 import os
 import platform
+import tempfile
 from pathlib import Path
 
 import pytest
@@ -13,10 +14,22 @@ pytest_plugins = ("pytest-taskgraph",)
 # interference with tests.
 # This ignores ~/.hgrc
 os.environ["HGRCPATH"] = ""
-# This ignores system-level git config (eg: in /etc). There apperas to be
-# no way to ignore ~/.gitconfig other than overriding $HOME, which is overkill.
+# This ignores system-level git config (eg: in /etc).
 os.environ["GIT_CONFIG_NOSYSTEM"] = "1"
 
+
+@pytest.fixture(scope="session", autouse=True)
+def git_global_config():
+    # run-task writes to the global git config, so point it at a real
+    # (writable) empty file instead of ~/.gitconfig.
+    fd, path = tempfile.mkstemp(prefix="taskgraph-gitconfig-")
+    os.close(fd)
+    os.environ["GIT_CONFIG_GLOBAL"] = path
+    yield
+    del os.environ["GIT_CONFIG_GLOBAL"]
+    os.unlink(path)
+
+
 # Some of the tests marked with this may be fixable on Windows; we should
 # look into these in more depth at some point.
 nowin = pytest.mark.skipif(

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-22.0.0}/test/test_decision.py

@@ -10,6 +10,8 @@ import tempfile
 import unittest
 from pathlib import Path
 
+import pytest
+
 from taskgraph import decision
 from taskgraph.util.vcs import GitRepository, HgRepository
 from taskgraph.util.yaml import load_yaml
@@ -137,3 +139,62 @@ class TestGetDecisionParameters(unittest.TestCase):
         self.options["tasks_for"] = "hg-push"
         params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, self.options)
         self.assertEqual(params["target_tasks_method"], "nothing")
+
+
+_BASE_OPTIONS = {
+    "base_repository": "https://hg.mozilla.org/mozilla-unified",
+    "base_rev": "aaaa",
+    "head_repository": "https://hg.mozilla.org/mozilla-central",
+    "head_rev": "bbbb",
+    "head_ref": "default",
+    "head_tag": "",
+    "project": "mozilla-central",
+    "pushlog_id": "1",
+    "pushdate": 0,
+    "repository_type": "git",
+    "owner": "nobody@mozilla.com",
+    "tasks_for": "github-push",
+    "level": "1",
+}
+
+
+@unittest.mock.patch.object(
+    GitRepository,
+    "get_note",
+    return_value=json.dumps({"build_number": 99}),
+)
+@unittest.mock.patch.object(GitRepository, "get_changed_files", return_value=[])
+def test_decision_parameters_note(mock_files_changed, mock_get_note):
+    options = {**_BASE_OPTIONS, "allow_parameter_override": True}
+    params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, options)
+    mock_get_note.assert_called_once_with(
+        "refs/notes/decision-parameters", _BASE_OPTIONS["head_repository"]
+    )
+    assert params["build_number"] == 99
+
+
+@unittest.mock.patch.object(
+    GitRepository,
+    "get_note",
+    return_value=json.dumps({"build_number": 99}),
+)
+@unittest.mock.patch.object(GitRepository, "get_changed_files", return_value=[])
+def test_decision_parameters_note_disallow_override(mock_files_changed, mock_get_note):
+    options = {**_BASE_OPTIONS, "allow_parameter_override": False}
+    params = decision.get_decision_parameters(FAKE_GRAPH_CONFIG, options)
+    mock_get_note.assert_not_called()
+    assert params["build_number"] == 1
+
+
+@unittest.mock.patch.object(
+    GitRepository,
+    "get_note",
+    return_value="not valid json {",
+)
+@unittest.mock.patch.object(GitRepository, "get_changed_files", return_value=[])
+def test_decision_parameters_note_invalid_json(mock_files_changed, mock_get_note):
+    options = {**_BASE_OPTIONS, "allow_parameter_override": True}
+    with pytest.raises(
+        Exception, match="Failed to parse refs/notes/decision-parameters as JSON"
+    ):
+        decision.get_decision_parameters(FAKE_GRAPH_CONFIG, options)

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-22.0.0}/test/test_util_vcs.py

@@ -3,6 +3,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os
+import shutil
 import subprocess
 from pathlib import Path
 from textwrap import dedent
@@ -648,3 +649,25 @@ def test_get_changed_files_with_null_base_revision_shallow_clone(
     assert "first_file" in changed_files
     assert "file1.txt" in changed_files
     assert "file2.txt" in changed_files
+
+
+def test_get_note_git(git_repo, tmpdir):
+    """get_note returns note content when present, None otherwise."""
+    src_path = str(tmpdir.join("git-src"))
+    dst_path = str(tmpdir.join("git-dst"))
+    shutil.copytree(git_repo, src_path)
+    shutil.copytree(git_repo, dst_path)
+    repo = get_repository(dst_path)
+
+    # No note yet on the remote
+    assert repo.get_note("try-config", src_path) is None
+
+    rev = repo.head_rev
+    subprocess.check_call(
+        ["git", "notes", "--ref=refs/notes/try-config", "add", "-m", "test note", rev],
+        cwd=src_path,
+    )
+
+    assert repo.get_note("try-config", src_path) == "test note"
+    assert repo.get_note("try-config", src_path, revision=rev) == "test note"
+    assert repo.get_note("other", src_path) is None

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-22.0.0}/test/test_util_verify.py

@@ -211,6 +211,17 @@ def make_task_treeherder(label, symbol, platform="linux/opt"):
             pytest.raises(DeprecationWarning),
             id="routes_notfication_filter: deprecated",
         ),
+        pytest.param(
+            "verify_index_route",
+            make_graph(
+                make_task(
+                    "valid_route",
+                    task_def={"routes": ["index.example.v2.latest.taskgraph.decision"]},
+                ),
+            ),
+            does_not_raise(),
+            id="verify_index_route: valid route",
+        ),
         pytest.param(
             "verify_index_route",
             make_graph(
@@ -226,6 +237,28 @@ def make_task_treeherder(label, symbol, platform="linux/opt"):
             pytest.raises(Exception),
             id="verify_index_route: invalid slash in route",
         ),
+        pytest.param(
+            "verify_index_route",
+            make_graph(
+                make_task(
+                    "invalid_plus",
+                    task_def={"routes": ["index.example.1.0+hotfix1.latest"]},
+                ),
+            ),
+            pytest.raises(Exception),
+            id="verify_index_route: invalid plus in route",
+        ),
+        pytest.param(
+            "verify_index_route",
+            make_graph(
+                make_task(
+                    "invalid_space",
+                    task_def={"routes": ["index.example.some namespace.latest"]},
+                ),
+            ),
+            pytest.raises(Exception),
+            id="verify_index_route: invalid space in route",
+        ),
         pytest.param(
             "verify_task_dependencies",
             make_graph(

{taskcluster_taskgraph-20.0.0 → taskcluster_taskgraph-22.0.0}/uv.lock

@@ -2587,7 +2587,7 @@ wheels = [
 
 [[package]]
 name = "taskcluster-taskgraph"
-version = "20.0.0"
+version = "22.0.0"
 source = { editable = "." }
 dependencies = [
     { name = "appdirs" },