taskcluster-taskgraph 23.1.0__tar.gz → 24.0.0__tar.gz

{taskcluster_taskgraph-23.1.0 → taskcluster_taskgraph-24.0.0}/.pre-commit-config.yaml

@@ -13,13 +13,13 @@ repos:
         exclude: template
       - id: check-added-large-files
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.15.12
+    rev: v0.15.15
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
       - id: ruff-format
   - repo: https://github.com/astral-sh/uv-pre-commit
-    rev: 0.11.8
+    rev: 0.11.17
     hooks:
       - id: uv-lock
   - repo: https://github.com/adrienverge/yamllint.git

{taskcluster_taskgraph-23.1.0 → taskcluster_taskgraph-24.0.0}/.taskcluster.yml

@@ -9,6 +9,8 @@ reporting: checks-v1
 autoCancelPreviousChecks: true
 policy:
     pullRequests: public_restricted
+hooks:
+    - name: lint/pre-commit-v1
 tasks:
     # NOTE: support for actions in ci-admin requires that the `tasks` property be an array *before* JSON-e rendering
     # takes place.

{taskcluster_taskgraph-23.1.0 → taskcluster_taskgraph-24.0.0}/CHANGELOG.md

@@ -1,5 +1,22 @@
 # Change Log
 
+## [24.0.0] - 2026-06-03
+
+### Changed
+
+- BREAKING CHANGE: Must pass in `--allow-parameter-override` if using try_task_config
+- BREAKING CHANGE: Default level 1 artifact expiry changed from 1 year to 28 days
+
+### Removed
+
+- BREAKING CHANGE: Removed `Parameters.is_try()` helper method
+
+## [23.2.0] - 2026-06-03
+
+### Added
+
+- `task-context` now calls `resolve_keyed_by` on `substitution-fields` (#968)
+
 ## [23.1.0] - 2026-05-26
 
 ### Added

{taskcluster_taskgraph-23.1.0 → taskcluster_taskgraph-24.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: taskcluster-taskgraph
-Version: 23.1.0
+Version: 24.0.0
 Summary: Build taskcluster taskgraphs
 Project-URL: Repository, https://github.com/taskcluster/taskgraph
 Project-URL: Issues, https://github.com/taskcluster/taskgraph/issues

{taskcluster_taskgraph-23.1.0 → taskcluster_taskgraph-24.0.0}/docs/reference/migrations.rst

@@ -3,6 +3,17 @@ Migration Guide
 
 This page can help when migrating Taskgraph across major versions.
 
+23.x -> 24.x
+------------
+
+* You must now pass in `--allow-parameter-override` into Decision tasks that
+  need to load the `try_task_config.json` file at the root of the repo.
+* `taskgraph.parameters.Parameters.is_try()` is removed. Either in-line the old
+  method to a utility or use alternative logic.
+* Level 1 artifacts' default expiry changed from "1 year" to "28 days". If
+  needed, set `task-expires-after` in `taskcluster/config.yml` to adjust the
+  default back.
+
 22.x -> 23.x
 ------------
 

{taskcluster_taskgraph-23.1.0 → taskcluster_taskgraph-24.0.0}/docs/reference/transforms/task_context.rst

@@ -4,8 +4,8 @@ Task Context Transforms
 =======================
 
 The :mod:`taskgraph.transforms.task_context` transform can be used to
-substitute values into any field in a task with data that is not known
-until ``taskgraph`` runs.
+resolve keys and/or substitute values into any field in a task with data
+that is not known until ``taskgraph`` runs.
 
 This data can be provided in a few ways, as described below.
 
@@ -37,7 +37,10 @@ Then create a ``task-context`` section in your task definition, e.g:
 
    tasks:
      build:
-       description: my description {foo}
+       description:
+        by-foo:
+          bar: super special description for bar tasks
+          default: my description of {foo}
        task-context:
          from-parameters:
            foo: foo
@@ -50,9 +53,30 @@ are used:
 
 .. code-block:: yaml
 
-   foo: with some extra
+   foo: bar
+
+...the description will end up with a value of "super special description for bar tasks".
 
-...the description will end up with a value of "my description with some extra".
+When ``foo`` is set to any other value (eg: ``bar``) the description will end with the
+``default`` value with the value of ``foo`` substituted in (eg: ``my description of bar``).
+
+This pattern is often used to configure things vary based on how tasks are created. For example, the following can be used to adjust the scopes a task is given depending on the GitHub event that created the decision task:
+
+.. code-block:: yaml
+
+   tasks:
+     build:
+       task-context:
+         from-parameters:
+           tasks_for: tasks_for
+         substitution-fields:
+           - scopes
+
+       scopes:
+         by-tasks-for:
+           github-push:
+             - secrets:get:project/foo/api_token
+           default: []
 
 When using ``from-parameters`` you may also provide an ordered list of keys to
 look for in the parameters, with the first one found being used. For example,
@@ -154,8 +178,11 @@ Finally, the name of the task is added to the context implicitly. For example:
 This will evaluate the description correctly, even though there are no
 ``task-context`` keys defined on the individual tasks.
 
-Precedence
-----------
+Order of Operations & Precedence
+--------------------------------
+
+Keys will be resolved on ``substitution-fields`` first, then substitution
+will be performed on the resolved value.
 
 If the same key is found in multiple places the order of precedence is as
 follows: ``from-parameters``, ``from-object`` keys, ``from-file`` and finally

{taskcluster_taskgraph-23.1.0 → taskcluster_taskgraph-24.0.0}/packages/pytest-taskgraph/src/pytest_taskgraph/fixtures/gen.py

@@ -123,9 +123,6 @@ def graph_config(datadir):
 class FakeParameters(dict):
     strict = True
 
-    def is_try(self):
-        return self["level"] != 3
-
     def file_url(self, path, pretty=False):
         return path
 

{taskcluster_taskgraph-23.1.0 → taskcluster_taskgraph-24.0.0}/pyproject.toml

@@ -1,7 +1,7 @@
 ### Project
 [project]
 name = "taskcluster-taskgraph"
-version = "23.1.0"
+version = "24.0.0"
 description = "Build taskcluster taskgraphs"
 readme = "README.rst"
 authors = [

{taskcluster_taskgraph-23.1.0 → taskcluster_taskgraph-24.0.0}/src/taskgraph/decision.py

@@ -252,31 +252,27 @@ def get_decision_parameters(graph_config, options):
         assert callable(decision_params)
         decision_params(graph_config, parameters)
 
-    if options.get("try_task_config_file"):
-        task_config_file = os.path.abspath(options.get("try_task_config_file"))
-    else:
-        # if try_task_config.json is present, load it
-        task_config_file = os.path.join(os.getcwd(), "try_task_config.json")
-
-    # load try settings
-    if ("try" in project and options["tasks_for"] == "hg-push") or options[
-        "tasks_for"
-    ] == "github-pull-request":
-        set_try_config(parameters, task_config_file)
-
-    # load extra parameters from vcs note if able
-    note_ref = "refs/notes/decision-parameters"
-    if options.get("allow_parameter_override") and (
-        note_params := repo.get_note(note_ref, parameters["head_repository"])
-    ):
-        try:
-            note_params = json.loads(note_params)
-            logger.info(
-                f"Overriding parameters from {note_ref}:\n{json.dumps(note_params, indent=2)}"
-            )
-            parameters.update(note_params)
-        except ValueError as e:
-            raise Exception(f"Failed to parse {note_ref} as JSON: {e}") from e
+    # load extra parameters from file or vcs note if able
+    if options.get("allow_parameter_override"):
+        note_ref = "refs/notes/decision-parameters"
+        if options.get("try_task_config_file"):
+            task_config_file = os.path.abspath(options.get("try_task_config_file"))
+        else:
+            # if try_task_config.json is present, load it
+            task_config_file = os.path.join(os.getcwd(), "try_task_config.json")
+
+        if os.path.isfile(task_config_file):
+            set_try_config(parameters, task_config_file)
+
+        elif note_params := repo.get_note(note_ref, parameters["head_repository"]):
+            try:
+                note_params = json.loads(note_params)
+                logger.info(
+                    f"Overriding parameters from {note_ref}:\n{json.dumps(note_params, indent=2)}"
+                )
+                parameters.update(note_params)
+            except ValueError as e:
+                raise Exception(f"Failed to parse {note_ref} as JSON: {e}") from e
 
     result = Parameters(**parameters)
     result.check()
@@ -284,23 +280,22 @@ def get_decision_parameters(graph_config, options):
 
 
 def set_try_config(parameters, task_config_file):
-    if os.path.isfile(task_config_file):
-        logger.info(f"using try tasks from {task_config_file}")
-        with open(task_config_file) as fh:
-            task_config = json.load(fh)
-        task_config_version = task_config.pop("version")
-        if task_config_version == 2:
-            validate_schema(
-                try_task_config_schema_v2,
-                task_config,
-                "Invalid v2 `try_task_config.json`.",
-            )
-            parameters.update(task_config["parameters"])
-            return
-        else:
-            raise Exception(
-                f"Unknown `try_task_config.json` version: {task_config_version}"
-            )
+    logger.info(f"using try tasks from {task_config_file}")
+    with open(task_config_file) as fh:
+        task_config = json.load(fh)
+    task_config_version = task_config.pop("version")
+    if task_config_version == 2:
+        validate_schema(
+            try_task_config_schema_v2,
+            task_config,
+            "Invalid v2 `try_task_config.json`.",
+        )
+        parameters.update(task_config["parameters"])
+        return
+    else:
+        raise Exception(
+            f"Unknown `try_task_config.json` version: {task_config_version}"
+        )
 
 
 def write_artifact(filename, data):

{taskcluster_taskgraph-23.1.0 → taskcluster_taskgraph-24.0.0}/src/taskgraph/parameters.py

@@ -273,13 +273,6 @@ class Parameters(ReadOnlyDict):
         except KeyError:
             raise KeyError(f"taskgraph parameter {k!r} not found")
 
-    def is_try(self):
-        """
-        Determine whether this graph is being built on a try project or for
-        `mach try fuzzy`.
-        """
-        return "try" in self["project"] or self["tasks_for"] == "github-pull-request"
-
     @property
     def moz_build_date(self):
         # XXX self["moz_build_date"] is left as a string because:

{taskcluster_taskgraph-23.1.0 → taskcluster_taskgraph-24.0.0}/src/taskgraph/transforms/docker_image.py

@@ -130,7 +130,7 @@ def fill_template(config, tasks):
                 "artifact_prefix": "public",
             },
             "always-target": True,
-            "expires-after": expires if config.params.is_try() else "1 year",
+            "expires-after": expires if config.params["level"] == "1" else "1 year",
             "scopes": [],
             "run-on-projects": [],
             "worker-type": "images",

{taskcluster_taskgraph-23.1.0 → taskcluster_taskgraph-24.0.0}/src/taskgraph/transforms/task.py

@@ -457,12 +457,10 @@ def build_docker_worker_payload(config, task, task_def):
         else:
             suffix = cache_version
 
-        skip_untrusted = config.params.is_try() or level == 1
-
         for cache in worker["caches"]:
             # Some caches aren't enabled in environments where we can't
             # guarantee certain behavior. Filter those out.
-            if cache.get("skip-untrusted") and skip_untrusted:
+            if cache.get("skip-untrusted") and level == 1:
                 continue
 
             name = "{trust_domain}-level-{level}-{name}-{suffix}".format(
@@ -484,7 +482,7 @@ def build_docker_worker_payload(config, task, task_def):
     if run_task and worker.get("volumes"):
         payload["env"]["TASKCLUSTER_VOLUMES"] = ";".join(sorted(worker["volumes"]))
 
-    if payload.get("cache") and skip_untrusted:  # type: ignore
+    if payload.get("cache") and level == 1:  # type: ignore
         payload["env"]["TASKCLUSTER_UNTRUSTED_CACHES"] = "1"
 
     if features:
@@ -999,7 +997,7 @@ def build_task(config, tasks):
         if "expires-after" not in task:
             task["expires-after"] = (
                 config.graph_config._config.get("task-expires-after", "28 days")
-                if config.params.is_try()
+                if config.params["level"] == "1"
                 else "1 year"
             )
 

{taskcluster_taskgraph-23.1.0 → taskcluster_taskgraph-24.0.0}/src/taskgraph/transforms/task_context.py

@@ -1,14 +1,20 @@
 from typing import Optional, Union
 
 from taskgraph.transforms.base import TransformSequence
-from taskgraph.util.schema import Schema
+from taskgraph.util.schema import Schema, resolve_keyed_by
 from taskgraph.util.templates import deep_get, substitute_task_fields
 from taskgraph.util.yaml import load_yaml
 
 
+class ResolveKeyedByConfigOptions(Schema):
+    # forwarded to `resolve_keyed_by`; see that function for details
+    defer: Optional[list[str]] = None
+    enforce_single_match: Optional[bool] = None
+
+
 class TaskContextConfig(Schema):
     # A list of fields in the task to substitute the provided values
-    # into.
+    # into and/or resolve keys in.
     substitution_fields: list[str]
     # Retrieve task context values from parameters. A single
     # parameter may be provided or a list of parameters in
@@ -22,13 +28,15 @@ class TaskContextConfig(Schema):
     from_file: Optional[str] = None
     # Key/value pairs to be used as task context
     from_object: Optional[object] = None
+    resolve_keyed_by_options: Optional[ResolveKeyedByConfigOptions] = None
 
 
 #: Schema for the task_context transforms
 class TaskContextSchema(Schema, forbid_unknown_fields=False, kw_only=True):
     name: Optional[str] = None
-    # `task-context` can be used to substitute values into any field in a
-    # task with data that is not known until `taskgraph` runs.
+    # `task-context` can be used to substitute values into any field or resolve
+    # keys in a field in a task with data that is not known until `taskgraph`
+    # runs.
     #
     # This data can be provided via `from-parameters` or `from-file`,
     # which can pull in values from parameters and a defined yml file
@@ -46,6 +54,9 @@ class TaskContextSchema(Schema, forbid_unknown_fields=False, kw_only=True):
     #   - File
     #
     # That is to say: parameters will always override anything else.
+    #
+    # Each entry in `substitution-fields` will first have `resolved_keyed_by`
+    # called on it, and then substitutions will be performed.
     task_context: Optional[TaskContextConfig] = None
 
 
@@ -89,6 +100,10 @@ def render_task(config, tasks):
         if "name" in task:
             subs.setdefault("name", task["name"])
 
-        # Now that we have our combined context, we can substitute.
+        # Now that we have our combined context, we can resolve keys and substitute.
+        task_name = task.get("name", task.get("label", "?"))
+        resolve_keyed_by_options = sub_config.get("resolve-keyed-by-options") or {}
+        for field in fields:
+            resolve_keyed_by(task, field, task_name, **resolve_keyed_by_options, **subs)
         substitute_task_fields(task, fields, **subs)
         yield task

taskcluster_taskgraph-24.0.0/test/test_transform_task_context.py

@@ -0,0 +1,222 @@
+"""
+Tests for the 'task_context' transforms.
+"""
+
+import os.path
+from copy import deepcopy
+from pprint import pprint
+
+import pytest
+from pytest_taskgraph import FakeParameters
+
+from taskgraph.transforms import task_context
+from taskgraph.transforms.base import TransformConfig
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+BASE_PARAMS = {
+    "base_repository": "http://hg.example.com",
+    "build_date": 0,
+    "build_number": 1,
+    "enable_always_target": True,
+    "head_repository": "http://hg.example.com",
+    "head_rev": "abcdef",
+    "head_ref": "default",
+    "level": "1",
+    "moz_build_date": 0,
+    "next_version": "1.0.1",
+    "owner": "some-owner",
+    "project": "some-project",
+    "pushlog_id": 1,
+    "repository_type": "hg",
+    "target_tasks_method": "test_method",
+    "tasks_for": "hg-push",
+    "try_mode": None,
+    "version": "1.0.0",
+}
+
+
+def make_config(graph_config, extra_params=None):
+    params_data = dict(BASE_PARAMS)
+    if extra_params:
+        params_data.update(extra_params)
+    params = FakeParameters(params_data)
+    return TransformConfig(
+        "test", str(here), {}, params, {}, graph_config, write_artifacts=False
+    )
+
+
+TASK_DEFAULTS = {
+    "description": "fake description {object} {file} {param} {object_and_file} "
+    "{object_and_param} {file_and_param} {object_file_and_param} {param_fallback} {name}",
+    "name": "fake-task-name",
+    "task-context": {
+        "from-parameters": {
+            "param": "param",
+            "object_and_param": "object_and_param",
+            "file_and_param": "file_and_param",
+            "object_file_and_param": "object_file_and_param",
+            "param_fallback": ["missing-param", "default"],
+        },
+        "from-file": f"{here}/data/task_context.yml",
+        "from-object": {
+            "object": "object",
+            "object_and_param": "shouldn't be used",
+            "object_and_file": "object-overrides-file",
+            "object_file_and_param": "shouldn't be used",
+        },
+        "substitution-fields": [
+            "description",
+        ],
+    },
+}
+EXPECTED_DESCRIPTION = (
+    "fake description object file param object-overrides-file "
+    "param-overrides-object param-overrides-file param-overrides-all default fake-task-name"
+)
+NO_CONTEXT = deepcopy(TASK_DEFAULTS)
+del NO_CONTEXT["task-context"]
+
+
+@pytest.mark.parametrize(
+    "task,description",
+    (
+        pytest.param(deepcopy(TASK_DEFAULTS), EXPECTED_DESCRIPTION, id="with-context"),
+        pytest.param(
+            deepcopy(NO_CONTEXT), TASK_DEFAULTS["description"], id="no-context"
+        ),
+    ),
+)
+def test_transforms(request, run_transform, graph_config, task, description):
+    config = make_config(
+        graph_config,
+        {
+            "param": "param",
+            "object_and_param": "param-overrides-object",
+            "file_and_param": "param-overrides-file",
+            "object_file_and_param": "param-overrides-all",
+            "default": "default",
+        },
+    )
+
+    task = run_transform(task_context.transforms, task, config=config)[0]
+    print("Dumping task:")
+    pprint(task, indent=2)
+
+    assert task["description"] == description
+
+
+def test_resolve_keyed_by_from_parameters(run_transform, graph_config):
+    task = {
+        "name": "fake-task-name",
+        "description": {
+            "by-platform": {
+                "linux": "resolved-from-parameters",
+                "default": "default-value",
+            },
+        },
+        "task-context": {
+            "from-parameters": {"platform": "platform"},
+            "substitution-fields": ["description"],
+        },
+    }
+    config = make_config(graph_config, {"platform": "linux"})
+
+    task = run_transform(task_context.transforms, task, config=config)[0]
+    pprint(task, indent=2)
+
+    assert task["description"] == "resolved-from-parameters"
+
+
+def test_resolve_keyed_by_from_file(run_transform, graph_config):
+    task = {
+        "name": "fake-task-name",
+        "description": {
+            "by-file": {
+                "file": "resolved-from-file",
+                "default": "default-value",
+            },
+        },
+        "task-context": {
+            "from-file": f"{here}/data/task_context.yml",
+            "substitution-fields": ["description"],
+        },
+    }
+    config = make_config(graph_config)
+
+    task = run_transform(task_context.transforms, task, config=config)[0]
+    pprint(task, indent=2)
+
+    assert task["description"] == "resolved-from-file"
+
+
+def test_resolve_keyed_by_from_object(run_transform, graph_config):
+    task = {
+        "name": "fake-task-name",
+        "description": {
+            "by-flavor": {
+                "chocolate": "resolved-from-object",
+                "default": "default-value",
+            },
+        },
+        "task-context": {
+            "from-object": {"flavor": "chocolate"},
+            "substitution-fields": ["description"],
+        },
+    }
+    config = make_config(graph_config)
+
+    task = run_transform(task_context.transforms, task, config=config)[0]
+    pprint(task, indent=2)
+
+    assert task["description"] == "resolved-from-object"
+
+
+def test_resolve_keyed_by_missing_context_no_defer(run_transform, graph_config):
+    task = {
+        "name": "fake-task-name",
+        "description": {
+            "by-missing": {
+                "linux": "should-not-resolve",
+            },
+        },
+        "task-context": {
+            "substitution-fields": ["description"],
+        },
+    }
+    config = make_config(graph_config)
+
+    with pytest.raises(Exception):
+        run_transform(task_context.transforms, task, config=config)
+
+
+def test_resolve_keyed_by_missing_context_with_defer(run_transform, graph_config):
+    task = {
+        "name": "fake-task-name",
+        "description": {
+            "by-flavor": {
+                "chocolate": {
+                    "by-missing": {
+                        "linux": "should-not-resolve",
+                        "default": "also-should-not-resolve",
+                    },
+                },
+            },
+        },
+        "task-context": {
+            "from-object": {"flavor": "chocolate"},
+            "resolve-keyed-by-options": {"defer": ["missing"]},
+            "substitution-fields": ["description"],
+        },
+    }
+    config = make_config(graph_config)
+
+    task = run_transform(task_context.transforms, task, config=config)[0]
+    pprint(task, indent=2)
+
+    assert task["description"] == {
+        "by-missing": {
+            "linux": "should-not-resolve",
+            "default": "also-should-not-resolve",
+        },
+    }