taskcluster-taskgraph 5.7.0__tar.gz → 6.0.0__tar.gz

{taskcluster-taskgraph-5.7.0/src/taskcluster_taskgraph.egg-info → taskcluster-taskgraph-6.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: taskcluster-taskgraph
-Version: 5.7.0
+Version: 6.0.0
 Summary: Build taskcluster taskgraphs
 Home-page: https://github.com/taskcluster/taskgraph
 Classifier: Development Status :: 5 - Production/Stable

{taskcluster-taskgraph-5.7.0 → taskcluster-taskgraph-6.0.0/src/taskcluster_taskgraph.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: taskcluster-taskgraph
-Version: 5.7.0
+Version: 6.0.0
 Summary: Build taskcluster taskgraphs
 Home-page: https://github.com/taskcluster/taskgraph
 Classifier: Development Status :: 5 - Production/Stable

{taskcluster-taskgraph-5.7.0 → taskcluster-taskgraph-6.0.0}/src/taskcluster_taskgraph.egg-info/SOURCES.txt

@@ -57,6 +57,7 @@ src/taskgraph/transforms/fetch.py
 src/taskgraph/transforms/from_deps.py
 src/taskgraph/transforms/notify.py
 src/taskgraph/transforms/task.py
+src/taskgraph/transforms/task_context.py
 src/taskgraph/transforms/job/__init__.py
 src/taskgraph/transforms/job/common.py
 src/taskgraph/transforms/job/index_search.py
@@ -104,6 +105,7 @@ test/test_scripts_run_task.py
 test/test_target_tasks.py
 test/test_taskgraph.py
 test/test_transform_docker_image.py
+test/test_transform_task_context.py
 test/test_transforms_base.py
 test/test_transforms_fetch.py
 test/test_transforms_from_deps.py

{taskcluster-taskgraph-5.7.0 → taskcluster-taskgraph-6.0.0}/src/taskgraph/__init__.py

@@ -2,7 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-__version__ = "5.7.0"
+__version__ = "6.0.0"
 
 # Maximum number of dependencies a single task can have
 # https://docs.taskcluster.net/reference/platform/taskcluster-queue/references/api#createTask

{taskcluster-taskgraph-5.7.0 → taskcluster-taskgraph-6.0.0}/src/taskgraph/transforms/from_deps.py

@@ -18,7 +18,7 @@ from voluptuous import Any, Extra, Optional, Required
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.attributes import attrmatch
 from taskgraph.util.dependencies import GROUP_BY_MAP
-from taskgraph.util.schema import Schema
+from taskgraph.util.schema import Schema, validate_schema
 
 FROM_DEPS_SCHEMA = Schema(
     {
@@ -139,7 +139,9 @@ def from_deps(config, tasks):
             group_by, arg = group_by.popitem()
             func = GROUP_BY_MAP[group_by]
             if func.schema:
-                func.schema(arg)
+                validate_schema(
+                    func.schema, arg, f"Invalid group-by {group_by} argument"
+                )
             groups = func(config, deps, arg)
         else:
             func = GROUP_BY_MAP[group_by]

{taskcluster-taskgraph-5.7.0 → taskcluster-taskgraph-6.0.0}/src/taskgraph/transforms/job/run_task.py

@@ -8,14 +8,13 @@ Support for running jobs that are invoked via the `run-task` script.
 import dataclasses
 import os
 
-from voluptuous import Any, Extra, Optional, Required
+from voluptuous import Any, Optional, Required
 
 from taskgraph.transforms.job import run_job_using
 from taskgraph.transforms.job.common import support_vcs_checkout
 from taskgraph.transforms.task import taskref_or_string
 from taskgraph.util import path, taskcluster
 from taskgraph.util.schema import Schema
-from taskgraph.util.yaml import load_yaml
 
 EXEC_COMMANDS = {
     "bash": ["bash", "-cx"],
@@ -46,16 +45,6 @@ run_task_schema = Schema(
         # it will be included in a single argument to the command specified by
         # `exec-with`.
         Required("command"): Any([taskref_or_string], taskref_or_string),
-        # Context to substitute into the command using format string
-        # substitution (e.g {value}). This is useful if certain aspects of the
-        # command need to be generated in transforms.
-        Optional("command-context"): {
-            # If present, loads a set of context variables from an unnested yaml
-            # file. If a value is present in both the provided file and directly
-            # in command-context, the latter will take priority.
-            Optional("from-file"): str,
-            Extra: object,
-        },
         # What to execute the command with in the event command is a string.
         Optional("exec-with"): Any(*list(EXEC_COMMANDS)),
         # Command used to invoke the `run-task` script. Can be used if the script
@@ -137,25 +126,6 @@ def script_url(config, script):
     return f"{tc_url}/api/queue/v1/task/{task_id}/artifacts/public/{script}"
 
 
-def substitute_command_context(command_context, command):
-    from_file = command_context.pop("from-file", None)
-    full_context = {}
-    if from_file:
-        full_context = load_yaml(from_file)
-    else:
-        full_context = {}
-
-    full_context.update(command_context)
-
-    if isinstance(command, list):
-        for i in range(len(command)):
-            command[i] = command[i].format(**full_context)
-    else:
-        command = command.format(**full_context)
-
-    return command
-
-
 @run_job_using(
     "docker-worker", "run-task", schema=run_task_schema, defaults=worker_defaults
 )
@@ -177,13 +147,6 @@ def docker_worker_run_task(config, job, taskdesc):
 
     run_command = run["command"]
 
-    if run.get("command-context"):
-        run_command = substitute_command_context(
-            run.get("command-context"), run["command"]
-        )
-    else:
-        run_command = run["command"]
-
     # dict is for the case of `{'task-reference': str}`.
     if isinstance(run_command, str) or isinstance(run_command, dict):
         exec_cmd = EXEC_COMMANDS[run.pop("exec-with", "bash")]
@@ -250,11 +213,6 @@ def generic_worker_run_task(config, job, taskdesc):
         exec_cmd = EXEC_COMMANDS[run.pop("exec-with", "bash")]
         run_command = exec_cmd + [run_command]
 
-    if run.get("command-context"):
-        run_command = substitute_command_context(
-            run.get("command-context"), run_command
-        )
-
     if run["run-as-root"]:
         command.extend(("--user", "root", "--group", "root"))
     command.append("--")

taskcluster-taskgraph-6.0.0/src/taskgraph/transforms/task_context.py

@@ -0,0 +1,121 @@
+from textwrap import dedent
+
+from voluptuous import ALLOW_EXTRA, Any, Optional, Required
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.schema import Schema
+from taskgraph.util.templates import deep_get, substitute
+from taskgraph.util.yaml import load_yaml
+
+SCHEMA = Schema(
+    {
+        Required(
+            "task-context",
+            description=dedent(
+                """
+            `task-context` can be used to substitute values into any field in a
+            task with data that is not known until `taskgraph` runs.
+
+            This data can be provided via `from-parameters` or `from-file`,
+            which can pull in values from parameters and a defined yml file
+            respectively.
+
+            Data may also be provided directly in the `from-object` section of
+            `task-context`. This can be useful in `kinds` that define most of
+            their contents in `task-defaults`, but have some values that may
+            differ for various concrete `tasks` in the `kind`.
+
+            If the same key is found in multiple places the order of precedence
+            is as follows:
+              - Parameters
+              - `from-object` keys
+              - File
+
+            That is to say: parameters will always override anything else.
+
+            """.lstrip(),
+            ),
+        ): {
+            Optional(
+                "from-parameters",
+                description=dedent(
+                    """
+                Retrieve task context values from parameters. A single
+                parameter may be provided or a list of parameters in
+                priority order. The latter can be useful in implementing a
+                "default" value if some other parameter is not provided.
+                """.lstrip()
+                ),
+            ): {str: Any([str], str)},
+            Optional(
+                "from-file",
+                description=dedent(
+                    """
+                Retrieve task context values from a yaml file. The provided
+                file should usually only contain top level keys and values
+                (eg: nested objects will not be interpolated - they will be
+                substituted as text representations of the object).
+                """.lstrip()
+                ),
+            ): str,
+            Optional(
+                "from-object",
+                description="Key/value pairs to be used as task context",
+            ): object,
+            Required(
+                "substitution-fields",
+                description=dedent(
+                    """
+                A list of fields in the task to substitute the provided values
+                into.
+                """.lstrip()
+                ),
+            ): [str],
+        },
+    },
+    extra=ALLOW_EXTRA,
+)
+
+transforms = TransformSequence()
+transforms.add_validate(SCHEMA)
+
+
+@transforms.add
+def render_task(config, jobs):
+    for job in jobs:
+        sub_config = job.pop("task-context")
+        params_context = {}
+        for var, path in sub_config.pop("from-parameters", {}).items():
+            if isinstance(path, str):
+                params_context[var] = deep_get(config.params, path)
+            else:
+                for choice in path:
+                    value = deep_get(config.params, choice)
+                    if value is not None:
+                        params_context[var] = value
+                        break
+
+        file_context = {}
+        from_file = sub_config.pop("from-file", None)
+        if from_file:
+            file_context = load_yaml(from_file)
+
+        fields = sub_config.pop("substitution-fields")
+
+        subs = {}
+        subs.update(file_context)
+        # We've popped away the configuration; everything left in `sub_config` is
+        # substitution key/value pairs.
+        subs.update(sub_config.pop("from-object", {}))
+        subs.update(params_context)
+
+        # Now that we have our combined context, we can substitute.
+        for field in fields:
+            container, subfield = job, field
+            while "." in subfield:
+                f, subfield = subfield.split(".", 1)
+                container = container[f]
+
+            container[subfield] = substitute(container[subfield], **subs)
+
+        yield job

{taskcluster-taskgraph-5.7.0 → taskcluster-taskgraph-6.0.0}/src/taskgraph/util/dependencies.py

@@ -14,6 +14,9 @@ GROUP_BY_MAP = {}
 
 def group_by(name, schema=None):
     def wrapper(func):
+        assert (
+            name not in GROUP_BY_MAP
+        ), f"duplicate group_by function name {name} ({func} and {GROUP_BY_MAP[name]})"
         GROUP_BY_MAP[name] = func
         func.schema = schema
         return func

{taskcluster-taskgraph-5.7.0 → taskcluster-taskgraph-6.0.0}/src/taskgraph/util/templates.py

@@ -48,3 +48,33 @@ def merge(*objects):
     if len(objects) == 1:
         return copy.deepcopy(objects[0])
     return merge_to(objects[-1], merge(*objects[:-1]))
+
+
+def deep_get(dict_, field):
+    container, subfield = dict_, field
+    while "." in subfield:
+        f, subfield = subfield.split(".", 1)
+        if f not in container:
+            return None
+
+        container = container[f]
+
+    return container.get(subfield)
+
+
+def substitute(item, **subs):
+    if isinstance(item, list):
+        for i in range(len(item)):
+            item[i] = substitute(item[i], **subs)
+    elif isinstance(item, dict):
+        new_dict = {}
+        for k, v in item.items():
+            k = k.format(**subs)
+            new_dict[k] = substitute(v, **subs)
+        item = new_dict
+    elif isinstance(item, str):
+        item = item.format(**subs)
+    else:
+        item = item
+
+    return item

taskcluster-taskgraph-6.0.0/test/test_transform_task_context.py

@@ -0,0 +1,84 @@
+"""
+Tests for the 'fetch' transforms.
+"""
+
+import os.path
+from copy import deepcopy
+from pprint import pprint
+
+from taskgraph.transforms import task_context
+from taskgraph.transforms.base import TransformConfig
+
+from .conftest import FakeParameters
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+TASK_DEFAULTS = {
+    "description": "fake description {object} {file} {param} {object_and_file}"
+    "{object_and_param} {file_and_param} {object_file_and_param} {param_fallback}",
+    "name": "fake-task-name",
+    "task-context": {
+        "from-parameters": {
+            "param": "param",
+            "object_and_param": "object_and_param",
+            "file_and_param": "file_and_param",
+            "object_file_and_param": "object_file_and_param",
+            "param_fallback": ["missing-param", "default"],
+        },
+        "from-file": f"{here}/data/task_context.yml",
+        "from-object": {
+            "object": "object",
+            "object_and_param": "shouldn't be used",
+            "object_and_file": "object-overrides-file",
+            "object_file_and_param": "shouldn't be used",
+        },
+        "substitution-fields": [
+            "description",
+        ],
+    },
+}
+
+
+def test_transforms(request, run_transform, graph_config):
+    task = deepcopy(TASK_DEFAULTS)
+
+    params = FakeParameters(
+        {
+            "param": "param",
+            "object_and_param": "param-overrides-object",
+            "file_and_param": "param-overrides-file",
+            "object_file_and_param": "param-overrides-all",
+            "default": "default",
+            "base_repository": "http://hg.example.com",
+            "build_date": 0,
+            "build_number": 1,
+            "enable_always_target": True,
+            "head_repository": "http://hg.example.com",
+            "head_rev": "abcdef",
+            "head_ref": "default",
+            "level": "1",
+            "moz_build_date": 0,
+            "next_version": "1.0.1",
+            "owner": "some-owner",
+            "project": "some-project",
+            "pushlog_id": 1,
+            "repository_type": "hg",
+            "target_tasks_method": "test_method",
+            "tasks_for": "hg-push",
+            "try_mode": None,
+            "version": "1.0.0",
+        },
+    )
+    transform_config = TransformConfig(
+        "test", str(here), {}, params, {}, graph_config, write_artifacts=False
+    )
+
+    task = run_transform(task_context.transforms, task, config=transform_config)[0]
+    print("Dumping task:")
+    pprint(task, indent=2)
+
+    assert (
+        task["description"]
+        == "fake description object file param object-overrides-file"
+        "param-overrides-object param-overrides-file param-overrides-all default"
+    )

{taskcluster-taskgraph-5.7.0 → taskcluster-taskgraph-6.0.0}/test/test_transforms_job.py

@@ -14,15 +14,12 @@ from unittest.mock import patch
 import pytest
 
 # prevent pytest thinking this is a test
-from taskcluster_urls import test_root_url as _test_root_url
-
 from taskgraph.task import Task
 from taskgraph.transforms import job
 from taskgraph.transforms.job import run_task  # noqa: F401
 from taskgraph.transforms.job.common import add_cache
 from taskgraph.transforms.task import payload_builders
 from taskgraph.util.schema import Schema, validate_schema
-from taskgraph.util.taskcluster import get_root_url
 from taskgraph.util.templates import merge
 
 here = os.path.abspath(os.path.dirname(__file__))
@@ -88,50 +85,6 @@ def test_worker_caches(task, transform):
     validate_schema(partial_schema, taskdesc["worker"][key], "validation error")
 
 
-@pytest.mark.parametrize(
-    "workerfn", [fn for fn, *_ in job.registry["run-task"].values()]
-)
-@pytest.mark.parametrize(
-    "task",
-    (
-        {
-            "worker-type": "t-linux",
-            "run": {
-                "checkout": True,
-                "comm-checkout": False,
-                "command": "echo '{output}'",
-                "command-context": {"output": "hello", "extra": None},
-                "run-as-root": False,
-                "sparse-profile": False,
-                "tooltool-downloads": False,
-            },
-        },
-    ),
-)
-def test_run_task_command_context(task, transform, workerfn, monkeypatch):
-    if "TASKCLUSTER_ROOT_URL" not in os.environ:
-        monkeypatch.setenv("TASKCLUSTER_ROOT_URL", _test_root_url())
-    # Clear memoized function
-    get_root_url.clear()
-
-    config, job_, taskdesc, _ = transform(task)
-    job_ = deepcopy(job_)
-
-    def assert_cmd(expected):
-        cmd = taskdesc["worker"]["command"]
-        while isinstance(cmd, list):
-            cmd = cmd[-1]
-        assert cmd == expected
-
-    workerfn(config, job_, taskdesc)
-    assert_cmd("echo 'hello'")
-
-    job_copy = job_.copy()
-    del job_copy["run"]["command-context"]
-    workerfn(config, job_copy, taskdesc)
-    assert_cmd("echo '{output}'")
-
-
 def assert_use_fetches_toolchain_env(task):
     assert task["worker"]["env"]["FOO"] == "1"
 

{taskcluster-taskgraph-5.7.0 → taskcluster-taskgraph-6.0.0}/test/test_transforms_job_run_task.py

@@ -20,13 +20,7 @@ TASK_DEFAULTS = {
         "os": "linux",
         "env": {},
     },
-    "run": {
-        "using": "run-task",
-        "command": "echo hello {extra_string}",
-        "command-context": {
-            "from-file": f"{here}/data/command_context.yaml",
-        },
-    },
+    "run": {"using": "run-task", "command": "echo hello world"},
 }
 
 

{taskcluster-taskgraph-5.7.0 → taskcluster-taskgraph-6.0.0}/src/taskcluster_taskgraph.egg-info/requires.txt

@@ -1,8 +1,8 @@
+PyYAML>=5.3.1
 appdirs>=1.4
 cookiecutter~=2.1
 json-e>=2.7
 mozilla-repo-urls
-PyYAML>=5.3.1
 redo>=2.0
 requests>=2.25
 requests_unixsocket>=0.2