taskcluster-taskgraph 8.2.0__tar.gz → 9.0.0__tar.gz

{taskcluster-taskgraph-8.2.0/src/taskcluster_taskgraph.egg-info → taskcluster-taskgraph-9.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: taskcluster-taskgraph
-Version: 8.2.0
+Version: 9.0.0
 Summary: Build taskcluster taskgraphs
 Home-page: https://github.com/taskcluster/taskgraph
 Classifier: Development Status :: 5 - Production/Stable

{taskcluster-taskgraph-8.2.0 → taskcluster-taskgraph-9.0.0}/pyproject.toml

@@ -13,6 +13,9 @@ branch = true
 source = ["src/taskgraph/", "taskgraph", "src/taskgraph/run-task/"]
 
 [tool.ruff]
+target-version = "py38"
+
+[tool.ruff.lint]
 select = [
     "E", "W",       # pycodestyle
     "F",            # pyflakes
@@ -25,9 +28,8 @@ ignore = [
     "E501",  # let black handle line-length
     "E741",
 ]
-target-version = "py38"
 
-[tool.ruff.isort]
+[tool.ruff.lint.isort]
 known-first-party = ["taskgraph"]
 
 [tool.pyright]

{taskcluster-taskgraph-8.2.0 → taskcluster-taskgraph-9.0.0/src/taskcluster_taskgraph.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: taskcluster-taskgraph
-Version: 8.2.0
+Version: 9.0.0
 Summary: Build taskcluster taskgraphs
 Home-page: https://github.com/taskcluster/taskgraph
 Classifier: Development Status :: 5 - Production/Stable

{taskcluster-taskgraph-8.2.0 → taskcluster-taskgraph-9.0.0}/src/taskcluster_taskgraph.egg-info/SOURCES.txt

@@ -67,11 +67,11 @@ src/taskgraph/util/__init__.py
 src/taskgraph/util/archive.py
 src/taskgraph/util/attributes.py
 src/taskgraph/util/cached_tasks.py
+src/taskgraph/util/copy.py
 src/taskgraph/util/dependencies.py
 src/taskgraph/util/docker.py
 src/taskgraph/util/hash.py
 src/taskgraph/util/keyed_by.py
-src/taskgraph/util/memoize.py
 src/taskgraph/util/parameterization.py
 src/taskgraph/util/path.py
 src/taskgraph/util/python_path.py
@@ -119,6 +119,7 @@ test/test_transforms_task.py
 test/test_util_archive.py
 test/test_util_attributes.py
 test/test_util_cached_tasks.py
+test/test_util_copy.py
 test/test_util_dependencies.py
 test/test_util_docker.py
 test/test_util_parameterization.py

{taskcluster-taskgraph-8.2.0 → taskcluster-taskgraph-9.0.0}/src/taskgraph/__init__.py

@@ -2,7 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-__version__ = "8.2.0"
+__version__ = "9.0.0"
 
 # Maximum number of dependencies a single task can have
 # https://docs.taskcluster.net/reference/platform/taskcluster-queue/references/api#createTask

{taskcluster-taskgraph-8.2.0 → taskcluster-taskgraph-9.0.0}/src/taskgraph/actions/registry.py

@@ -3,6 +3,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 
+import functools
 import json
 from collections import namedtuple
 from types import FunctionType
@@ -13,7 +14,6 @@ from taskgraph import create
 from taskgraph.config import load_graph_config
 from taskgraph.parameters import Parameters
 from taskgraph.util import hash, taskcluster, yaml
-from taskgraph.util.memoize import memoize
 from taskgraph.util.python_path import import_sibling_modules
 
 actions = []
@@ -31,13 +31,13 @@ def is_json(data):
     return True
 
 
-@memoize
+@functools.lru_cache(maxsize=None)
 def read_taskcluster_yml(filename):
-    """Load and parse .taskcluster.yml, memoized to save some time"""
+    """Load and parse .taskcluster.yml, cached to save some time"""
     return yaml.load_yaml(filename)
 
 
-@memoize
+@functools.lru_cache(maxsize=None)
 def hash_taskcluster_yml(filename):
     """
     Generate a hash of the given .taskcluster.yml.  This is the first 10 digits

{taskcluster-taskgraph-8.2.0 → taskcluster-taskgraph-9.0.0}/src/taskgraph/docker.py

@@ -102,7 +102,7 @@ def build_context(name, outputFile, args=None):
 
     image_dir = docker.image_path(name)
     if not os.path.isdir(image_dir):
-        raise Exception("image directory does not exist: %s" % image_dir)
+        raise Exception(f"image directory does not exist: {image_dir}")
 
     docker.create_context_tar(".", image_dir, outputFile, args)
 
@@ -117,7 +117,7 @@ def build_image(name, tag, args=None):
 
     image_dir = docker.image_path(name)
     if not os.path.isdir(image_dir):
-        raise Exception("image directory does not exist: %s" % image_dir)
+        raise Exception(f"image directory does not exist: {image_dir}")
 
     tag = tag or docker.docker_image(name, by_tag=True)
 

{taskcluster-taskgraph-8.2.0 → taskcluster-taskgraph-9.0.0}/src/taskgraph/transforms/base.py

@@ -3,6 +3,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 
+import functools
 import re
 from dataclasses import dataclass, field
 from typing import Dict, List, Union
@@ -11,7 +12,6 @@ from taskgraph.task import Task
 
 from ..config import GraphConfig
 from ..parameters import Parameters
-from ..util.memoize import memoize
 from ..util.schema import Schema, validate_schema
 
 
@@ -58,7 +58,7 @@ class TransformConfig:
     write_artifacts: bool
 
     @property
-    @memoize
+    @functools.lru_cache(maxsize=None)
     def repo_configs(self):
         repositories = self.graph_config["taskgraph"]["repositories"]
         if len(repositories) == 1:

{taskcluster-taskgraph-8.2.0 → taskcluster-taskgraph-9.0.0}/src/taskgraph/transforms/cached_tasks.py

@@ -77,7 +77,14 @@ def cache_task(config, tasks):
                         task["label"], p
                     )
                 )
+
         digest_data = cache["digest-data"] + sorted(dependency_digests)
+
+        # Chain of trust affects task artifacts therefore it should influence
+        # cache digest.
+        if task.get("worker", {}).get("chain-of-trust"):
+            digest_data.append(str(task["worker"]["chain-of-trust"]))
+
         add_optimization(
             config,
             task,

{taskcluster-taskgraph-8.2.0 → taskcluster-taskgraph-9.0.0}/src/taskgraph/transforms/fetch.py

@@ -128,7 +128,7 @@ def make_task(config, tasks):
             "name": name,
             "description": task["description"],
             "expires-after": task.get("expires-after", expires),
-            "label": "fetch-%s" % name,
+            "label": f"fetch-{name}",
             "run-on-projects": [],
             "run": {
                 "using": "run-task",
@@ -265,12 +265,7 @@ def create_fetch_url_task(config, name, fetch):
         for k, v in fetch["headers"].items():
             command.extend(["-H", f"{k}:{v}"])
 
-    command.extend(
-        [
-            fetch["url"],
-            "/builds/worker/artifacts/%s" % artifact_name,
-        ]
-    )
+    command.extend([fetch["url"], f"/builds/worker/artifacts/{artifact_name}"])
 
     return {
         "command": command,
@@ -316,7 +311,7 @@ def create_git_fetch_task(config, name, fetch):
         path_prefix,
         fetch["repo"],
         fetch["revision"],
-        "/builds/worker/artifacts/%s" % artifact_name,
+        f"/builds/worker/artifacts/{artifact_name}",
     ]
 
     ssh_key = fetch.get("ssh-key")

{taskcluster-taskgraph-8.2.0 → taskcluster-taskgraph-9.0.0}/src/taskgraph/transforms/run/toolchain.py

@@ -14,6 +14,7 @@ from taskgraph.transforms.run.common import (
     generic_worker_add_artifacts,
     get_vcsdir_name,
 )
+from taskgraph.util import path as mozpath
 from taskgraph.util.hash import hash_paths
 from taskgraph.util.schema import Schema
 from taskgraph.util.shell import quote as shell_quote
@@ -57,7 +58,8 @@ toolchain_run_schema = Schema(
 def get_digest_data(config, run, taskdesc):
     files = list(run.pop("resources", []))
     # The script
-    files.append("taskcluster/scripts/toolchain/{}".format(run["script"]))
+    script = mozpath.join("taskcluster/scripts/toolchain/", run["script"])
+    files.append(mozpath.normpath(script))
 
     # Accumulate dependency hashes for index generation.
     data = [hash_paths(config.graph_config.vcs_root, files)]
@@ -126,16 +128,14 @@ def common_toolchain(config, task, taskdesc, is_docker):
             "digest-data": get_digest_data(config, run, taskdesc),
         }
 
-    script = run.pop("script")
+    script = mozpath.join("taskcluster/scripts/toolchain/", run.pop("script"))
     run["using"] = "run-task"
     run["cwd"] = "{checkout}/.."
 
     if script.endswith(".ps1"):
         run["exec-with"] = "powershell"
 
-    command = [f"{srcdir}/taskcluster/scripts/toolchain/{script}"] + run.pop(
-        "arguments", []
-    )
+    command = [f"{srcdir}/{mozpath.normpath(script)}"] + run.pop("arguments", [])
 
     if not is_docker:
         # Don't quote the first item in the command because it purposely contains

{taskcluster-taskgraph-8.2.0 → taskcluster-taskgraph-9.0.0}/src/taskgraph/transforms/task.py

@@ -9,6 +9,7 @@ complexities of worker implementations, scopes, and treeherder annotations.
 """
 
 
+import functools
 import hashlib
 import os
 import re
@@ -23,7 +24,6 @@ from taskgraph import MAX_DEPENDENCIES
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.hash import hash_path
 from taskgraph.util.keyed_by import evaluate_keyed_by
-from taskgraph.util.memoize import memoize
 from taskgraph.util.schema import (
     OptimizationSchema,
     Schema,
@@ -43,7 +43,7 @@ RUN_TASK = os.path.join(
 )
 
 
-@memoize
+@functools.lru_cache(maxsize=None)
 def _run_task_suffix():
     """String to append to cache names under control of run-task."""
     return hash_path(RUN_TASK)[0:20]
@@ -214,14 +214,14 @@ def get_branch_rev(config):
     return config.params["head_rev"]
 
 
-@memoize
+@functools.lru_cache(maxsize=None)
 def get_default_priority(graph_config, project):
     return evaluate_keyed_by(
         graph_config["task-priority"], "Graph Config", {"project": project}
     )
 
 
-@memoize
+@functools.lru_cache(maxsize=None)
 def get_default_deadline(graph_config, project):
     return evaluate_keyed_by(
         graph_config["task-deadline-after"], "Graph Config", {"project": project}
@@ -380,10 +380,10 @@ def build_docker_worker_payload(config, task, task_def):
             for v in sorted(volumes):
                 if v in worker["volumes"]:
                     raise Exception(
-                        "volume %s already defined; "
+                        f"volume {v} already defined; "
                         "if it is defined in a Dockerfile, "
                         "it does not need to be specified in the "
-                        "worker definition" % v
+                        "worker definition"
                     )
 
                 worker["volumes"].append(v)
@@ -544,9 +544,7 @@ def build_docker_worker_payload(config, task, task_def):
                 suffix=suffix,
             )
             caches[name] = cache["mount-point"]
-            task_def["scopes"].append(
-                {"task-reference": "docker-worker:cache:%s" % name}
-            )
+            task_def["scopes"].append({"task-reference": f"docker-worker:cache:{name}"})
 
         # Assertion: only run-task is interested in this.
         if run_task:

{taskcluster-taskgraph-8.2.0 → taskcluster-taskgraph-9.0.0}/src/taskgraph/util/archive.py

@@ -80,14 +80,14 @@ def create_tar_from_files(fp, files):
             ti.type = tarfile.REGTYPE
 
             if not ti.isreg():
-                raise ValueError("not a regular file: %s" % f)
+                raise ValueError(f"not a regular file: {f}")
 
             # Disallow setuid and setgid bits. This is an arbitrary restriction.
             # However, since we set uid/gid to root:root, setuid and setgid
             # would be a glaring security hole if the archive were
             # uncompressed as root.
             if ti.mode & (stat.S_ISUID | stat.S_ISGID):
-                raise ValueError("cannot add file with setuid or setgid set: " "%s" % f)
+                raise ValueError(f"cannot add file with setuid or setgid set: {f}")
 
             # Set uid, gid, username, and group as deterministic values.
             ti.uid = 0

taskcluster-taskgraph-9.0.0/src/taskgraph/util/copy.py

@@ -0,0 +1,47 @@
+from typing import Any
+
+from taskgraph.task import Task
+from taskgraph.util.readonlydict import ReadOnlyDict
+
+immutable_types = {int, float, bool, str, type(None), ReadOnlyDict}
+
+
+def deepcopy(obj: Any) -> Any:
+    """Perform a deep copy of an object with a tree like structure.
+
+    This is a re-implementation of Python's `copy.deepcopy` function with a few key differences:
+
+    1. Unlike the stdlib, this does *not* support copying graph-like structure,
+    which allows it to be more efficient than deepcopy on tree-like structures
+    (such as Tasks).
+    2. This special cases support for `taskgraph.task.Task` objects.
+
+    Args:
+        obj: The object to deep copy.
+
+    Returns:
+        A deep copy of the object.
+    """
+    ty = type(obj)
+    if ty in immutable_types:
+        return obj
+    if ty is dict:
+        return {k: deepcopy(v) for k, v in obj.items()}
+    if ty is list:
+        return [deepcopy(elt) for elt in obj]
+    if ty is Task:
+        task = Task(
+            kind=deepcopy(obj.kind),
+            label=deepcopy(obj.label),
+            attributes=deepcopy(obj.attributes),
+            task=deepcopy(obj.task),
+            description=deepcopy(obj.description),
+            optimization=deepcopy(obj.optimization),
+            dependencies=deepcopy(obj.dependencies),
+            soft_dependencies=deepcopy(obj.soft_dependencies),
+            if_dependencies=deepcopy(obj.if_dependencies),
+        )
+        if obj.task_id:
+            task.task_id = obj.task_id
+        return task
+    raise NotImplementedError(f"copying '{ty}' from '{obj}'")

{taskcluster-taskgraph-8.2.0 → taskcluster-taskgraph-9.0.0}/src/taskgraph/util/docker.py

@@ -3,6 +3,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 
+import functools
 import hashlib
 import io
 import os
@@ -10,7 +11,6 @@ import re
 from typing import Optional
 
 from taskgraph.util.archive import create_tar_gz_from_files
-from taskgraph.util.memoize import memoize
 
 IMAGE_DIR = os.path.join(".", "taskcluster", "docker")
 
@@ -177,15 +177,15 @@ def stream_context_tar(topsrcdir, context_dir, out_file, image_name=None, args=N
 
             p = line[len("# %include ") :].strip()
             if os.path.isabs(p):
-                raise Exception("extra include path cannot be absolute: %s" % p)
+                raise Exception(f"extra include path cannot be absolute: {p}")
 
             fs_path = os.path.normpath(os.path.join(topsrcdir, p))
             # Check for filesystem traversal exploits.
             if not fs_path.startswith(topsrcdir):
-                raise Exception("extra include path outside topsrcdir: %s" % p)
+                raise Exception(f"extra include path outside topsrcdir: {p}")
 
             if not os.path.exists(fs_path):
-                raise Exception("extra include path does not exist: %s" % p)
+                raise Exception(f"extra include path does not exist: {p}")
 
             if os.path.isdir(fs_path):
                 for root, dirs, files in os.walk(fs_path):
@@ -205,7 +205,7 @@ def stream_context_tar(topsrcdir, context_dir, out_file, image_name=None, args=N
     return writer.hexdigest()
 
 
-@memoize
+@functools.lru_cache(maxsize=None)
 def image_paths():
     """Return a map of image name to paths containing their Dockerfile."""
     config = load_yaml("taskcluster", "kinds", "docker-image", "kind.yml")
@@ -222,7 +222,7 @@ def image_path(name):
     return os.path.join(IMAGE_DIR, name)
 
 
-@memoize
+@functools.lru_cache(maxsize=None)
 def parse_volumes(image):
     """Parse VOLUME entries from a Dockerfile for an image."""
     volumes = set()

{taskcluster-taskgraph-8.2.0 → taskcluster-taskgraph-9.0.0}/src/taskgraph/util/hash.py

@@ -2,14 +2,14 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import functools
 import hashlib
 from pathlib import Path
 
 from taskgraph.util import path as mozpath
-from taskgraph.util.memoize import memoize
 
 
-@memoize
+@functools.lru_cache(maxsize=None)
 def hash_path(path):
     """Hash a single file.
 
@@ -36,7 +36,7 @@ def hash_paths(base_path, patterns):
         if matches:
             found.update(matches)
         else:
-            raise Exception("%s did not match anything" % pattern)
+            raise Exception(f"{pattern} did not match anything")
     for path in sorted(found):
         h.update(
             f"{hash_path(mozpath.abspath(mozpath.join(base_path, path)))} {mozpath.normsep(path)}\n".encode()
@@ -44,13 +44,13 @@ def hash_paths(base_path, patterns):
     return h.hexdigest()
 
 
-@memoize
+@functools.lru_cache(maxsize=None)
 def _find_matching_files(base_path, pattern):
     files = _get_all_files(base_path)
     return [path for path in files if mozpath.match(path, pattern)]
 
 
-@memoize
+@functools.lru_cache(maxsize=None)
 def _get_all_files(base_path):
     return [
         mozpath.normsep(str(path))

{taskcluster-taskgraph-8.2.0 → taskcluster-taskgraph-9.0.0}/src/taskgraph/util/taskcluster.py

@@ -16,7 +16,6 @@ from requests.packages.urllib3.util.retry import Retry
 
 from taskgraph.task import Task
 from taskgraph.util import yaml
-from taskgraph.util.memoize import memoize
 
 logger = logging.getLogger(__name__)
 
@@ -31,7 +30,7 @@ PRODUCTION_TASKCLUSTER_ROOT_URL = None
 CONCURRENCY = 50
 
 
-@memoize
+@functools.lru_cache(maxsize=None)
 def get_root_url(use_proxy):
     """Get the current TASKCLUSTER_ROOT_URL.
 
@@ -106,7 +105,7 @@ def requests_retry_session(
     return session
 
 
-@memoize
+@functools.lru_cache(maxsize=None)
 def get_session():
     return requests_retry_session(retries=5)
 
@@ -277,7 +276,7 @@ def get_task_url(task_id, use_proxy=False):
     return task_tmpl.format(task_id)
 
 
-@memoize
+@functools.lru_cache(maxsize=None)
 def get_task_definition(task_id, use_proxy=False):
     response = _do_request(get_task_url(task_id, use_proxy))
     return response.json()
@@ -446,7 +445,7 @@ def list_task_group_incomplete_tasks(task_group_id):
             break
 
 
-@memoize
+@functools.lru_cache(maxsize=None)
 def _get_deps(task_ids, use_proxy):
     upstream_tasks = {}
     for task_id in task_ids:

{taskcluster-taskgraph-8.2.0 → taskcluster-taskgraph-9.0.0}/src/taskgraph/util/templates.py

@@ -2,8 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-
-import copy
+from taskgraph.util.copy import deepcopy
 
 
 def merge_to(source, dest):
@@ -18,9 +17,19 @@ def merge_to(source, dest):
     """
 
     for key, value in source.items():
+        if (
+            isinstance(value, dict)
+            and len(value) == 1
+            and list(value)[0].startswith("by-")
+        ):
+            # Do not merge by-* values as it will almost certainly not do what
+            # the user expects.
+            dest[key] = value
+            continue
+
         # Override mismatching or empty types
         if type(value) != type(dest.get(key)):  # noqa
-            dest[key] = source[key]
+            dest[key] = value
             continue
 
         # Merge dict
@@ -29,10 +38,10 @@ def merge_to(source, dest):
             continue
 
         if isinstance(value, list):
-            dest[key] = dest[key] + source[key]
+            dest[key] = dest[key] + value
             continue
 
-        dest[key] = source[key]
+        dest[key] = value
 
     return dest
 
@@ -46,7 +55,7 @@ def merge(*objects):
     Returns the result without modifying any arguments.
     """
     if len(objects) == 1:
-        return copy.deepcopy(objects[0])
+        return deepcopy(objects[0])
     return merge_to(objects[-1], merge(*objects[:-1]))
 
 

{taskcluster-taskgraph-8.2.0 → taskcluster-taskgraph-9.0.0}/src/taskgraph/util/verify.py

@@ -5,6 +5,7 @@
 
 import logging
 import sys
+import warnings
 from abc import ABC, abstractmethod
 from dataclasses import dataclass, field
 from typing import Callable, Dict, List, Union
@@ -192,7 +193,17 @@ def verify_routes_notification_filters(
     if task is None:
         return
     route_prefix = "notify."
-    valid_filters = ("on-any", "on-completed", "on-failed", "on-exception")
+    valid_filters = (
+        "on-any",
+        "on-completed",
+        "on-defined",
+        "on-failed",
+        "on-exception",
+        "on-pending",
+        "on-resolved",
+        "on-running",
+        "on-transition",
+    )
     task_dict = task.task
     routes = task_dict.get("routes", [])
 
@@ -204,6 +215,13 @@ def verify_routes_notification_filters(
                 raise Exception(
                     f"{task.label} has invalid notification filter ({route_filter})"
                 )
+            if route_filter == "on-any":
+                warnings.warn(
+                    DeprecationWarning(
+                        f"notification filter '{route_filter}' is deprecated. Use "
+                        "'on-transition' or 'on-resolved'."
+                    )
+                )
 
 
 @verifications.add("full_task_graph")

{taskcluster-taskgraph-8.2.0 → taskcluster-taskgraph-9.0.0}/src/taskgraph/util/workertypes.py

@@ -2,10 +2,10 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import functools
 from dataclasses import dataclass
 
 from .keyed_by import evaluate_keyed_by
-from .memoize import memoize
 
 
 @dataclass
@@ -29,7 +29,7 @@ _BUILTIN_TYPES = {
 }
 
 
-@memoize
+@functools.lru_cache(maxsize=None)
 def worker_type_implementation(graph_config, worker_type):
     """Get the worker implementation and OS for the given workerType, where the
     OS represents the host system, not the target OS, in the case of
@@ -46,7 +46,7 @@ def worker_type_implementation(graph_config, worker_type):
     return worker_config["implementation"], worker_config.get("os")
 
 
-@memoize
+@functools.lru_cache(maxsize=None)
 def get_worker_type(graph_config, alias, level):
     """
     Get the worker type based, evaluating aliases from the graph config.

{taskcluster-taskgraph-8.2.0 → taskcluster-taskgraph-9.0.0}/test/test_actions_registry.py

@@ -1,9 +1,8 @@
-from test import does_not_raise
-
 import pytest
 from mozilla_repo_urls import InvalidRepoUrlError
 
 from taskgraph.actions import registry
+from test import does_not_raise
 
 
 @pytest.mark.parametrize(