taskcluster-taskgraph 19.0.0__tar.gz → 19.2.0__tar.gz

{taskcluster_taskgraph-19.0.0 → taskcluster_taskgraph-19.2.0}/CHANGELOG.md

@@ -1,5 +1,24 @@
 # Change Log
 
+## [19.2.0] - 2026-02-18
+
+### Fixed
+
+- Restore `payload_builder` wrapping for dict schemas
+- Make nested struct fields optional to match voluptuous defaults
+
+## [19.1.0] - 2026-02-17
+
+### Added
+
+- Converted all remaining voluptuous schemas to msgspec
+
+### Fixed
+
+- Parent image name lookup in `build-image` now uses task ids
+- Decision task environment now passes full ref names for chain-of-trust verification
+- Exclude `__pycache__` directories when generating docker contexts
+
 ## [19.0.0] - 2026-02-06
 
 ### Added

{taskcluster_taskgraph-19.0.0 → taskcluster_taskgraph-19.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: taskcluster-taskgraph
-Version: 19.0.0
+Version: 19.2.0
 Summary: Build taskcluster taskgraphs
 Project-URL: Repository, https://github.com/taskcluster/taskgraph
 Project-URL: Issues, https://github.com/taskcluster/taskgraph/issues
@@ -20,7 +20,7 @@ Requires-Python: >=3.9
 Requires-Dist: appdirs>=1.4
 Requires-Dist: cookiecutter~=2.1
 Requires-Dist: json-e>=2.7
-Requires-Dist: mozilla-repo-urls
+Requires-Dist: mozilla-repo-urls>=0.1.1
 Requires-Dist: msgspec>=0.20.0
 Requires-Dist: pyyaml>=5.3.1
 Requires-Dist: redo>=2.0
@@ -30,9 +30,9 @@ Requires-Dist: taskcluster-urls>=11.0
 Requires-Dist: taskcluster>=92.0
 Requires-Dist: voluptuous>=0.12.1
 Provides-Extra: load-image
-Requires-Dist: zstandard; extra == 'load-image'
+Requires-Dist: zstandard>=0.23.0; extra == 'load-image'
 Provides-Extra: orjson
-Requires-Dist: orjson; extra == 'orjson'
+Requires-Dist: orjson>=3; extra == 'orjson'
 Description-Content-Type: text/x-rst
 
 
@@ -40,10 +40,6 @@ Description-Content-Type: text/x-rst
    :target: https://firefox-ci-tc.services.mozilla.com/api/github/v1/repository/taskcluster/taskgraph/main/latest
    :alt: Task Status
 
-.. image:: https://results.pre-commit.ci/badge/github/taskcluster/taskgraph/main.svg
-   :target: https://results.pre-commit.ci/latest/github/taskcluster/taskgraph/main
-   :alt: pre-commit.ci status
-
 .. image:: https://codecov.io/gh/taskcluster/taskgraph/branch/main/graph/badge.svg?token=GJIV52ZQNP
    :target: https://codecov.io/gh/taskcluster/taskgraph
    :alt: Code Coverage

{taskcluster_taskgraph-19.0.0 → taskcluster_taskgraph-19.2.0}/README.rst

@@ -3,10 +3,6 @@
    :target: https://firefox-ci-tc.services.mozilla.com/api/github/v1/repository/taskcluster/taskgraph/main/latest
    :alt: Task Status
 
-.. image:: https://results.pre-commit.ci/badge/github/taskcluster/taskgraph/main.svg
-   :target: https://results.pre-commit.ci/latest/github/taskcluster/taskgraph/main
-   :alt: pre-commit.ci status
-
 .. image:: https://codecov.io/gh/taskcluster/taskgraph/branch/main/graph/badge.svg?token=GJIV52ZQNP
    :target: https://codecov.io/gh/taskcluster/taskgraph
    :alt: Code Coverage

{taskcluster_taskgraph-19.0.0 → taskcluster_taskgraph-19.2.0}/docs/conf.py

@@ -43,6 +43,11 @@ templates_path = ["_templates"]
 # This pattern also affects html_static_path and html_extra_path.
 exclude_patterns = ["docs/_build", "Thumbs.db", ".DS_Store"]
 
+# Suppress ambiguous cross-reference warnings from autodoc.
+# Multiple Schema classes define fields with the same name (e.g. "path"),
+# which Sphinx can't disambiguate.
+suppress_warnings = ["ref.python"]
+
 
 # -- Options for HTML output -------------------------------------------------
 

{taskcluster_taskgraph-19.0.0 → taskcluster_taskgraph-19.2.0}/pyproject.toml

@@ -1,7 +1,7 @@
 ### Project
 [project]
 name = "taskcluster-taskgraph"
-version = "19.0.0"
+version = "19.2.0"
 description = "Build taskcluster taskgraphs"
 readme = "README.rst"
 authors = [
@@ -24,7 +24,7 @@ dependencies = [
   "appdirs>=1.4",
   "cookiecutter~=2.1",
   "json-e>=2.7",
-  "mozilla-repo-urls",
+  "mozilla-repo-urls>=0.1.1",
   "msgspec>=0.20.0",
   "PyYAML>=5.3.1",
   "redo>=2.0",
@@ -37,26 +37,26 @@ dependencies = [
 
 [dependency-groups]
 dev = [
-  "coverage",
-  "mock",
-  "pre-commit",
-  "pyright",
-  "pytest",
-  "pytest-mock",
-  "pytest-taskgraph",
-  "responses",
-  "sphinx",
-  "sphinx-autobuild",
-  "sphinx-argparse",
-  "sphinx-book-theme >=1",
-  "sphinx-taskgraph",
-  "sphinxcontrib-mermaid",
-  "zstandard",
+  "coverage>=7.10.7",
+  "mock>=5.2.0",
+  "pre-commit>=4.3.0",
+  "pyright>=1.1.406",
+  "pytest>=8.4.2",
+  "pytest-mock>=3.15.1",
+  "pytest-taskgraph>=0.2.0",
+  "responses>=0.25.8",
+  "sphinx>=7.4.7",
+  "sphinx-autobuild>=2024.04.16",
+  "sphinx-argparse>=0.4.0",
+  "sphinx-book-theme >=1.1.4",
+  "sphinx-taskgraph>=0.1.0",
+  "sphinxcontrib-mermaid>=1.0.0",
+  "zstandard>=0.25.0",
 ]
 
 [project.optional-dependencies]
-load-image = ["zstandard"]
-orjson = ["orjson"]
+load-image = ["zstandard>=0.23.0"]
+orjson = ["orjson>=3"]
 
 [project.scripts]
 taskgraph = "taskgraph.main:main"

taskcluster_taskgraph-19.2.0/src/taskgraph/config.py

@@ -0,0 +1,170 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import logging
+import os
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Literal, Optional, Union
+
+from .util.python_path import find_object
+from .util.schema import Schema, TaskPriority, optionally_keyed_by, validate_schema
+from .util.vcs import get_repository
+from .util.yaml import load_yaml
+
+logger = logging.getLogger(__name__)
+
+CacheType = Literal["cargo", "checkout", "npm", "pip", "uv"]
+
+
+class WorkerAlias(Schema):
+    provisioner: optionally_keyed_by("level", str, use_msgspec=True)  # type: ignore
+    implementation: str
+    os: str
+    worker_type: optionally_keyed_by("level", str, use_msgspec=True)  # type: ignore
+
+
+class WorkersConfig(Schema):
+    aliases: dict[str, WorkerAlias]
+
+
+class RunConfig(Schema):
+    # List of caches to enable, or a boolean to enable/disable all of them.
+    use_caches: Optional[Union[bool, list[CacheType]]] = None
+
+
+class RepositoryConfig(Schema, forbid_unknown_fields=False, kw_only=True):
+    name: str
+    project_regex: Optional[str] = None
+    ssh_secret_name: Optional[str] = None
+    # FIXME: Extra keys allowed via forbid_unknown_fields=False
+
+
+class TaskgraphConfig(Schema):
+    repositories: dict[str, RepositoryConfig]
+    # Python function to call to register extensions.
+    register: Optional[str] = None
+    decision_parameters: Optional[str] = None
+    # The taskcluster index prefix to use for caching tasks.
+    # Defaults to `trust-domain`.
+    cached_task_prefix: Optional[str] = None
+    # Should tasks from pull requests populate the cache
+    cache_pull_requests: Optional[bool] = None
+    # Regular expressions matching index paths to be summarized.
+    index_path_regexes: Optional[list[str]] = None
+    # Configuration related to the 'run' transforms.
+    run: Optional[RunConfig] = None
+
+    def __post_init__(self):
+        # Validate repositories has at least 1 entry (was All(..., Length(min=1)))
+        if not self.repositories:
+            raise ValueError("'repositories' must have at least one entry")
+
+
+#: Schema for the graph config
+class GraphConfigSchema(Schema, forbid_unknown_fields=False, kw_only=True):
+    # The trust-domain for this graph.
+    trust_domain: str
+    task_priority: optionally_keyed_by(  # type: ignore
+        "project", "level", TaskPriority, use_msgspec=True
+    )
+    workers: WorkersConfig
+    taskgraph: TaskgraphConfig
+    # Name of the docker image kind (default: docker-image)
+    docker_image_kind: Optional[str] = None
+    # Default 'deadline' for tasks, in relative date format. Eg: '1 week'
+    task_deadline_after: Optional[
+        optionally_keyed_by("project", str, use_msgspec=True)  # type: ignore
+    ] = None
+    # Default 'expires-after' for level 1 tasks, in relative date format.
+    # Eg: '90 days'
+    task_expires_after: Optional[str] = None
+
+
+graph_config_schema = GraphConfigSchema
+
+
+@dataclass(frozen=True, eq=False)
+class GraphConfig:
+    _config: dict
+    root_dir: str
+
+    _PATH_MODIFIED = False
+
+    def __post_init__(self):
+        # ensure we have an absolute path; this is required for assumptions
+        # made later, such as the `vcs_root` being a directory above `root_dir`
+        object.__setattr__(self, "root_dir", os.path.abspath(self.root_dir))
+
+    def __getitem__(self, name):
+        return self._config[name]
+
+    def __contains__(self, name):
+        return name in self._config
+
+    def get(self, name, default=None):
+        return self._config.get(name, default)
+
+    def register(self):
+        """
+        Add the project's taskgraph directory to the python path, and register
+        any extensions present.
+        """
+        if GraphConfig._PATH_MODIFIED:
+            if GraphConfig._PATH_MODIFIED == self.root_dir:
+                # Already modified path with the same root_dir.
+                # We currently need to do this to enable actions to call
+                # taskgraph_decision, e.g. relpro.
+                return
+            raise Exception("Can't register multiple directories on python path.")
+        GraphConfig._PATH_MODIFIED = self.root_dir
+        sys.path.insert(0, self.root_dir)
+        register_path = self["taskgraph"].get("register")
+        if register_path:
+            register = find_object(register_path)
+            assert callable(register)
+            register(self)
+
+    @property
+    def vcs_root(self):
+        try:
+            repo = get_repository(self.root_dir)
+            return Path(repo.path)
+        except RuntimeError:
+            root = Path(self.root_dir)
+            if root.parts[-1:] != ("taskcluster",):
+                raise Exception(
+                    "Not guessing path to vcs root. Graph config in non-standard location."
+                )
+            return root.parent
+
+    @property
+    def taskcluster_yml(self):
+        return os.path.join(self.vcs_root, ".taskcluster.yml")
+
+    @property
+    def docker_dir(self):
+        return os.path.join(self.root_dir, "docker")
+
+    @property
+    def kinds_dir(self):
+        return os.path.join(self.root_dir, "kinds")
+
+
+def validate_graph_config(config):
+    validate_schema(graph_config_schema, config, "Invalid graph configuration:")
+
+
+def load_graph_config(root_dir):
+    config_yml = os.path.join(root_dir, "config.yml")
+    if not os.path.exists(config_yml):
+        raise Exception(f"Couldn't find taskgraph configuration: {config_yml}")
+
+    logger.debug(f"loading config from `{config_yml}`")
+    config = load_yaml(config_yml)
+
+    validate_graph_config(config)
+    return GraphConfig(config, root_dir=root_dir)

{taskcluster_taskgraph-19.0.0 → taskcluster_taskgraph-19.2.0}/src/taskgraph/decision.py

@@ -9,9 +9,9 @@ import pathlib
 import shutil
 import time
 from pathlib import Path
+from typing import Optional
 
 import yaml
-from voluptuous import Optional
 
 from taskgraph.actions import render_actions_json
 from taskgraph.create import create_tasks
@@ -20,7 +20,7 @@ from taskgraph.parameters import Parameters, get_version
 from taskgraph.taskgraph import TaskGraph
 from taskgraph.util import json
 from taskgraph.util.python_path import find_object
-from taskgraph.util.schema import LegacySchema, validate_schema
+from taskgraph.util.schema import Schema, validate_schema
 from taskgraph.util.vcs import get_repository
 from taskgraph.util.yaml import load_yaml
 
@@ -40,11 +40,11 @@ PER_PROJECT_PARAMETERS = {
 
 
 #: Schema for try_task_config.json version 2
-try_task_config_schema_v2 = LegacySchema(
-    {
-        Optional("parameters"): {str: object},
-    }
-)
+class TryTaskConfigSchemaV2(Schema, forbid_unknown_fields=True):
+    parameters: Optional[dict[str, object]] = None
+
+
+try_task_config_schema_v2 = TryTaskConfigSchemaV2
 
 
 def full_task_graph_to_runnable_tasks(full_task_json):

{taskcluster_taskgraph-19.0.0 → taskcluster_taskgraph-19.2.0}/src/taskgraph/docker.py

@@ -211,7 +211,9 @@ def build_image(
                     raise
 
                 # Parent id doesn't exist, needs to be re-built as well.
-                parent = task.dependencies["parent"][len("docker-image-") :]
+                parent = next(
+                    t for t in image_tasks if image_tasks[t].task_id == parent_id
+                ).removeprefix("docker-image-")
                 parent_tar = temp_dir / "parent.tar"
                 build_image(graph_config, parent, save_image=str(parent_tar))
                 volumes.append((str(parent_tar), "/workspace/parent.tar"))

{taskcluster_taskgraph-19.0.0 → taskcluster_taskgraph-19.2.0}/src/taskgraph/transforms/base.py

@@ -12,7 +12,7 @@ from taskgraph.task import Task
 
 from ..config import GraphConfig
 from ..parameters import Parameters
-from ..util.schema import LegacySchema, validate_schema
+from ..util.schema import Schema, validate_schema
 
 
 @dataclass(frozen=True)
@@ -138,7 +138,7 @@ class TransformSequence:
 
 @dataclass
 class ValidateSchema:
-    schema: LegacySchema
+    schema: Schema
 
     def __call__(self, config, tasks):
         for task in tasks:

{taskcluster_taskgraph-19.0.0 → taskcluster_taskgraph-19.2.0}/src/taskgraph/transforms/chunking.py

@@ -2,49 +2,30 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 import copy
-from textwrap import dedent
-
-from voluptuous import ALLOW_EXTRA, Optional, Required
+from typing import Optional
 
 from taskgraph.transforms.base import TransformSequence
-from taskgraph.util.schema import LegacySchema
+from taskgraph.util.schema import Schema
 from taskgraph.util.templates import substitute
 
+
+class ChunkConfig(Schema):
+    # The total number of chunks to split the task into.
+    total_chunks: int
+    # A list of fields that need to have `{this_chunk}` and/or
+    # `{total_chunks}` replaced in them.
+    substitution_fields: list[str] = []
+
+
 #: Schema for chunking transforms
-CHUNK_SCHEMA = LegacySchema(
-    {
-        # Optional, so it can be used for a subset of tasks in a kind
-        Optional(
-            "chunk",
-            description=dedent(
-                """
-            `chunk` can be used to split one task into `total-chunks`
-            tasks, substituting `this_chunk` and `total_chunks` into any
-            fields in `substitution-fields`.
-            """.lstrip()
-            ),
-        ): {
-            Required(
-                "total-chunks",
-                description=dedent(
-                    """
-                The total number of chunks to split the task into.
-                """.lstrip()
-                ),
-            ): int,
-            Optional(
-                "substitution-fields",
-                description=dedent(
-                    """
-                A list of fields that need to have `{this_chunk}` and/or
-                `{total_chunks}` replaced in them.
-                """.lstrip()
-                ),
-            ): [str],
-        }
-    },
-    extra=ALLOW_EXTRA,
-)
+class ChunkSchema(Schema, forbid_unknown_fields=False, kw_only=True):
+    # `chunk` can be used to split one task into `total-chunks`
+    # tasks, substituting `this_chunk` and `total_chunks` into any
+    # fields in `substitution-fields`.
+    chunk: Optional[ChunkConfig] = None
+
+
+CHUNK_SCHEMA = ChunkSchema
 
 transforms = TransformSequence()
 transforms.add_validate(CHUNK_SCHEMA)

{taskcluster_taskgraph-19.0.0 → taskcluster_taskgraph-19.2.0}/src/taskgraph/transforms/docker_image.py

@@ -5,17 +5,13 @@
 import logging
 import os
 import re
-from textwrap import dedent
-
-from voluptuous import Optional, Required
+from typing import Optional
 
 import taskgraph
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util import json
 from taskgraph.util.docker import create_context_tar, generate_context_hash
-from taskgraph.util.schema import LegacySchema
-
-from .task import task_description_schema
+from taskgraph.util.schema import IndexSchema, Schema
 
 logger = logging.getLogger(__name__)
 
@@ -31,84 +27,31 @@ IMAGE_BUILDER_IMAGE = (
 
 transforms = TransformSequence()
 
+
 #: Schema for docker_image transforms
-docker_image_schema = LegacySchema(
-    {
-        Required(
-            "name",
-            description=dedent(
-                """
-                Name of the docker image.
-                """
-            ).lstrip(),
-        ): str,
-        Optional(
-            "parent",
-            description=dedent(
-                """
-                Name of the parent docker image.
-                """
-            ).lstrip(),
-        ): str,
-        Optional(
-            "symbol",
-            description=dedent(
-                """
-                Treeherder symbol.
-                """
-            ).lstrip(),
-        ): str,
-        Optional(
-            "task-from",
-            description=dedent(
-                """
-                Relative path (from config.path) to the file the docker image was defined in.
-                """
-            ).lstrip(),
-        ): str,
-        Optional(
-            "args",
-            description=dedent(
-                """
-                Arguments to use for the Dockerfile.
-                """
-            ).lstrip(),
-        ): {str: str},
-        Optional(
-            "definition",
-            description=dedent(
-                """
-                Name of the docker image definition under taskcluster/docker, when
-                different from the docker image name.
-                """
-            ).lstrip(),
-        ): str,
-        Optional(
-            "packages",
-            description=dedent(
-                """
-                List of package tasks this docker image depends on.
-                """
-            ).lstrip(),
-        ): [str],
-        Optional(
-            "index",
-            description=dedent(
-                """
-                Information for indexing this build so its artifacts can be discovered.
-                """
-            ).lstrip(),
-        ): task_description_schema["index"],
-        Optional(
-            "cache",
-            description=dedent(
-                """
-                Whether this image should be cached based on inputs.
-                """
-            ).lstrip(),
-        ): bool,
-    }
-)
+class DockerImageSchema(Schema):
+    # Name of the docker image.
+    name: str
+    # Name of the parent docker image.
+    parent: Optional[str] = None
+    # Treeherder symbol.
+    symbol: Optional[str] = None
+    # Relative path (from config.path) to the file the docker image was defined in.
+    task_from: Optional[str] = None
+    # Arguments to use for the Dockerfile.
+    args: Optional[dict[str, str]] = None
+    # Name of the docker image definition under taskcluster/docker, when
+    # different from the docker image name.
+    definition: Optional[str] = None
+    # List of package tasks this docker image depends on.
+    packages: Optional[list[str]] = None
+    # Information for indexing this build so its artifacts can be discovered.
+    index: Optional[IndexSchema] = None
+    # Whether this image should be cached based on inputs.
+    cache: Optional[bool] = None
+
+
+docker_image_schema = DockerImageSchema
 
 
 transforms.add_validate(docker_image_schema)