taskcluster-taskgraph 18.1.0__tar.gz → 19.1.0__tar.gz

{taskcluster_taskgraph-18.1.0 → taskcluster_taskgraph-19.1.0}/.pre-commit-config.yaml

@@ -13,17 +13,17 @@ repos:
         exclude: template
       - id: check-added-large-files
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.14.10
+    rev: v0.14.14
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
       - id: ruff-format
   - repo: https://github.com/astral-sh/uv-pre-commit
-    rev: 0.9.21
+    rev: 0.9.28
     hooks:
       - id: uv-lock
   - repo: https://github.com/adrienverge/yamllint.git
-    rev: v1.37.1
+    rev: v1.38.0
     hooks:
       - id: yamllint
         exclude: template

{taskcluster_taskgraph-18.1.0 → taskcluster_taskgraph-19.1.0}/CHANGELOG.md

@@ -1,5 +1,34 @@
 # Change Log
 
+## [19.1.0] - 2026-02-17
+
+### Added
+
+- Converted all remaining voluptuous schemas to msgspec
+
+### Fixed
+
+- Parent image name lookup in `build-image` now uses task ids
+- Decision task environment now passes full ref names for chain-of-trust verification
+- Exclude `__pycache__` directories when generating docker contexts
+
+## [19.0.0] - 2026-02-06
+
+### Added
+
+- BREAKING CHANGE: Support for msgspec schemas; voluptuous schemas renamed to `LegacySchema`
+
+### Changed
+
+- Updated taskcluster dependency to v92; artifact functions now handle redirects manually
+- Updated workerType in template `.taskcluster.yml`
+- Disabled unnecessary `taskcluster-proxy` for Windows tests
+
+### Fixed
+
+- Race condition in `docker_image` transforms when creating docker-contexts directory
+- Wrong location for `extra` in `.taskcluster.yml` template
+
 ## [18.1.0] - 2026-01-13
 
 ### Added

{taskcluster_taskgraph-18.1.0 → taskcluster_taskgraph-19.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: taskcluster-taskgraph
-Version: 18.1.0
+Version: 19.1.0
 Summary: Build taskcluster taskgraphs
 Project-URL: Repository, https://github.com/taskcluster/taskgraph
 Project-URL: Issues, https://github.com/taskcluster/taskgraph/issues
@@ -20,18 +20,19 @@ Requires-Python: >=3.9
 Requires-Dist: appdirs>=1.4
 Requires-Dist: cookiecutter~=2.1
 Requires-Dist: json-e>=2.7
-Requires-Dist: mozilla-repo-urls
+Requires-Dist: mozilla-repo-urls>=0.1.1
+Requires-Dist: msgspec>=0.20.0
 Requires-Dist: pyyaml>=5.3.1
 Requires-Dist: redo>=2.0
 Requires-Dist: requests>=2.25
 Requires-Dist: slugid>=2.0
 Requires-Dist: taskcluster-urls>=11.0
-Requires-Dist: taskcluster<92.0,>=91.0
+Requires-Dist: taskcluster>=92.0
 Requires-Dist: voluptuous>=0.12.1
 Provides-Extra: load-image
-Requires-Dist: zstandard; extra == 'load-image'
+Requires-Dist: zstandard>=0.23.0; extra == 'load-image'
 Provides-Extra: orjson
-Requires-Dist: orjson; extra == 'orjson'
+Requires-Dist: orjson>=3; extra == 'orjson'
 Description-Content-Type: text/x-rst
 
 
@@ -39,10 +40,6 @@ Description-Content-Type: text/x-rst
    :target: https://firefox-ci-tc.services.mozilla.com/api/github/v1/repository/taskcluster/taskgraph/main/latest
    :alt: Task Status
 
-.. image:: https://results.pre-commit.ci/badge/github/taskcluster/taskgraph/main.svg
-   :target: https://results.pre-commit.ci/latest/github/taskcluster/taskgraph/main
-   :alt: pre-commit.ci status
-
 .. image:: https://codecov.io/gh/taskcluster/taskgraph/branch/main/graph/badge.svg?token=GJIV52ZQNP
    :target: https://codecov.io/gh/taskcluster/taskgraph
    :alt: Code Coverage

{taskcluster_taskgraph-18.1.0 → taskcluster_taskgraph-19.1.0}/README.rst

@@ -3,10 +3,6 @@
    :target: https://firefox-ci-tc.services.mozilla.com/api/github/v1/repository/taskcluster/taskgraph/main/latest
    :alt: Task Status
 
-.. image:: https://results.pre-commit.ci/badge/github/taskcluster/taskgraph/main.svg
-   :target: https://results.pre-commit.ci/latest/github/taskcluster/taskgraph/main
-   :alt: pre-commit.ci status
-
 .. image:: https://codecov.io/gh/taskcluster/taskgraph/branch/main/graph/badge.svg?token=GJIV52ZQNP
    :target: https://codecov.io/gh/taskcluster/taskgraph
    :alt: Code Coverage

{taskcluster_taskgraph-18.1.0 → taskcluster_taskgraph-19.1.0}/docs/concepts/transforms.rst

@@ -108,9 +108,9 @@ about the state of the tasks at given points. Here is an example:
    from voluptuous import Optional, Required
 
    from taskgraph.transforms.base import TransformSequence
-   from taskgraph.util.schema import Schema
+   from taskgraph.util.schema import LegacySchema
 
-   my_schema = Schema({
+   my_schema = LegacySchema({
        Required("foo"): str,
        Optional("bar"): bool,
    })

{taskcluster_taskgraph-18.1.0 → taskcluster_taskgraph-19.1.0}/docs/conf.py

@@ -43,6 +43,11 @@ templates_path = ["_templates"]
 # This pattern also affects html_static_path and html_extra_path.
 exclude_patterns = ["docs/_build", "Thumbs.db", ".DS_Store"]
 
+# Suppress ambiguous cross-reference warnings from autodoc.
+# Multiple Schema classes define fields with the same name (e.g. "path"),
+# which Sphinx can't disambiguate.
+suppress_warnings = ["ref.python"]
+
 
 # -- Options for HTML output -------------------------------------------------
 

{taskcluster_taskgraph-18.1.0 → taskcluster_taskgraph-19.1.0}/docs/howto/resolve-keyed-by.rst

@@ -210,10 +210,10 @@ It can be used to generate a valid schema that allows a field to either use
 
 .. code-block:: python
 
-   from taskgraph.util.schema import Schema, optionally_keyed_by
+   from taskgraph.util.schema import LegacySchema, optionally_keyed_by
 
 
-   schema = Schema({
+   schema = LegacySchema({
        # ...
        Optional("worker"): {
            Optional("max-run-time"): optionally_keyed_by("platform", int),

{taskcluster_taskgraph-18.1.0 → taskcluster_taskgraph-19.1.0}/docs/reference/migrations.rst

@@ -3,6 +3,37 @@ Migration Guide
 
 This page can help when migrating Taskgraph across major versions.
 
+18.x -> 19.x
+------------
+
+* The ``Schema`` class in ``taskgraph.util.schema`` is now based on msgspec
+  instead of voluptuous. The old voluptuous-based class has been renamed to
+  ``LegacySchema``. Both will continue to be supported.
+
+  If you have custom schemas using ``Schema``, you have two options:
+
+  **Option 1**: Switch to ``LegacySchema`` to keep using voluptuous:
+
+  .. code-block:: python
+
+     from taskgraph.util.schema import LegacySchema
+
+  **Option 2**: Migrate to the new msgspec-based ``Schema``:
+
+  .. code-block:: python
+
+     from taskgraph.util.schema import Schema
+
+     class MySchema(Schema):
+         foo: str
+         bar: int = 10
+
+  The new ``Schema`` class uses kebab-case renaming and forbids
+  unknown fields by default. See the msgspec documentation for more details.
+
+* ``validate_schema`` now supports both voluptuous and msgspec schemas, and
+  ``optionally_keyed_by`` accepts a ``use_msgspec=True`` flag for msgspec output.
+
 17.x -> 18.x
 ------------
 

{taskcluster_taskgraph-18.1.0 → taskcluster_taskgraph-19.1.0}/docs/reference/transforms/from_deps.rst

@@ -176,9 +176,9 @@ allows tasks to pass down additional context (such as with the built-in
    from taskgraph.task import Task
    from taskgraph.transforms.base import TransformConfig
    from taskgraph.util.dependencies import group_by
-   from taskgraph.util.schema import Schema
+   from taskgraph.util.schema import LegacySchema
 
-   @group_by("custom-name", schema=Schema(str))
+   @group_by("custom-name", schema=LegacySchema(str))
    def group_by(config: TransformConfig, tasks: List[Task], ctx: str) -> List[List[Task]]:
       pass
 

{taskcluster_taskgraph-18.1.0 → taskcluster_taskgraph-19.1.0}/docs/tutorials/creating-a-task-graph.rst

@@ -139,10 +139,10 @@ comments for explanations):
    from voluptuous import Optional, Required
 
    from taskgraph.transforms.base import TransformSequence
-   from taskgraph.util.schema import Schema
+   from taskgraph.util.schema import LegacySchema
 
    # Define the schema. We use the `voluptuous` package to handle validation.
-   hello_description_schema = Schema({
+   hello_description_schema = LegacySchema({
        Required("text"): str,
        Optional("description"): str,
    })

{taskcluster_taskgraph-18.1.0 → taskcluster_taskgraph-19.1.0}/packages/pytest-taskgraph/pyproject.toml

@@ -6,8 +6,9 @@ readme = "README.md"
 authors = [
     { name = "Andrew Halberstadt", email = "ahal@mozilla.com" }
 ]
-requires-python = ">=3.8"
-dependencies = ["pytest", "taskcluster-taskgraph>=12.1.0"]
+dependencies = [
+    "pytest>=8.4.2",
+    "taskcluster-taskgraph>=12.1.0"]
 
 [project.entry-points.pytest11]
 pytest-taskgraph = "pytest_taskgraph"

{taskcluster_taskgraph-18.1.0 → taskcluster_taskgraph-19.1.0}/packages/sphinx-taskgraph/pyproject.toml

@@ -6,10 +6,9 @@ readme = "README.md"
 authors = [
     { name = "Andrew Halberstadt", email = "ahal@mozilla.com" }
 ]
-requires-python = ">=3.8"
 dependencies = [
-    "sphinx",
-    "voluptuous",
+    "sphinx>=7.0.0",
+    "voluptuous>=0.12.1",
 ]
 
 [build-system]

{taskcluster_taskgraph-18.1.0 → taskcluster_taskgraph-19.1.0}/pyproject.toml

@@ -1,7 +1,7 @@
 ### Project
 [project]
 name = "taskcluster-taskgraph"
-version = "18.1.0"
+version = "19.1.0"
 description = "Build taskcluster taskgraphs"
 readme = "README.rst"
 authors = [
@@ -24,38 +24,39 @@ dependencies = [
   "appdirs>=1.4",
   "cookiecutter~=2.1",
   "json-e>=2.7",
-  "mozilla-repo-urls",
+  "mozilla-repo-urls>=0.1.1",
+  "msgspec>=0.20.0",
   "PyYAML>=5.3.1",
   "redo>=2.0",
   "requests>=2.25",
   "slugid>=2.0",
-  "taskcluster>=91.0,<92.0",
+  "taskcluster>=92.0",
   "taskcluster-urls>=11.0",
   "voluptuous>=0.12.1",
 ]
 
 [dependency-groups]
 dev = [
-  "coverage",
-  "mock",
-  "pre-commit",
-  "pyright",
-  "pytest",
-  "pytest-mock",
-  "pytest-taskgraph",
-  "responses",
-  "sphinx",
-  "sphinx-autobuild",
-  "sphinx-argparse",
-  "sphinx-book-theme >=1",
-  "sphinx-taskgraph",
-  "sphinxcontrib-mermaid",
-  "zstandard",
+  "coverage>=7.10.7",
+  "mock>=5.2.0",
+  "pre-commit>=4.3.0",
+  "pyright>=1.1.406",
+  "pytest>=8.4.2",
+  "pytest-mock>=3.15.1",
+  "pytest-taskgraph>=0.2.0",
+  "responses>=0.25.8",
+  "sphinx>=7.4.7",
+  "sphinx-autobuild>=2024.04.16",
+  "sphinx-argparse>=0.4.0",
+  "sphinx-book-theme >=1.1.4",
+  "sphinx-taskgraph>=0.1.0",
+  "sphinxcontrib-mermaid>=1.0.0",
+  "zstandard>=0.25.0",
 ]
 
 [project.optional-dependencies]
-load-image = ["zstandard"]
-orjson = ["orjson"]
+load-image = ["zstandard>=0.23.0"]
+orjson = ["orjson>=3"]
 
 [project.scripts]
 taskgraph = "taskgraph.main:main"

taskcluster_taskgraph-19.1.0/src/taskgraph/config.py

@@ -0,0 +1,170 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import logging
+import os
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Literal, Optional, Union
+
+from .util.python_path import find_object
+from .util.schema import Schema, TaskPriority, optionally_keyed_by, validate_schema
+from .util.vcs import get_repository
+from .util.yaml import load_yaml
+
+logger = logging.getLogger(__name__)
+
+CacheType = Literal["cargo", "checkout", "npm", "pip", "uv"]
+
+
+class WorkerAlias(Schema):
+    provisioner: optionally_keyed_by("level", str, use_msgspec=True)  # type: ignore
+    implementation: str
+    os: str
+    worker_type: optionally_keyed_by("level", str, use_msgspec=True)  # type: ignore
+
+
+class WorkersConfig(Schema):
+    aliases: dict[str, WorkerAlias]
+
+
+class RunConfig(Schema):
+    # List of caches to enable, or a boolean to enable/disable all of them.
+    use_caches: Optional[Union[bool, list[CacheType]]] = None
+
+
+class RepositoryConfig(Schema, forbid_unknown_fields=False, kw_only=True):
+    name: str
+    project_regex: Optional[str] = None
+    ssh_secret_name: Optional[str] = None
+    # FIXME: Extra keys allowed via forbid_unknown_fields=False
+
+
+class TaskgraphConfig(Schema):
+    repositories: dict[str, RepositoryConfig]
+    # Python function to call to register extensions.
+    register: Optional[str] = None
+    decision_parameters: Optional[str] = None
+    # The taskcluster index prefix to use for caching tasks.
+    # Defaults to `trust-domain`.
+    cached_task_prefix: Optional[str] = None
+    # Should tasks from pull requests populate the cache
+    cache_pull_requests: Optional[bool] = None
+    # Regular expressions matching index paths to be summarized.
+    index_path_regexes: Optional[list[str]] = None
+    # Configuration related to the 'run' transforms.
+    run: Optional[RunConfig] = None
+
+    def __post_init__(self):
+        # Validate repositories has at least 1 entry (was All(..., Length(min=1)))
+        if not self.repositories:
+            raise ValueError("'repositories' must have at least one entry")
+
+
+#: Schema for the graph config
+class GraphConfigSchema(Schema, forbid_unknown_fields=False, kw_only=True):
+    # The trust-domain for this graph.
+    trust_domain: str
+    task_priority: optionally_keyed_by(  # type: ignore
+        "project", "level", TaskPriority, use_msgspec=True
+    )
+    workers: WorkersConfig
+    taskgraph: TaskgraphConfig
+    # Name of the docker image kind (default: docker-image)
+    docker_image_kind: Optional[str] = None
+    # Default 'deadline' for tasks, in relative date format. Eg: '1 week'
+    task_deadline_after: Optional[
+        optionally_keyed_by("project", str, use_msgspec=True)  # type: ignore
+    ] = None
+    # Default 'expires-after' for level 1 tasks, in relative date format.
+    # Eg: '90 days'
+    task_expires_after: Optional[str] = None
+
+
+graph_config_schema = GraphConfigSchema
+
+
+@dataclass(frozen=True, eq=False)
+class GraphConfig:
+    _config: dict
+    root_dir: str
+
+    _PATH_MODIFIED = False
+
+    def __post_init__(self):
+        # ensure we have an absolute path; this is required for assumptions
+        # made later, such as the `vcs_root` being a directory above `root_dir`
+        object.__setattr__(self, "root_dir", os.path.abspath(self.root_dir))
+
+    def __getitem__(self, name):
+        return self._config[name]
+
+    def __contains__(self, name):
+        return name in self._config
+
+    def get(self, name, default=None):
+        return self._config.get(name, default)
+
+    def register(self):
+        """
+        Add the project's taskgraph directory to the python path, and register
+        any extensions present.
+        """
+        if GraphConfig._PATH_MODIFIED:
+            if GraphConfig._PATH_MODIFIED == self.root_dir:
+                # Already modified path with the same root_dir.
+                # We currently need to do this to enable actions to call
+                # taskgraph_decision, e.g. relpro.
+                return
+            raise Exception("Can't register multiple directories on python path.")
+        GraphConfig._PATH_MODIFIED = self.root_dir
+        sys.path.insert(0, self.root_dir)
+        register_path = self["taskgraph"].get("register")
+        if register_path:
+            register = find_object(register_path)
+            assert callable(register)
+            register(self)
+
+    @property
+    def vcs_root(self):
+        try:
+            repo = get_repository(self.root_dir)
+            return Path(repo.path)
+        except RuntimeError:
+            root = Path(self.root_dir)
+            if root.parts[-1:] != ("taskcluster",):
+                raise Exception(
+                    "Not guessing path to vcs root. Graph config in non-standard location."
+                )
+            return root.parent
+
+    @property
+    def taskcluster_yml(self):
+        return os.path.join(self.vcs_root, ".taskcluster.yml")
+
+    @property
+    def docker_dir(self):
+        return os.path.join(self.root_dir, "docker")
+
+    @property
+    def kinds_dir(self):
+        return os.path.join(self.root_dir, "kinds")
+
+
+def validate_graph_config(config):
+    validate_schema(graph_config_schema, config, "Invalid graph configuration:")
+
+
+def load_graph_config(root_dir):
+    config_yml = os.path.join(root_dir, "config.yml")
+    if not os.path.exists(config_yml):
+        raise Exception(f"Couldn't find taskgraph configuration: {config_yml}")
+
+    logger.debug(f"loading config from `{config_yml}`")
+    config = load_yaml(config_yml)
+
+    validate_graph_config(config)
+    return GraphConfig(config, root_dir=root_dir)

{taskcluster_taskgraph-18.1.0 → taskcluster_taskgraph-19.1.0}/src/taskgraph/decision.py

@@ -9,9 +9,9 @@ import pathlib
 import shutil
 import time
 from pathlib import Path
+from typing import Optional
 
 import yaml
-from voluptuous import Optional
 
 from taskgraph.actions import render_actions_json
 from taskgraph.create import create_tasks
@@ -40,11 +40,11 @@ PER_PROJECT_PARAMETERS = {
 
 
 #: Schema for try_task_config.json version 2
-try_task_config_schema_v2 = Schema(
-    {
-        Optional("parameters"): {str: object},
-    }
-)
+class TryTaskConfigSchemaV2(Schema, forbid_unknown_fields=True):
+    parameters: Optional[dict[str, object]] = None
+
+
+try_task_config_schema_v2 = TryTaskConfigSchemaV2
 
 
 def full_task_graph_to_runnable_tasks(full_task_json):

{taskcluster_taskgraph-18.1.0 → taskcluster_taskgraph-19.1.0}/src/taskgraph/docker.py

@@ -211,7 +211,9 @@ def build_image(
                     raise
 
                 # Parent id doesn't exist, needs to be re-built as well.
-                parent = task.dependencies["parent"][len("docker-image-") :]
+                parent = next(
+                    t for t in image_tasks if image_tasks[t].task_id == parent_id
+                ).removeprefix("docker-image-")
                 parent_tar = temp_dir / "parent.tar"
                 build_image(graph_config, parent, save_image=str(parent_tar))
                 volumes.append((str(parent_tar), "/workspace/parent.tar"))

{taskcluster_taskgraph-18.1.0 → taskcluster_taskgraph-19.1.0}/src/taskgraph/transforms/chunking.py

@@ -2,49 +2,30 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 import copy
-from textwrap import dedent
-
-from voluptuous import ALLOW_EXTRA, Optional, Required
+from typing import Optional
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.schema import Schema
 from taskgraph.util.templates import substitute
 
+
+class ChunkConfig(Schema):
+    # The total number of chunks to split the task into.
+    total_chunks: int
+    # A list of fields that need to have `{this_chunk}` and/or
+    # `{total_chunks}` replaced in them.
+    substitution_fields: list[str] = []
+
+
 #: Schema for chunking transforms
-CHUNK_SCHEMA = Schema(
-    {
-        # Optional, so it can be used for a subset of tasks in a kind
-        Optional(
-            "chunk",
-            description=dedent(
-                """
-            `chunk` can be used to split one task into `total-chunks`
-            tasks, substituting `this_chunk` and `total_chunks` into any
-            fields in `substitution-fields`.
-            """.lstrip()
-            ),
-        ): {
-            Required(
-                "total-chunks",
-                description=dedent(
-                    """
-                The total number of chunks to split the task into.
-                """.lstrip()
-                ),
-            ): int,
-            Optional(
-                "substitution-fields",
-                description=dedent(
-                    """
-                A list of fields that need to have `{this_chunk}` and/or
-                `{total_chunks}` replaced in them.
-                """.lstrip()
-                ),
-            ): [str],
-        }
-    },
-    extra=ALLOW_EXTRA,
-)
+class ChunkSchema(Schema, forbid_unknown_fields=False, kw_only=True):
+    # `chunk` can be used to split one task into `total-chunks`
+    # tasks, substituting `this_chunk` and `total_chunks` into any
+    # fields in `substitution-fields`.
+    chunk: Optional[ChunkConfig] = None
+
+
+CHUNK_SCHEMA = ChunkSchema
 
 transforms = TransformSequence()
 transforms.add_validate(CHUNK_SCHEMA)