taskcluster-taskgraph 17.2.0__tar.gz → 17.3.0__tar.gz

{taskcluster_taskgraph-17.2.0 → taskcluster_taskgraph-17.3.0}/.pre-commit-config.yaml

@@ -13,13 +13,13 @@ repos:
         exclude: template
       - id: check-added-large-files
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.13.2
+    rev: v0.14.3
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
       - id: ruff-format
   - repo: https://github.com/astral-sh/uv-pre-commit
-    rev: 0.8.22
+    rev: 0.9.7
     hooks:
       - id: uv-lock
   - repo: https://github.com/adrienverge/yamllint.git
@@ -37,15 +37,17 @@ repos:
                 test/test_util_path.py
             )$
   - repo: https://github.com/compilerla/conventional-pre-commit
-    rev: v4.2.0
+    rev: v4.3.0
     hooks:
       - id: conventional-pre-commit
         stages: [commit-msg]
-# TODO remove 'fetch-content' once Gecko no longer needs to use it with Python 3.8
+# TODO remove 'fetch-content' and 'run-task' once Gecko no longer needs to use
+# them with Python 3.8:
 # https://bugzilla.mozilla.org/show_bug.cgi?id=1990567#c7
 exclude: |
   (?x)^(
     src/taskgraph/run-task/fetch-content |
     src/taskgraph/run-task/robustcheckout.py |
+    src/taskgraph/run-task/run-task |
     taskcluster/scripts/external_tools
   )

{taskcluster_taskgraph-17.2.0 → taskcluster_taskgraph-17.3.0}/CHANGELOG.md

@@ -1,5 +1,25 @@
 # Change Log
 
+## [17.3.0] - 2025-11-07
+
+### Added
+
+- Support for local volumes to `taskgraph load-task -v host_path:container_path`
+
+### Fixed
+
+- `load-task` now takes run-task-{hg,git} into account when checking for run-task
+- Properly detect dependency cycles in the `cached_tasks` transform and raise
+  an exception instead of it being an infinite loop
+
+## [17.2.1] - 2025-10-31
+
+### Fixed
+
+- Exception when using `run-task` script on Windows
+- `run-task` script temporarily supports Python 3.8 again
+- Optimization phase updates `dependencies` property in `Task` objects
+
 ## [17.2.0] - 2025-10-30
 
 ### Added

{taskcluster_taskgraph-17.2.0 → taskcluster_taskgraph-17.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: taskcluster-taskgraph
-Version: 17.2.0
+Version: 17.3.0
 Summary: Build taskcluster taskgraphs
 Project-URL: Repository, https://github.com/taskcluster/taskgraph
 Project-URL: Issues, https://github.com/taskcluster/taskgraph/issues
@@ -26,7 +26,7 @@ Requires-Dist: redo>=2.0
 Requires-Dist: requests>=2.25
 Requires-Dist: slugid>=2.0
 Requires-Dist: taskcluster-urls>=11.0
-Requires-Dist: taskcluster>=91.0
+Requires-Dist: taskcluster<92.0,>=91.0
 Requires-Dist: voluptuous>=0.12.1
 Provides-Extra: load-image
 Requires-Dist: zstandard; extra == 'load-image'

{taskcluster_taskgraph-17.2.0 → taskcluster_taskgraph-17.3.0}/packages/pytest-taskgraph/src/pytest_taskgraph/fixtures/gen.py

@@ -32,8 +32,12 @@ def fake_loader(kind, path, config, parameters, loaded_tasks, write_artifacts):
                 "i": i,
                 "metadata": {"name": f"{kind}-t-{i}"},
                 "deadline": "soon",
+                "workerType": "linux",
+                "provisionerId": "prov",
             },
             "dependencies": dependencies,
+            "if-dependencies": [],
+            "soft-dependencies": [],
         }
         if "task-defaults" in config:
             task = merge(config["task-defaults"], task)
@@ -250,6 +254,7 @@ def make_task(
     task_id=None,
     dependencies=None,
     if_dependencies=None,
+    soft_dependencies=None,
     attributes=None,
 ):
     task_def = task_def or {
@@ -258,7 +263,9 @@ def make_task(
     }
     task = Task(
         attributes=attributes or {},
+        dependencies=dependencies or {},
         if_dependencies=if_dependencies or [],
+        soft_dependencies=soft_dependencies or [],
         kind=kind,
         label=label,
         task=task_def,
@@ -266,7 +273,13 @@ def make_task(
     task.optimization = optimization
     task.task_id = task_id
     if dependencies is not None:
-        task.task["dependencies"] = sorted(dependencies)
+        # The dependencies dict is converted from a dict to a list during the
+        # optimization phase. This is pretty confusing and means this utility
+        # might break assumptions about the format of 'dependencies'.
+        if isinstance(dependencies, dict):
+            task.task["dependencies"] = sorted(dependencies.values())
+        else:
+            task.task["dependencies"] = sorted(dependencies)
     return task
 
 

{taskcluster_taskgraph-17.2.0 → taskcluster_taskgraph-17.3.0}/pyproject.toml

@@ -1,7 +1,7 @@
 ### Project
 [project]
 name = "taskcluster-taskgraph"
-version = "17.2.0"
+version = "17.3.0"
 description = "Build taskcluster taskgraphs"
 readme = "README.rst"
 authors = [
@@ -29,7 +29,7 @@ dependencies = [
   "redo>=2.0",
   "requests>=2.25",
   "slugid>=2.0",
-  "taskcluster>=91.0",
+  "taskcluster>=91.0,<92.0",
   "taskcluster-urls>=11.0",
   "voluptuous>=0.12.1",
 ]

{taskcluster_taskgraph-17.2.0 → taskcluster_taskgraph-17.3.0}/src/taskgraph/actions/cancel.py

@@ -5,8 +5,6 @@
 
 import logging
 
-import requests
-
 from taskcluster import TaskclusterRestFailure
 from taskgraph.util.taskcluster import cancel_task
 
@@ -28,14 +26,8 @@ def cancel_action(parameters, graph_config, input, task_group_id, task_id):
     # only cancel tasks with the level-specific schedulerId.
     try:
         cancel_task(task_id)
-    except (requests.HTTPError, TaskclusterRestFailure) as e:
-        status_code = None
-        if isinstance(e, requests.HTTPError):
-            status_code = e.response.status_code if e.response else None
-        elif isinstance(e, TaskclusterRestFailure):
-            status_code = e.status_code
-
-        if status_code == 409:
+    except TaskclusterRestFailure as e:
+        if e.status_code == 409:
             # A 409 response indicates that this task is past its deadline.  It
             # cannot be cancelled at this time, but it's also not running
             # anymore, so we can ignore this error.

{taskcluster_taskgraph-17.2.0 → taskcluster_taskgraph-17.3.0}/src/taskgraph/actions/cancel_all.py

@@ -7,8 +7,6 @@ import logging
 import os
 from concurrent import futures
 
-import requests
-
 from taskcluster import TaskclusterRestFailure
 from taskgraph.util.taskcluster import (
     CONCURRENCY,
@@ -37,14 +35,8 @@ def cancel_all_action(parameters, graph_config, input, task_group_id, task_id):
         logger.info(f"Cancelling task {task_id}")
         try:
             cancel_task(task_id)
-        except (requests.HTTPError, TaskclusterRestFailure) as e:
-            status_code = None
-            if isinstance(e, requests.HTTPError):
-                status_code = e.response.status_code if e.response else None
-            elif isinstance(e, TaskclusterRestFailure):
-                status_code = e.status_code
-
-            if status_code == 409:
+        except TaskclusterRestFailure as e:
+            if e.status_code == 409:
                 # A 409 response indicates that this task is past its deadline.  It
                 # cannot be cancelled at this time, but it's also not running
                 # anymore, so we can ignore this error.

{taskcluster_taskgraph-17.2.0 → taskcluster_taskgraph-17.3.0}/src/taskgraph/docker.py

@@ -38,6 +38,7 @@ from taskgraph.util.taskcluster import (
 )
 
 logger = logging.getLogger(__name__)
+RUN_TASK_RE = re.compile(r"run-task(-(git|hg))?$")
 
 
 def get_image_digest(image_name: str) -> str:
@@ -188,11 +189,11 @@ def build_image(
 
         output_dir = temp_dir / "out"
         output_dir.mkdir()
-        volumes = {
+        volumes = [
             # TODO write artifacts to tmpdir
-            str(output_dir): "/workspace/out",
-            str(image_context): "/workspace/context.tar.gz",
-        }
+            (str(output_dir), "/workspace/out"),
+            (str(image_context), "/workspace/context.tar.gz"),
+        ]
 
         assert label in image_tasks
         task = image_tasks[label]
@@ -211,7 +212,7 @@ def build_image(
                 parent = task.dependencies["parent"][len("docker-image-") :]
                 parent_tar = temp_dir / "parent.tar"
                 build_image(graph_config, parent, save_image=str(parent_tar))
-                volumes[str(parent_tar)] = "/workspace/parent.tar"
+                volumes.append((str(parent_tar), "/workspace/parent.tar"))
 
         task_def["payload"]["env"]["CHOWN_OUTPUT"] = f"{os.getuid()}:{os.getgid()}"
         load_task(
@@ -418,6 +419,11 @@ def _resolve_image(image: Union[str, dict[str, str]], graph_config: GraphConfig)
     return load_image_by_task_id(image_task_id)
 
 
+def _is_run_task(task_def: dict[str, str]):
+    cmd = task_def["payload"].get("command")  # type: ignore
+    return cmd and re.search(RUN_TASK_RE, cmd[0])
+
+
 def load_task(
     graph_config: GraphConfig,
     task: Union[str, dict[str, Any]],
@@ -425,7 +431,7 @@ def load_task(
     user: Optional[str] = None,
     custom_image: Optional[str] = None,
     interactive: Optional[bool] = False,
-    volumes: Optional[dict[str, str]] = None,
+    volumes: Optional[list[tuple[str, str]]] = None,
 ) -> int:
     """Load and run a task interactively in a Docker container.
 
@@ -465,9 +471,9 @@ def load_task(
 
         return 1
 
-    task_command = task_def["payload"].get("command")  # type: ignore
-    if interactive and (not task_command or not task_command[0].endswith("run-task")):
-        logger.error("Only tasks using `run-task` are supported with interactive!")
+    is_run_task = _is_run_task(task_def)
+    if interactive and not is_run_task:
+        logger.error("Only tasks using `run-task` are supported with --interactive!")
         return 1
 
     try:
@@ -477,6 +483,7 @@ def load_task(
         logger.exception(e)
         return 1
 
+    task_command = task_def["payload"].get("command")  # type: ignore
     exec_command = task_cwd = None
     if interactive:
         # Remove the payload section of the task's command. This way run-task will
@@ -523,9 +530,20 @@ def load_task(
     env.update(task_def["payload"].get("env", {}))  # type: ignore
 
     # run-task expects the worker to mount a volume for each path defined in
-    # TASKCLUSTER_CACHES, delete them to avoid needing to do the same.
+    # TASKCLUSTER_CACHES; delete them to avoid needing to do the same, unless
+    # they're passed in as volumes.
     if "TASKCLUSTER_CACHES" in env:
-        del env["TASKCLUSTER_CACHES"]
+        if volumes:
+            caches = env["TASKCLUSTER_CACHES"].split(";")
+            caches = [
+                cache for cache in caches if any(path == cache for _, path in volumes)
+            ]
+        else:
+            caches = []
+        if caches:
+            env["TASKCLUSTER_CACHES"] = ";".join(caches)
+        else:
+            del env["TASKCLUSTER_CACHES"]
 
     envfile = None
     initfile = None
@@ -570,7 +588,7 @@ def load_task(
             command.extend(["-v", f"{initfile.name}:/builds/worker/.bashrc"])
 
         if volumes:
-            for k, v in volumes.items():
+            for k, v in volumes:
                 command.extend(["-v", f"{k}:{v}"])
 
         command.append(image_tag)

{taskcluster_taskgraph-17.2.0 → taskcluster_taskgraph-17.3.0}/src/taskgraph/graph.py

@@ -97,14 +97,18 @@ class Graph(_Graph):
                 indegree[dependent] -= 1
                 if indegree[dependent] == 0:
                     queue.append(dependent)
+        loopy_nodes = {node for node, degree in indegree.items() if degree > 0}
+        if loopy_nodes:
+            raise Exception(
+                f"Dependency loop detected involving the following nodes: {loopy_nodes}"
+            )
 
     def visit_postorder(self):
         """
         Generate a sequence of nodes in postorder, such that every node is
         visited *after* any nodes it links to.
 
-        Behavior is undefined (read: it will hang) if the graph contains a
-        cycle.
+        Raises an exception if the graph contains a cycle.
         """
         return self._visit(False)
 

{taskcluster_taskgraph-17.2.0 → taskcluster_taskgraph-17.3.0}/src/taskgraph/main.py

@@ -792,6 +792,14 @@ def image_digest(args):
     default="taskcluster",
     help="Relative path to the root of the Taskgraph definition.",
 )
+@argument(
+    "--volume",
+    "-v",
+    metavar="HOST_DIR:CONTAINER_DIR",
+    default=[],
+    action="append",
+    help="Mount local path into the container.",
+)
 def load_task(args):
     from taskgraph.config import load_graph_config  # noqa: PLC0415
     from taskgraph.docker import load_task  # noqa: PLC0415
@@ -806,6 +814,19 @@ def load_task(args):
         except ValueError:
             args["task"] = data  # assume it is a taskId
 
+    volumes = []
+    for vol in args["volume"]:
+        if ":" not in vol:
+            raise ValueError(
+                "Invalid volume specification '{vol}', expected HOST_DIR:CONTAINER_DIR"
+            )
+        k, v = vol.split(":", 1)
+        if not k or not v:
+            raise ValueError(
+                "Invalid volume specification '{vol}', expected HOST_DIR:CONTAINER_DIR"
+            )
+        volumes.append((k, v))
+
     root = args["root"]
     graph_config = load_graph_config(root)
     return load_task(
@@ -815,6 +836,7 @@ def load_task(args):
         remove=args["remove"],
         user=args["user"],
         custom_image=args["image"],
+        volumes=volumes,
     )
 
 

{taskcluster_taskgraph-17.2.0 → taskcluster_taskgraph-17.3.0}/src/taskgraph/optimize/base.py

@@ -437,6 +437,7 @@ def get_subgraph(
         )
         deps = task.task.setdefault("dependencies", [])
         deps.extend(sorted(named_task_dependencies.values()))
+        task.dependencies.update(named_task_dependencies)
         tasks_by_taskid[task.task_id] = task
 
     # resolve edges to taskIds

{taskcluster_taskgraph-17.2.0 → taskcluster_taskgraph-17.3.0}/src/taskgraph/run-task/run-task

@@ -16,12 +16,10 @@ current time to improve log usefulness.
 import argparse
 import datetime
 import errno
-import grp
 import io
 import json
 import os
 import platform
-import pwd
 import re
 import shutil
 import socket
@@ -33,7 +31,7 @@ import time
 import urllib.error
 import urllib.request
 from pathlib import Path
-from typing import Optional
+from typing import Dict, Optional
 
 SECRET_BASEURL_TPL = "{}/secrets/v1/secret/{{}}".format(
     os.environ.get("TASKCLUSTER_PROXY_URL", "http://taskcluster").rstrip("/")
@@ -311,6 +309,9 @@ def run_command(prefix, args, *, extra_env=None, cwd=None):
 
 
 def get_posix_user_group(user, group):
+    import grp  # noqa: PLC0415
+    import pwd  # noqa: PLC0415
+
     try:
         user_record = pwd.getpwnam(user)
     except KeyError:
@@ -572,7 +573,7 @@ def git_fetch(
     remote: str = "origin",
     tags: bool = False,
     shallow: bool = False,
-    env: Optional[dict[str, str]] = None,
+    env: Optional[Dict[str, str]] = None,
 ):
     args = ["git", "fetch"]
     if tags:

{taskcluster_taskgraph-17.2.0 → taskcluster_taskgraph-17.3.0}/src/taskgraph/transforms/cached_tasks.py

@@ -3,9 +3,8 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 
-from collections import deque
-
 import taskgraph
+from taskgraph.graph import Graph
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.cached_tasks import add_optimization
 
@@ -15,25 +14,19 @@ transforms = TransformSequence()
 def order_tasks(config, tasks):
     """Iterate image tasks in an order where parent tasks come first."""
     kind_prefix = config.kind + "-"
-
-    pending = deque(tasks)
-    task_labels = {task["label"] for task in pending}
-    emitted = set()
-    while True:
-        try:
-            task = pending.popleft()
-        except IndexError:
-            break
-        parents = {
-            task
-            for task in task.get("dependencies", {}).values()
-            if task.startswith(kind_prefix)
-        }
-        if parents and not emitted.issuperset(parents & task_labels):
-            pending.append(task)
-            continue
-        emitted.add(task["label"])
-        yield task
+    pending = {task["label"]: task for task in tasks}
+    nodes = set(pending)
+    edges = {
+        (label, dep, "")
+        for label in pending
+        for dep in pending[label].get("dependencies", {}).values()
+        if dep.startswith(kind_prefix)
+    }
+    graph = Graph(nodes, edges)
+
+    for label in graph.visit_postorder():
+        yield pending.pop(label)
+    assert not pending
 
 
 def format_task_digest(cached_task):

{taskcluster_taskgraph-17.2.0 → taskcluster_taskgraph-17.3.0}/src/taskgraph/transforms/task.py

@@ -11,7 +11,6 @@ complexities of worker implementations, scopes, and treeherder annotations.
 import functools
 import hashlib
 import os
-import re
 import time
 from copy import deepcopy
 from dataclasses import dataclass
@@ -20,7 +19,6 @@ from typing import Callable
 
 from voluptuous import All, Any, Extra, NotIn, Optional, Required
 
-from taskgraph import MAX_DEPENDENCIES
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.hash import hash_path
 from taskgraph.util.keyed_by import evaluate_keyed_by
@@ -44,7 +42,7 @@ RUN_TASK = os.path.join(
 
 
 @functools.cache
-def _run_task_suffix():
+def run_task_suffix():
     """String to append to cache names under control of run-task."""
     return hash_path(RUN_TASK)[0:20]
 
@@ -715,7 +713,7 @@ def build_docker_worker_payload(config, task, task_def):
         cache_version = "v3"
 
         if run_task:
-            suffix = f"{cache_version}-{_run_task_suffix()}"
+            suffix = f"{cache_version}-{run_task_suffix()}"
 
             if out_of_tree_image:
                 name_hash = hashlib.sha256(
@@ -763,8 +761,6 @@ def build_docker_worker_payload(config, task, task_def):
     if capabilities:
         payload["capabilities"] = capabilities
 
-    check_caches_are_volumes(task)
-
 
 @payload_builder(
     "generic-worker",
@@ -1457,135 +1453,3 @@ def chain_of_trust(config, tasks):
                     "task-reference": "<docker-image>"
                 }
         yield task
-
-
-@transforms.add
-def check_task_identifiers(config, tasks):
-    """Ensures that all tasks have well defined identifiers:
-    ``^[a-zA-Z0-9_-]{1,38}$``
-    """
-    e = re.compile("^[a-zA-Z0-9_-]{1,38}$")
-    for task in tasks:
-        for attrib in ("workerType", "provisionerId"):
-            if not e.match(task["task"][attrib]):
-                raise Exception(
-                    "task {}.{} is not a valid identifier: {}".format(
-                        task["label"], attrib, task["task"][attrib]
-                    )
-                )
-        yield task
-
-
-@transforms.add
-def check_task_dependencies(config, tasks):
-    """Ensures that tasks don't have more than 100 dependencies."""
-    for task in tasks:
-        number_of_dependencies = (
-            len(task["dependencies"])
-            + len(task["if-dependencies"])
-            + len(task["soft-dependencies"])
-        )
-        if number_of_dependencies > MAX_DEPENDENCIES:
-            raise Exception(
-                "task {}/{} has too many dependencies ({} > {})".format(
-                    config.kind,
-                    task["label"],
-                    number_of_dependencies,
-                    MAX_DEPENDENCIES,
-                )
-            )
-        yield task
-
-
-def check_caches_are_volumes(task):
-    """Ensures that all cache paths are defined as volumes.
-
-    Caches and volumes are the only filesystem locations whose content
-    isn't defined by the Docker image itself. Some caches are optional
-    depending on the task environment. We want paths that are potentially
-    caches to have as similar behavior regardless of whether a cache is
-    used. To help enforce this, we require that all paths used as caches
-    to be declared as Docker volumes. This check won't catch all offenders.
-    But it is better than nothing.
-    """
-    volumes = set(task["worker"]["volumes"])
-    paths = {c["mount-point"] for c in task["worker"].get("caches", [])}
-    missing = paths - volumes
-
-    if not missing:
-        return
-
-    raise Exception(
-        "task {} (image {}) has caches that are not declared as "
-        "Docker volumes: {} "
-        "(have you added them as VOLUMEs in the Dockerfile?)".format(
-            task["label"], task["worker"]["docker-image"], ", ".join(sorted(missing))
-        )
-    )
-
-
-@transforms.add
-def check_run_task_caches(config, tasks):
-    """Audit for caches requiring run-task.
-
-    run-task manages caches in certain ways. If a cache managed by run-task
-    is used by a non run-task task, it could cause problems. So we audit for
-    that and make sure certain cache names are exclusive to run-task.
-
-    IF YOU ARE TEMPTED TO MAKE EXCLUSIONS TO THIS POLICY, YOU ARE LIKELY
-    CONTRIBUTING TECHNICAL DEBT AND WILL HAVE TO SOLVE MANY OF THE PROBLEMS
-    THAT RUN-TASK ALREADY SOLVES. THINK LONG AND HARD BEFORE DOING THAT.
-    """
-    re_reserved_caches = re.compile(
-        """^
-        (checkouts|tooltool-cache)
-    """,
-        re.VERBOSE,
-    )
-
-    cache_prefix = "{trust_domain}-level-{level}-".format(
-        trust_domain=config.graph_config["trust-domain"],
-        level=config.params["level"],
-    )
-
-    suffix = _run_task_suffix()
-
-    for task in tasks:
-        payload = task["task"].get("payload", {})
-        command = payload.get("command") or [""]
-
-        main_command = command[0] if isinstance(command[0], str) else ""
-        run_task = main_command.endswith("run-task")
-
-        for cache in payload.get("cache", {}).get(
-            "task-reference", payload.get("cache", {})
-        ):
-            if not cache.startswith(cache_prefix):
-                raise Exception(
-                    "{} is using a cache ({}) which is not appropriate "
-                    "for its trust-domain and level. It should start with {}.".format(
-                        task["label"], cache, cache_prefix
-                    )
-                )
-
-            cache = cache[len(cache_prefix) :]
-
-            if not re_reserved_caches.match(cache):
-                continue
-
-            if not run_task:
-                raise Exception(
-                    f"{task['label']} is using a cache ({cache}) reserved for run-task "
-                    "change the task to use run-task or use a different "
-                    "cache name"
-                )
-
-            if suffix not in cache:
-                raise Exception(
-                    f"{task['label']} is using a cache ({cache}) reserved for run-task "
-                    "but the cache name is not dependent on the contents "
-                    "of run-task; change the cache name to conform to the "
-                    "naming requirements"
-                )
-
-        yield task

{taskcluster_taskgraph-17.2.0 → taskcluster_taskgraph-17.3.0}/src/taskgraph/util/taskcluster.py

@@ -467,15 +467,11 @@ def get_ancestors(task_ids: Union[list[str], str]) -> dict[str, str]:
     for task_id in task_ids:
         try:
             task_def = get_task_definition(task_id)
-        except (requests.HTTPError, taskcluster.TaskclusterRestFailure) as e:
+        except taskcluster.TaskclusterRestFailure as e:
             # Task has most likely expired, which means it's no longer a
             # dependency for the purposes of this function.
-            if isinstance(e, requests.HTTPError):
-                if e.response and e.response.status_code == 404:
-                    continue
-            elif isinstance(e, taskcluster.TaskclusterRestFailure):
-                if e.status_code == 404:
-                    continue
+            if e.status_code == 404:
+                continue
             raise e
 
         dependencies = task_def.get("dependencies", [])