taskcluster-taskgraph 5.4.0__tar.gz → 5.6.0__tar.gz

{taskcluster-taskgraph-5.4.0/src/taskcluster_taskgraph.egg-info → taskcluster-taskgraph-5.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: taskcluster-taskgraph
-Version: 5.4.0
+Version: 5.6.0
 Summary: Build taskcluster taskgraphs
 Home-page: https://github.com/taskcluster/taskgraph
 Classifier: Development Status :: 5 - Production/Stable

{taskcluster-taskgraph-5.4.0 → taskcluster-taskgraph-5.6.0/src/taskcluster_taskgraph.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: taskcluster-taskgraph
-Version: 5.4.0
+Version: 5.6.0
 Summary: Build taskcluster taskgraphs
 Home-page: https://github.com/taskcluster/taskgraph
 Classifier: Development Status :: 5 - Production/Stable

{taskcluster-taskgraph-5.4.0 → taskcluster-taskgraph-5.6.0}/src/taskgraph/__init__.py

@@ -2,7 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-__version__ = "5.4.0"
+__version__ = "5.6.0"
 
 # Maximum number of dependencies a single task can have
 # https://docs.taskcluster.net/reference/platform/taskcluster-queue/references/api#createTask

{taskcluster-taskgraph-5.4.0 → taskcluster-taskgraph-5.6.0}/src/taskgraph/config.py

@@ -35,6 +35,11 @@ graph_config_schema = Schema(
                 "lowest",
             ),
         ),
+        Optional(
+            "task-deadline-after",
+            description="Default 'deadline' for tasks, in relative date format. "
+            "Eg: '1 week'",
+        ): optionally_keyed_by("project", str),
         Required("workers"): {
             Required("aliases"): {
                 str: {

{taskcluster-taskgraph-5.4.0 → taskcluster-taskgraph-5.6.0}/src/taskgraph/run-task/robustcheckout.py

@@ -41,7 +41,9 @@ from mercurial import (
 # Causes worker to purge caches on process exit and for task to retry.
 EXIT_PURGE_CACHE = 72
 
-testedwith = b"4.5 4.6 4.7 4.8 4.9 5.0 5.1 5.2 5.3 5.4 5.5 5.6 5.7 5.8 5.9"
+testedwith = (
+    b"4.5 4.6 4.7 4.8 4.9 5.0 5.1 5.2 5.3 5.4 5.5 5.6 5.7 5.8 5.9 6.0 6.1 6.2 6.3 6.4"
+)
 minimumhgversion = b"4.5"
 
 cmdtable = {}
@@ -496,6 +498,10 @@ def _docheckout(
             ui.warn(b"ssl error: %s\n" % pycompat.bytestr(str(e)))
             handlenetworkfailure()
             return True
+        elif isinstance(e, urllibcompat.urlerr.httperror) and e.code >= 500:
+            ui.warn(b"http error: %s\n" % pycompat.bytestr(str(e.reason)))
+            handlenetworkfailure()
+            return True
         elif isinstance(e, urllibcompat.urlerr.urlerror):
             if isinstance(e.reason, socket.error):
                 ui.warn(b"socket error: %s\n" % pycompat.bytestr(str(e.reason)))
@@ -806,7 +812,14 @@ def _docheckout(
             # one to change the sparse profile and another to update to the new
             # revision. This is not desired. But there's not a good API in
             # Mercurial to do this as one operation.
-            with repo.wlock(), repo.dirstate.parentchange(), timeit(
+            # TRACKING hg64 - Mercurial 6.4 and later require call to
+            # dirstate.changing_parents(repo)
+            def parentchange(repo):
+                if util.safehasattr(repo.dirstate, "changing_parents"):
+                    return repo.dirstate.changing_parents(repo)
+                return repo.dirstate.parentchange()
+
+            with repo.wlock(), parentchange(repo), timeit(
                 "sparse_update_config", "sparse-update-config"
             ):
                 # pylint --py3k: W1636

{taskcluster-taskgraph-5.4.0 → taskcluster-taskgraph-5.6.0}/src/taskgraph/run-task/run-task

@@ -14,20 +14,17 @@ current time to improve log usefulness.
 """
 
 import sys
-from typing import Optional
 
 if sys.version_info[0:2] < (3, 5):
     print("run-task requires Python 3.5+")
     sys.exit(1)
 
-
 import argparse
 import datetime
 import errno
 import io
 import json
 import os
-from pathlib import Path
 import platform
 import re
 import shutil
@@ -36,11 +33,11 @@ import socket
 import stat
 import subprocess
 import time
-
 import urllib.error
 import urllib.request
-
+from pathlib import Path
 from threading import Thread
+from typing import Optional
 
 SECRET_BASEURL_TPL = "http://taskcluster/secrets/v1/secret/{}"
 
@@ -711,6 +708,7 @@ def git_checkout(
             "git",
             "submodule",
             "update",
+            "--force",  # Overrides any potential local changes
         ]
 
         run_required_command(b"vcs", args, cwd=destination_path)
@@ -926,7 +924,6 @@ def collect_vcs_options(args, project, name):
 
 
 def vcs_checkout_from_args(options):
-
     if not options["checkout"]:
         if options["ref"] and not options["revision"]:
             print("task should be defined in terms of non-symbolic revision")
@@ -1072,10 +1069,10 @@ def maybe_run_resource_monitoring():
     monitor_process.start()
     return process
 
+
 def _display_python_version():
     print_line(
-        b"setup",
-        b"Python version: "+ bytes(platform.python_version(), 'utf-8')
+        b"setup", b"Python version: %s\n" % platform.python_version().encode("utf-8")
     )
 
 

{taskcluster-taskgraph-5.4.0 → taskcluster-taskgraph-5.6.0}/src/taskgraph/transforms/job/common.py

@@ -60,31 +60,6 @@ def add_cache(job, taskdesc, name, mount_point, skip_untrusted=False):
         pass
 
 
-def docker_worker_add_workspace_cache(config, job, taskdesc, extra=None):
-    """Add the workspace cache.
-
-    Args:
-        config (TransformConfig): Transform configuration object.
-        job (dict): Task's job description.
-        taskdesc (dict): Target task description to modify.
-        extra (str): Optional context passed in that supports extending the cache
-            key name to avoid undesired conflicts with other caches.
-    """
-    cache_name = "{}-build-{}-{}-workspace".format(
-        config.params["project"],
-        taskdesc["attributes"]["build_platform"],
-        taskdesc["attributes"]["build_type"],
-    )
-    if extra:
-        cache_name = f"{cache_name}-{extra}"
-
-    mount_point = "{workdir}/workspace".format(**job["run"])
-
-    # Don't enable the workspace cache when we can't guarantee its
-    # behavior, like on Try.
-    add_cache(job, taskdesc, cache_name, mount_point, skip_untrusted=True)
-
-
 def add_artifacts(config, job, taskdesc, path):
     taskdesc["worker"].setdefault("artifacts", []).append(
         {

{taskcluster-taskgraph-5.4.0 → taskcluster-taskgraph-5.6.0}/src/taskgraph/transforms/task.py

@@ -221,6 +221,13 @@ def get_default_priority(graph_config, project):
     )
 
 
+@memoize
+def get_default_deadline(graph_config, project):
+    return evaluate_keyed_by(
+        graph_config["task-deadline-after"], "Graph Config", {"project": project}
+    )
+
+
 # define a collection of payload builders, depending on the worker implementation
 payload_builders = {}
 
@@ -617,6 +624,11 @@ def build_docker_worker_payload(config, task, task_def):
         Required("env"): {str: taskref_or_string},
         # the maximum time to run, in seconds
         Required("max-run-time"): int,
+        # the exit status code(s) that indicates the task should be retried
+        Optional("retry-exit-status"): [int],
+        # the exit status code(s) that indicates the caches used by the task
+        # should be purged
+        Optional("purge-caches-exit-status"): [int],
         # os user groups for test task workers
         Optional("os-groups"): [str],
         # feature for test task to run as administarotr
@@ -639,6 +651,8 @@ def build_generic_worker_payload(config, task, task_def):
     on_exit_status = {}
     if "retry-exit-status" in worker:
         on_exit_status["retry"] = worker["retry-exit-status"]
+    if "purge-caches-exit-status" in worker:
+        on_exit_status["purgeCaches"] = worker["purge-caches-exit-status"]
     if worker["os"] == "windows":
         on_exit_status.setdefault("retry", []).extend(
             [
@@ -1062,7 +1076,12 @@ def build_task(config, tasks):
             task["expires-after"] = "28 days" if config.params.is_try() else "1 year"
 
         if "deadline-after" not in task:
-            task["deadline-after"] = "1 day"
+            if "task-deadline-after" in config.graph_config:
+                task["deadline-after"] = get_default_deadline(
+                    config.graph_config, config.params["project"]
+                )
+            else:
+                task["deadline-after"] = "1 day"
 
         if "priority" not in task:
             task["priority"] = get_default_priority(

{taskcluster-taskgraph-5.4.0 → taskcluster-taskgraph-5.6.0}/src/taskgraph/util/dependencies.py

@@ -27,6 +27,11 @@ def group_by_single(config, tasks):
         yield [task]
 
 
+@group_by("all")
+def group_by_all(config, tasks):
+    return [[task for task in tasks]]
+
+
 @group_by("attribute", schema=Schema(str))
 def group_by_attribute(config, tasks, attr):
     groups = {}

{taskcluster-taskgraph-5.4.0 → taskcluster-taskgraph-5.6.0}/src/taskgraph/util/vcs.py

@@ -166,7 +166,10 @@ class Repository(ABC):
     @abstractmethod
     def find_latest_common_revision(self, base_ref_or_rev, head_rev):
         """Find the latest revision that is common to both the given
-        ``head_rev`` and ``base_ref_or_rev``"""
+        ``head_rev`` and ``base_ref_or_rev``.
+
+        If no common revision exists, ``Repository.NULL_REVISION`` will
+        be returned."""
 
     @abstractmethod
     def does_revision_exist_locally(self, revision):
@@ -296,13 +299,14 @@ class HgRepository(Repository):
         return self.run("update", "--check", ref)
 
     def find_latest_common_revision(self, base_ref_or_rev, head_rev):
-        return self.run(
+        ancestor = self.run(
             "log",
             "-r",
             f"last(ancestors('{base_ref_or_rev}') and ancestors('{head_rev}'))",
             "--template",
             "{node}",
         ).strip()
+        return ancestor or self.NULL_REVISION
 
     def does_revision_exist_locally(self, revision):
         try:
@@ -482,7 +486,10 @@ class GitRepository(Repository):
         self.run("checkout", ref)
 
     def find_latest_common_revision(self, base_ref_or_rev, head_rev):
-        return self.run("merge-base", base_ref_or_rev, head_rev).strip()
+        try:
+            return self.run("merge-base", base_ref_or_rev, head_rev).strip()
+        except subprocess.CalledProcessError:
+            return self.NULL_REVISION
 
     def does_revision_exist_locally(self, revision):
         try:

{taskcluster-taskgraph-5.4.0 → taskcluster-taskgraph-5.6.0}/test/test_transforms_from_deps.py

@@ -74,6 +74,18 @@ def assert_copy_attributes(tasks):
     }
 
 
+def assert_group_by_all(tasks):
+    handle_exception(tasks)
+    assert len(tasks) == 1
+    assert tasks[0]["dependencies"] == {"foo": "a", "bar": "bar-b"}
+
+
+def assert_group_by_all_dupe_allowed(tasks):
+    handle_exception(tasks)
+    assert len(tasks) == 1
+    assert tasks[0]["dependencies"] == {"a": "a", "b": "b", "c": "c"}
+
+
 @pytest.mark.parametrize(
     "task, kind_config, deps",
     (
@@ -178,6 +190,37 @@ def assert_copy_attributes(tasks):
             },
             id="copy_attributes",
         ),
+        pytest.param(
+            # task
+            {
+                "from-deps": {
+                    "group-by": "all",
+                }
+            },
+            # kind config
+            None,
+            # deps
+            None,
+            id="group_by_all",
+        ),
+        pytest.param(
+            # task
+            {
+                "from-deps": {
+                    "unique-kinds": False,
+                    "group-by": "all",
+                }
+            },
+            # kind config
+            None,
+            # deps
+            {
+                "a": make_task("a", kind="foo"),
+                "b": make_task("b", kind="foo"),
+                "c": make_task("c", kind="foo"),
+            },
+            id="group_by_all_dupe_allowed",
+        ),
     ),
 )
 def test_transforms(

{taskcluster-taskgraph-5.4.0 → taskcluster-taskgraph-5.6.0}/test/test_transforms_task.py

@@ -794,3 +794,77 @@ def test_check_task_dependencies(graph_config, test_task, expectation):
         assert (
             len(list(task.check_task_dependencies(transform_config, [test_task]))) == 1
         )
+
+
+@pytest.mark.parametrize(
+    "deadline_after, test_task",
+    (
+        (
+            None,
+            {
+                "description": "fake description",
+                "name": "fake-task-name",
+                "worker-type": "t-linux",
+                "worker": {
+                    "docker-image": "fake-image-name",
+                    "max-run-time": 1800,
+                },
+            },
+        ),
+        (
+            "2 weeks",
+            {
+                "description": "fake description",
+                "name": "fake-task-name",
+                "worker-type": "t-linux",
+                "worker": {
+                    "docker-image": "fake-image-name",
+                    "max-run-time": 1800,
+                },
+            },
+        ),
+    ),
+)
+def test_default_deadline_after(run_transform, graph_config, deadline_after, test_task):
+    if deadline_after:
+        graph_config._config["task-deadline-after"] = deadline_after
+
+    params = FakeParameters(
+        {
+            "base_repository": "git@github.com://github.com/mozilla/example.git",
+            "build_date": 0,
+            "build_number": 1,
+            "head_repository": "git@github.com://github.com/mozilla/example.git",
+            "head_rev": "abcdef",
+            "head_ref": "default",
+            "level": "1",
+            "moz_build_date": 0,
+            "next_version": "1.0.1",
+            "owner": "some-owner",
+            "project": "some-project",
+            "pushlog_id": 1,
+            "repository_type": "git",
+            "target_tasks_method": "test_method",
+            "tasks_for": "github-pull-request",
+            "try_mode": None,
+            "version": "1.0.0",
+        },
+    )
+
+    transform_config = TransformConfig(
+        "check_deadline",
+        str(here),
+        {},
+        params,
+        {},
+        graph_config,
+        write_artifacts=False,
+    )
+
+    task_dict = deepcopy(test_task)
+
+    task_dict = run_transform(task.transforms, task_dict, config=transform_config)[0]
+    if deadline_after:
+        assert task_dict["task"]["deadline"] == {"relative-datestamp": deadline_after}
+    else:
+        assert task_dict["task"]["deadline"] == {"relative-datestamp": "1 day"}

{taskcluster-taskgraph-5.4.0 → taskcluster-taskgraph-5.6.0}/test/test_util_vcs.py

@@ -421,7 +421,17 @@ def test_find_latest_common_revision(repo_with_remote):
             repo.find_latest_common_revision(base_ref, repo.head_rev)
             == expected_latest_common_revision
         )
-    else:
+
+        # Test no common ancestors
+        repo.run("checkout", "--orphan", "new_branch")
+        repo.run("add", ".")
+        repo.run("commit", "-m", "Add another new revision")
+        base_ref = f"{remote_name}/{repo.branch}"
+        assert (
+            repo.find_latest_common_revision(base_ref, repo.head_rev)
+            == Repository.NULL_REVISION
+        )
+    elif repo.tool == "hg":
         # hg doesn't have the concept of remote branches
         assert (
             repo.find_latest_common_revision(
@@ -430,6 +440,20 @@ def test_find_latest_common_revision(repo_with_remote):
             == expected_latest_common_revision
         )
 
+        # Test no common ancestors
+        repo.run("update", Repository.NULL_REVISION)
+        with open(os.path.join(repo.path, "some_file"), "w") as f:
+            f.write("some content")
+
+        repo.run("add", ".")
+        repo.run("commit", "-m", "Add another new revision")
+        assert (
+            repo.find_latest_common_revision(
+                repo.head_rev, expected_latest_common_revision
+            )
+            == Repository.NULL_REVISION
+        )
+
 
 def test_does_revision_exist_locally(repo):
     first_revision = repo.head_rev

{taskcluster-taskgraph-5.4.0 → taskcluster-taskgraph-5.6.0}/src/taskcluster_taskgraph.egg-info/requires.txt

@@ -1,8 +1,8 @@
-PyYAML>=5.4
 appdirs>=1.4
 cookiecutter~=2.1
 json-e>=2.7
 mozilla-repo-urls
+PyYAML>=5.4
 redo>=2.0
 requests>=2.25
 requests_unixsocket>=0.2