taskcluster-taskgraph 7.2.1__tar.gz → 7.3.0__tar.gz

taskcluster-taskgraph-7.3.0/PKG-INFO

@@ -0,0 +1,112 @@
+Metadata-Version: 2.1
+Name: taskcluster-taskgraph
+Version: 7.3.0
+Summary: Build taskcluster taskgraphs
+Home-page: https://github.com/taskcluster/taskgraph
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Environment :: Console
+Classifier: License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development
+Provides-Extra: load-image
+License-File: LICENSE
+
+
+.. image:: https://firefox-ci-tc.services.mozilla.com/api/github/v1/repository/taskcluster/taskgraph/main/badge.svg
+   :target: https://firefox-ci-tc.services.mozilla.com/api/github/v1/repository/taskcluster/taskgraph/main/latest
+   :alt: Task Status
+
+.. image:: https://results.pre-commit.ci/badge/github/taskcluster/taskgraph/main.svg
+   :target: https://results.pre-commit.ci/latest/github/taskcluster/taskgraph/main
+   :alt: pre-commit.ci status
+
+.. image:: https://codecov.io/gh/taskcluster/taskgraph/branch/main/graph/badge.svg?token=GJIV52ZQNP
+   :target: https://codecov.io/gh/taskcluster/taskgraph
+   :alt: Code Coverage
+
+.. image:: https://badge.fury.io/py/taskcluster-taskgraph.svg
+   :target: https://badge.fury.io/py/taskcluster-taskgraph
+   :alt: Pypi Version
+
+.. image:: https://readthedocs.org/projects/taskcluster-taskgraph/badge/?version=latest
+   :target: https://taskcluster-taskgraph.readthedocs.io/en/latest/?badge=latest
+   :alt: Documentation Status
+
+.. image:: https://img.shields.io/badge/license-MPL%202.0-orange.svg
+   :target: http://mozilla.org/MPL/2.0
+   :alt: License
+
+Taskgraph
+=========
+
+Taskgraph is a Python library to generate graphs of tasks for the `Taskcluster
+CI`_ service. It is the recommended approach for configuring tasks once your
+project outgrows a single `.taskcluster.yml`_ file and is what powers the over
+30,000 tasks and counting that make up Firefox's CI.
+
+For more information and usage instructions, `see the docs`_.
+
+How It Works
+------------
+
+Taskgraph leverages the fact that Taskcluster is a generic task execution
+platform. This means that tasks can be scheduled via its `comprehensive API`_,
+and aren't limited to being triggered in response to supported events.
+
+Taskgraph leverages this execution platform to allow CI systems to scale to any
+size or complexity.
+
+1. A *decision task* is created via Taskcluster's normal `.taskcluster.yml`_
+   file. This task invokes ``taskgraph``.
+2. Taskgraph evaluates a series of yaml based task definitions (similar to
+   those other CI offerings provide).
+3. Taskgraph applies transforms on top of these task definitions. Transforms
+   are Python functions that can programmatically alter or even clone a task
+   definition.
+4. Taskgraph applies some optional optimization logic to remove unnecessary
+   tasks.
+5. Taskgraph submits the resulting *task graph* to Taskcluster via its API.
+
+Taskgraph's combination of declarative task configuration combined with
+programmatic alteration are what allow it to support CI systems of any scale.
+Taskgraph is the library that powers the 30,000+ tasks making up `Firefox's
+CI`_.
+
+.. _Taskcluster CI: https://taskcluster.net/
+.. _comprehensive API: https://docs.taskcluster.net/docs/reference/platform/queue/api
+.. _.taskcluster.yml: https://docs.taskcluster.net/docs/reference/integrations/github/taskcluster-yml-v1
+.. _Firefox's CI: https://treeherder.mozilla.org/jobs?repo=mozilla-central
+.. _see the docs: https://taskcluster-taskgraph.readthedocs.io
+
+Installation
+------------
+
+Taskgraph supports Python 3.8 and up, and can be installed from Pypi:
+
+.. code-block::
+
+  pip install taskcluster-taskgraph
+
+
+Alternatively, the repo can be cloned and installed directly:
+
+.. code-block::
+
+  git clone https://github.com/taskcluster/taskgraph
+  cd taskgraph
+  python setup.py install
+
+In both cases, it's recommended to use a Python `virtual environment`_.
+
+.. _virtual environment: https://docs.python.org/3/tutorial/venv.html
+
+Get Involved
+------------
+
+If you'd like to get involved, please see our `contributing docs`_!
+
+.. _contributing docs: https://github.com/taskcluster/taskgraph/blob/main/CONTRIBUTING.rst

{taskcluster-taskgraph-7.2.1 → taskcluster-taskgraph-7.3.0}/setup.py

@@ -13,10 +13,14 @@ with open(version_file) as fh:
 with open(os.path.join(project_dir, "requirements/base.in")) as fp:
     requirements = fp.read().splitlines()
 
+with open(os.path.join(project_dir, "README.rst")) as fh:
+    long_description = fh.read()
+
 setup(
     name="taskcluster-taskgraph",
     version=namespace["__version__"],
     description="Build taskcluster taskgraphs",
+    long_description=long_description,
     url="https://github.com/taskcluster/taskgraph",
     packages=find_packages("src"),
     package_dir={"": "src"},

taskcluster-taskgraph-7.3.0/src/taskcluster_taskgraph.egg-info/PKG-INFO

@@ -0,0 +1,112 @@
+Metadata-Version: 2.1
+Name: taskcluster-taskgraph
+Version: 7.3.0
+Summary: Build taskcluster taskgraphs
+Home-page: https://github.com/taskcluster/taskgraph
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Environment :: Console
+Classifier: License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development
+Provides-Extra: load-image
+License-File: LICENSE
+
+
+.. image:: https://firefox-ci-tc.services.mozilla.com/api/github/v1/repository/taskcluster/taskgraph/main/badge.svg
+   :target: https://firefox-ci-tc.services.mozilla.com/api/github/v1/repository/taskcluster/taskgraph/main/latest
+   :alt: Task Status
+
+.. image:: https://results.pre-commit.ci/badge/github/taskcluster/taskgraph/main.svg
+   :target: https://results.pre-commit.ci/latest/github/taskcluster/taskgraph/main
+   :alt: pre-commit.ci status
+
+.. image:: https://codecov.io/gh/taskcluster/taskgraph/branch/main/graph/badge.svg?token=GJIV52ZQNP
+   :target: https://codecov.io/gh/taskcluster/taskgraph
+   :alt: Code Coverage
+
+.. image:: https://badge.fury.io/py/taskcluster-taskgraph.svg
+   :target: https://badge.fury.io/py/taskcluster-taskgraph
+   :alt: Pypi Version
+
+.. image:: https://readthedocs.org/projects/taskcluster-taskgraph/badge/?version=latest
+   :target: https://taskcluster-taskgraph.readthedocs.io/en/latest/?badge=latest
+   :alt: Documentation Status
+
+.. image:: https://img.shields.io/badge/license-MPL%202.0-orange.svg
+   :target: http://mozilla.org/MPL/2.0
+   :alt: License
+
+Taskgraph
+=========
+
+Taskgraph is a Python library to generate graphs of tasks for the `Taskcluster
+CI`_ service. It is the recommended approach for configuring tasks once your
+project outgrows a single `.taskcluster.yml`_ file and is what powers the over
+30,000 tasks and counting that make up Firefox's CI.
+
+For more information and usage instructions, `see the docs`_.
+
+How It Works
+------------
+
+Taskgraph leverages the fact that Taskcluster is a generic task execution
+platform. This means that tasks can be scheduled via its `comprehensive API`_,
+and aren't limited to being triggered in response to supported events.
+
+Taskgraph leverages this execution platform to allow CI systems to scale to any
+size or complexity.
+
+1. A *decision task* is created via Taskcluster's normal `.taskcluster.yml`_
+   file. This task invokes ``taskgraph``.
+2. Taskgraph evaluates a series of yaml based task definitions (similar to
+   those other CI offerings provide).
+3. Taskgraph applies transforms on top of these task definitions. Transforms
+   are Python functions that can programmatically alter or even clone a task
+   definition.
+4. Taskgraph applies some optional optimization logic to remove unnecessary
+   tasks.
+5. Taskgraph submits the resulting *task graph* to Taskcluster via its API.
+
+Taskgraph's combination of declarative task configuration combined with
+programmatic alteration are what allow it to support CI systems of any scale.
+Taskgraph is the library that powers the 30,000+ tasks making up `Firefox's
+CI`_.
+
+.. _Taskcluster CI: https://taskcluster.net/
+.. _comprehensive API: https://docs.taskcluster.net/docs/reference/platform/queue/api
+.. _.taskcluster.yml: https://docs.taskcluster.net/docs/reference/integrations/github/taskcluster-yml-v1
+.. _Firefox's CI: https://treeherder.mozilla.org/jobs?repo=mozilla-central
+.. _see the docs: https://taskcluster-taskgraph.readthedocs.io
+
+Installation
+------------
+
+Taskgraph supports Python 3.8 and up, and can be installed from Pypi:
+
+.. code-block::
+
+  pip install taskcluster-taskgraph
+
+
+Alternatively, the repo can be cloned and installed directly:
+
+.. code-block::
+
+  git clone https://github.com/taskcluster/taskgraph
+  cd taskgraph
+  python setup.py install
+
+In both cases, it's recommended to use a Python `virtual environment`_.
+
+.. _virtual environment: https://docs.python.org/3/tutorial/venv.html
+
+Get Involved
+------------
+
+If you'd like to get involved, please see our `contributing docs`_!
+
+.. _contributing docs: https://github.com/taskcluster/taskgraph/blob/main/CONTRIBUTING.rst

{taskcluster-taskgraph-7.2.1 → taskcluster-taskgraph-7.3.0}/src/taskcluster_taskgraph.egg-info/SOURCES.txt

@@ -116,6 +116,7 @@ test/test_transforms_run.py
 test/test_transforms_run_run_task.py
 test/test_transforms_run_toolchain.py
 test/test_transforms_task.py
+test/test_util_archive.py
 test/test_util_attributes.py
 test/test_util_cached_tasks.py
 test/test_util_dependencies.py

{taskcluster-taskgraph-7.2.1 → taskcluster-taskgraph-7.3.0}/src/taskgraph/__init__.py

@@ -2,7 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-__version__ = "7.2.1"
+__version__ = "7.3.0"
 
 # Maximum number of dependencies a single task can have
 # https://docs.taskcluster.net/reference/platform/taskcluster-queue/references/api#createTask

{taskcluster-taskgraph-7.2.1 → taskcluster-taskgraph-7.3.0}/src/taskgraph/actions/add_new_jobs.py

@@ -40,7 +40,7 @@ from taskgraph.actions.util import (
 )
 def add_new_jobs_action(parameters, graph_config, input, task_group_id, task_id):
     decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels(
-        parameters, graph_config
+        parameters, graph_config, task_group_id=task_group_id
     )
 
     to_run = []

{taskcluster-taskgraph-7.2.1 → taskcluster-taskgraph-7.3.0}/src/taskgraph/actions/rebuild_cached_tasks.py

@@ -18,7 +18,7 @@ def rebuild_cached_tasks_action(
     parameters, graph_config, input, task_group_id, task_id
 ):
     decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels(
-        parameters, graph_config
+        parameters, graph_config, task_group_id=task_group_id
     )
     cached_tasks = [
         label

{taskcluster-taskgraph-7.2.1 → taskcluster-taskgraph-7.3.0}/src/taskgraph/actions/registry.py

@@ -163,11 +163,11 @@ def register_callback_action(
             actionPerm = "generic" if generic else cb_name
 
             # gather up the common decision-task-supplied data for this action
-            repo_param = "head_repository"
             repository = {
-                "url": parameters[repo_param],
+                "url": parameters["head_repository"],
                 "project": parameters["project"],
                 "level": parameters["level"],
+                "base_url": parameters["base_repository"],
             }
 
             revision = parameters["head_rev"]
@@ -179,6 +179,9 @@ def register_callback_action(
             branch = parameters.get("head_ref")
             if branch:
                 push["branch"] = branch
+            base_branch = parameters.get("base_ref")
+            if base_branch and branch != base_branch:
+                push["base_branch"] = base_branch
 
             action = {
                 "name": name,
@@ -213,11 +216,16 @@ def register_callback_action(
             if "/" in actionPerm:
                 raise Exception("`/` is not allowed in action names; use `-`")
 
+            if parameters["tasks_for"].startswith("github-pull-request"):
+                hookId = f"in-tree-pr-action-{level}-{actionPerm}/{tcyml_hash}"
+            else:
+                hookId = f"in-tree-action-{level}-{actionPerm}/{tcyml_hash}"
+
             rv.update(
                 {
                     "kind": "hook",
                     "hookGroupId": f"project-{trustDomain}",
-                    "hookId": f"in-tree-action-{level}-{actionPerm}/{tcyml_hash}",
+                    "hookId": hookId,
                     "hookPayload": {
                         # provide the decision-task parameters as context for triggerHook
                         "decision": {
@@ -293,16 +301,20 @@ def sanity_check_task_scope(callback, parameters, graph_config):
 
     actionPerm = "generic" if action.generic else action.cb_name
 
-    repo_param = "head_repository"
-    raw_url = parameters[repo_param]
+    raw_url = parameters["base_repository"]
     parsed_url = parse(raw_url)
-    expected_scope = f"assume:{parsed_url.taskcluster_role_prefix}:action:{actionPerm}"
+    action_scope = f"assume:{parsed_url.taskcluster_role_prefix}:action:{actionPerm}"
+    pr_action_scope = (
+        f"assume:{parsed_url.taskcluster_role_prefix}:pr-action:{actionPerm}"
+    )
 
     # the scope should appear literally; no need for a satisfaction check. The use of
     # get_current_scopes here calls the auth service through the Taskcluster Proxy, giving
     # the precise scopes available to this task.
-    if expected_scope not in taskcluster.get_current_scopes():
-        raise ValueError(f"Expected task scope {expected_scope} for this action")
+    if not set((action_scope, pr_action_scope)) & set(taskcluster.get_current_scopes()):
+        raise ValueError(
+            f"Expected task scope {action_scope} or {pr_action_scope} for this action"
+        )
 
 
 def trigger_action_callback(

{taskcluster-taskgraph-7.2.1 → taskcluster-taskgraph-7.3.0}/src/taskgraph/actions/retrigger.py

@@ -65,7 +65,9 @@ def retrigger_decision_action(parameters, graph_config, input, task_group_id, ta
     # absolute timestamps relative to the current time.
     task = taskcluster.get_task_definition(task_id)
     task = relativize_datestamps(task)
-    create_task_from_def(slugid(), task, parameters["level"])
+    create_task_from_def(
+        slugid(), task, parameters["level"], graph_config["trust-domain"]
+    )
 
 
 @register_callback_action(
@@ -142,7 +144,7 @@ def retrigger_decision_action(parameters, graph_config, input, task_group_id, ta
 )
 def retrigger_action(parameters, graph_config, input, task_group_id, task_id):
     decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels(
-        parameters, graph_config
+        parameters, graph_config, task_group_id=task_group_id
     )
 
     task = taskcluster.get_task_definition(task_id)
@@ -199,7 +201,7 @@ def rerun_action(parameters, graph_config, input, task_group_id, task_id):
     task = taskcluster.get_task_definition(task_id)
     parameters = dict(parameters)
     decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels(
-        parameters, graph_config
+        parameters, graph_config, task_group_id=task_group_id
     )
     label = task["metadata"]["name"]
     if task_id not in label_to_taskid.values():
@@ -257,7 +259,7 @@ def _rerun_task(task_id, label):
 )
 def retrigger_multiple(parameters, graph_config, input, task_group_id, task_id):
     decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels(
-        parameters, graph_config
+        parameters, graph_config, task_group_id=task_group_id
     )
 
     suffixes = []

{taskcluster-taskgraph-7.2.1 → taskcluster-taskgraph-7.3.0}/src/taskgraph/actions/util.py

@@ -32,8 +32,15 @@ def get_parameters(decision_task_id):
     return get_artifact(decision_task_id, "public/parameters.yml")
 
 
-def fetch_graph_and_labels(parameters, graph_config):
-    decision_task_id = find_decision_task(parameters, graph_config)
+def fetch_graph_and_labels(parameters, graph_config, task_group_id=None):
+    try:
+        # Look up the decision_task id in the index
+        decision_task_id = find_decision_task(parameters, graph_config)
+    except KeyError:
+        if not task_group_id:
+            raise
+        # Not found (e.g. from github-pull-request), fall back to the task group id.
+        decision_task_id = task_group_id
 
     # First grab the graph and labels generated during the initial decision task
     full_task_graph = get_artifact(decision_task_id, "public/full-task-graph.json")
@@ -90,7 +97,7 @@ def fetch_graph_and_labels(parameters, graph_config):
     return (decision_task_id, full_task_graph, label_to_taskid)
 
 
-def create_task_from_def(task_id, task_def, level):
+def create_task_from_def(task_id, task_def, level, trust_domain):
     """Create a new task from a definition rather than from a label
     that is already in the full-task-graph. The task definition will
     have {relative-datestamp': '..'} rendered just like in a decision task.
@@ -98,7 +105,7 @@ def create_task_from_def(task_id, task_def, level):
     It is useful if you want to "edit" the full_task_graph and then hand
     it to this function. No dependencies will be scheduled. You must handle
     this yourself. Seeing how create_tasks handles it might prove helpful."""
-    task_def["schedulerId"] = f"gecko-level-{level}"
+    task_def["schedulerId"] = f"{trust_domain}-level-{level}"
     label = task_def["metadata"]["name"]
     session = get_session()
     create.create_task(session, task_id, label, task_def)

{taskcluster-taskgraph-7.2.1 → taskcluster-taskgraph-7.3.0}/src/taskgraph/util/archive.py

@@ -12,6 +12,40 @@ import tarfile
 DEFAULT_MTIME = 1451606400
 
 
+# Python 3.9 contains this change:
+#  https://github.com/python/cpython/commit/674935b8caf33e47c78f1b8e197b1b77a04992d2
+# which changes the output of tar creation compared to earlier versions.
+# As this code is used to generate tar files that are meant to be deterministic
+# across versions of python (specifically, it's used as part of computing the hash
+# of docker images, which needs to be identical between CI (which uses python 3.8),
+# and developer environments (using arbitrary versions of python, at this point,
+# most probably more recent than 3.9)).
+# What we do is subblass TarInfo so that if used on python >= 3.9, it reproduces the
+# behavior from python < 3.9.
+# Here's how it goes:
+# - the behavior in python >= 3.9 is the same as python < 3.9 when the type encoded
+# in the tarinfo is CHRTYPE or BLKTYPE.
+# - the value of the type is only compared in the context of choosing which behavior
+# to take
+# - we replace the type with the same value (so that using the value has no changes)
+# but that pretends to be the same as CHRTYPE so that the condition that enables the
+# old behavior is taken.
+class HackedType(bytes):
+    def __eq__(self, other):
+        if other == tarfile.CHRTYPE:
+            return True
+        return self == other
+
+
+class TarInfo(tarfile.TarInfo):
+    @staticmethod
+    def _create_header(info, format, encoding, errors):
+        info["type"] = HackedType(info["type"])
+        # ignore type checking because it looks like pyright complains because we're calling a
+        # non-public method
+        return tarfile.TarInfo._create_header(info, format, encoding, errors)  # type: ignore
+
+
 def create_tar_from_files(fp, files):
     """Create a tar file deterministically.
 
@@ -25,15 +59,23 @@ def create_tar_from_files(fp, files):
 
     FUTURE accept a filename argument (or create APIs to write files)
     """
-    with tarfile.open(name="", mode="w", fileobj=fp, dereference=True) as tf:
+    # The format is explicitly set to tarfile.GNU_FORMAT, because this default format
+    # has been changed in Python 3.8.
+    with tarfile.open(
+        name="", mode="w", fileobj=fp, dereference=True, format=tarfile.GNU_FORMAT
+    ) as tf:
         for archive_path, f in sorted(files.items()):
             if isinstance(f, str):
-                mode = os.stat(f).st_mode
+                s = os.stat(f)
+                mode = s.st_mode
+                size = s.st_size
                 f = open(f, "rb")
             else:
                 mode = 0o0644
+                size = len(f.read())
+                f.seek(0)
 
-            ti = tarfile.TarInfo(archive_path)
+            ti = TarInfo(archive_path)
             ti.mode = mode
             ti.type = tarfile.REGTYPE
 
@@ -56,9 +98,7 @@ def create_tar_from_files(fp, files):
             # Set mtime to a constant value.
             ti.mtime = DEFAULT_MTIME
 
-            f.seek(0, 2)
-            ti.size = f.tell()
-            f.seek(0, 0)
+            ti.size = size
             # tarfile wants to pass a size argument to read(). So just
             # wrap/buffer in a proper file object interface.
             tf.addfile(ti, f)

{taskcluster-taskgraph-7.2.1 → taskcluster-taskgraph-7.3.0}/test/test_actions_registry.py

@@ -12,34 +12,40 @@ from taskgraph.actions import registry
         ("non-existing-action", {}, [], pytest.raises(ValueError)),
         (
             "retrigger",
-            {"head_repository": "https://some.git.repo"},
+            {"base_repository": "https://some.git.repo"},
             [],
             pytest.raises(InvalidRepoUrlError),
         ),
         (
             "retrigger",
-            {"head_repository": "https://hg.mozilla.org/try"},
+            {"base_repository": "https://hg.mozilla.org/try"},
             ["unrelated:scope"],
             pytest.raises(ValueError),
         ),
         (
             "retrigger",
-            {"head_repository": "https://hg.mozilla.org/mozilla-central"},
+            {"base_repository": "https://hg.mozilla.org/mozilla-central"},
             ["assume:repo:hg.mozilla.org/mozilla-central:action:generic"],
             does_not_raise(),
         ),
         (
             "retrigger",
-            {"head_repository": "https://github.com/taskcluster/taskgraph"},
+            {"base_repository": "https://github.com/taskcluster/taskgraph"},
             ["assume:repo:github.com/taskcluster/taskgraph:action:generic"],
             does_not_raise(),
         ),
         (
             "retrigger",
-            {"head_repository": "git@github.com:mozilla-mobile/firefox-android.git"},
+            {"base_repository": "git@github.com:mozilla-mobile/firefox-android.git"},
             ["assume:repo:github.com/mozilla-mobile/firefox-android:action:generic"],
             does_not_raise(),
         ),
+        (
+            "retrigger",
+            {"base_repository": "git@github.com:mozilla-mobile/firefox-android.git"},
+            ["assume:repo:github.com/mozilla-mobile/firefox-android:pr-action:generic"],
+            does_not_raise(),
+        ),
     ),
 )
 def test_sanity_check_task_scope(