scriptworker 63.5.0__tar.gz → 63.6.0__tar.gz

{scriptworker-63.5.0 → scriptworker-63.6.0}/HISTORY.rst

@@ -4,6 +4,16 @@ Change Log
 All notable changes to this project will be documented in this file.
 This project adheres to `Semantic Versioning <http://semver.org/>`__.
 
+63.6.0 - 2026-10-06
+-------------------
+
+Added
+~~~~~
+
+- Accept `github-push` decision tasks for the Firefox CoT product
+- Add `comm-autoland` as a trusted project
+- Obtain a repository scoped Github token from the Taskcluster Auth service
+
 63.5.0 - 2026-09-23
 -------------------
 

{scriptworker-63.5.0 → scriptworker-63.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: scriptworker
-Version: 63.5.0
+Version: 63.6.0
 Summary: Taskcluster scriptworker
 Author-email: Mozilla Release Engineering <releng@mozilla.com>
 License-File: LICENSE
@@ -21,7 +21,7 @@ Requires-Dist: json-e>=2.5.0
 Requires-Dist: jsonschema[format-nongpl]
 Requires-Dist: pyyaml
 Requires-Dist: taskcluster-taskgraph
-Requires-Dist: taskcluster>=40
+Requires-Dist: taskcluster>=106
 Description-Content-Type: text/x-rst
 
 ===================

{scriptworker-63.5.0 → scriptworker-63.6.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scriptworker"
-version = "63.5.0"
+version = "63.6.0"
 description = "Taskcluster scriptworker"
 authors = [
     { name = "Mozilla Release Engineering", email = "releng@mozilla.com" },
@@ -25,7 +25,7 @@ dependencies = [
     "jsonschema[format-nongpl]",
     "json-e>=2.5.0",
     "PyYAML",
-    "taskcluster>=40",
+    "taskcluster>=106",
     "taskcluster-taskgraph",
 ]
 

{scriptworker-63.5.0 → scriptworker-63.6.0}/scriptworker.yaml.tmpl

@@ -45,9 +45,15 @@ verify_cot_signature: false
 # Chain of Trust job type, e.g. signing
 cot_job_type: scriptworker
 cot_product: firefox
-# Calls to Github API are limited to 60 an hour. Using an API token allows to raise the limit to
-# 5000 per hour. https://developer.github.com/v3/#rate-limiting
-github_oauth_token: somegithubtoken
+
+# Scriptworker first tries to obtain a repository scoped token from Taskcluster's auth service,
+# using the Github App registered under this name.
+# github_app_name: read
+#
+# `github_oauth_token` is used as a fallback if that fails (e.g. missing scopes or app not
+# configured). Without either, calls to the Github API are unauthenticated and limited to 60 an
+# hour. See https://developer.github.com/v3/#rate-limiting
+# github_oauth_token: somegithubtoken
 
 
 #-----------------------------------------------------------------------------------------------

{scriptworker-63.5.0 → scriptworker-63.6.0}/src/scriptworker/constants.py

@@ -79,7 +79,12 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
         "max_chain_length": 20,
         # Calls to Github API are limited to 60 an hour. Using an API token allows to raise the limit to
         # 5000 per hour. https://developer.github.com/v3/#rate-limiting
+        # Scriptworker first tries to obtain a repository scoped token from Taskcluster's auth
+        # service, falling back to this token if that fails.
         "github_oauth_token": "",
+        # The name of the Github App registered with Taskcluster's auth service, used to obtain
+        # a repository scoped token via `auth.githubRepoToken`.
+        "github_app_name": "read",
         # ed25519 settings
         "ed25519_private_key_path": "...",
         "ed25519_public_keys": immutabledict(
@@ -142,7 +147,8 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
                         "adhoc": "github",
                         "app-services": "github",
                         "enterprise": "github",
-                        "firefox": "hg",
+                        # Firefox is migrating from hg.mozilla.org to Github (bug 2070846)
+                        "firefox": ("hg", "github"),
                         "glean": "github",
                         "mobile": "github",
                         "mozillavpn": "github",
@@ -408,6 +414,7 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
                                     "netlocs": ("hg.mozilla.org",),
                                     "path_regexes": (
                                         r"^(?P<path>/comm-central)(/|$)",
+                                        r"^(?P<path>/integration/comm-autoland)(/|$)",
                                         r"^(?P<path>/releases/comm-(beta|release|esr\d+))(/|$)",
                                     ),
                                 }
@@ -492,7 +499,15 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
                             "github-release",
                             "pr-action",
                         ),
-                        "firefox": ("hg-push", "cron", "action"),
+                        "firefox": (
+                            "action",
+                            "cron",
+                            "github-pull-request",
+                            "github-pull-request-untrusted",
+                            "github-push",
+                            "hg-push",
+                            "pr-action",
+                        ),
                         "glean": (
                             "action",
                             "cron",
@@ -826,6 +841,7 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
                                     "/releases/comm-esr153",
                                 ),
                                 "nightly": ("/comm-central",),
+                                "autoland": ("/integration/comm-autoland",),
                             }
                         ),
                         "translations": immutabledict(

{scriptworker-63.5.0 → scriptworker-63.6.0}/src/scriptworker/cot/verify.py

@@ -20,7 +20,6 @@ import tempfile
 from copy import deepcopy
 from urllib.parse import urlparse
 
-import aiohttp
 import dictdiffer
 import jsone
 from immutabledict import immutabledict
@@ -45,6 +44,7 @@ from scriptworker.github import (
     extract_github_repo_full_name,
     extract_github_repo_owner_and_name,
     extract_github_repo_ssh_url,
+    is_github_url,
 )
 from scriptworker.log import contextual_log_handler, get_chain_of_trust_log_filename
 from scriptworker.task import (
@@ -67,6 +67,7 @@ from scriptworker.task import (
     get_triggered_by,
     get_worker_pool_id,
     is_action,
+    is_github_task,
     is_try_or_pull_request,
     retry_get_task_definition,
 )
@@ -1126,7 +1127,7 @@ async def _get_additional_github_releases_jsone_context(decision_link):
     repo_owner, repo_name = extract_github_repo_owner_and_name(repo_url)
     tag_name = get_revision(task, source_env_prefix)
 
-    github_repo = GitHubRepository(repo_owner, repo_name, context.config["github_oauth_token"])
+    github_repo = GitHubRepository(context, repo_owner, repo_name)
     release_data = await github_repo.get_release(tag_name)
 
     # The release data expose by the API[1] is not the same as the original event[2]. That's why
@@ -1208,17 +1209,16 @@ async def _get_additional_github_pull_request_jsone_context(decision_link):
     repo_url = repo_url.replace("git@github.com:", "ssh://github.com/", 1)
     repo_owner, repo_name = extract_github_repo_owner_and_name(repo_url)
     pull_request_number = get_pull_request_number(task, source_env_prefix)
-    token = context.config["github_oauth_token"]
 
-    github_repo = GitHubRepository(repo_owner, repo_name, token)
-    repo_definition = github_repo.definition
+    github_repo = GitHubRepository(context, repo_owner, repo_name)
+    repo_definition = await github_repo.get_definition()
 
     # We need to query the repository where the pull request was made to extract
     # pull request data. The pull request could be created on the same repo as
     # the commit, or an upstream repo. We can compare the base and head repo URLs
     # to infer where the pull request lives.
     if repo_definition["fork"] and base_repo_url != repo_url:
-        github_repo = GitHubRepository(owner=repo_definition["parent"]["owner"]["login"], repo_name=repo_definition["parent"]["name"], token=token)
+        github_repo = GitHubRepository(context, repo_definition["parent"]["owner"]["login"], repo_definition["parent"]["name"])
 
     pull_request_data = await github_repo.get_pull_request(pull_request_number)
     # Even though pull_request_data['head']['repo']['pushed_at'] does exist,
@@ -1253,7 +1253,7 @@ async def _get_additional_github_push_jsone_context(decision_link):
     repo_owner, repo_name = extract_github_repo_owner_and_name(repo_url)
     commit_hash = get_revision(task, source_env_prefix)
 
-    github_repo = GitHubRepository(repo_owner, repo_name, context.config["github_oauth_token"])
+    github_repo = GitHubRepository(context, repo_owner, repo_name)
     commit_data = await github_repo.get_commit(commit_hash)
 
     committer = commit_data["committer"] or {}
@@ -1351,7 +1351,18 @@ async def populate_jsone_context(chain, parent_link, decision_link, tasks_for):
         "taskId": None,
     }
 
-    if chain.context.config["cot_product_type"] == "github":
+    cot_product_type = chain.context.config["cot_product_type"]
+    # A tuple becomes a list when the config round-trips through json/yaml.
+    if isinstance(cot_product_type, (tuple, list)):
+        # Products migrating between VCSes support several types (bug 2070846).
+        # tasks_for like `action` and `cron` exist on both hg and github, so
+        # determine the type from the decision task itself.
+        detected_type = "github" if is_github_task(decision_link.task) else "hg"
+        if detected_type not in cot_product_type:
+            raise CoTError('cot_product_type "{}" is not enabled for cot_product "{}"!'.format(detected_type, chain.context.config["cot_product"]))
+        cot_product_type = detected_type
+
+    if cot_product_type == "github":
         if tasks_for == "github-release":
             jsone_context.update(await _get_additional_github_releases_jsone_context(decision_link))
         elif tasks_for == "cron":
@@ -1364,7 +1375,7 @@ async def populate_jsone_context(chain, parent_link, decision_link, tasks_for):
             jsone_context.update(await _get_additional_github_push_jsone_context(decision_link))
         else:
             raise CoTError('Unknown tasks_for "{}" for github cot_product "{}"!'.format(tasks_for, chain.context.config["cot_product"]))
-    elif chain.context.config["cot_product_type"] == "hg":
+    elif cot_product_type == "hg":
         source_url = get_source_url(decision_link)
         project = await get_project(chain.context, source_url)
         jsone_context["repository"] = {
@@ -1384,9 +1395,7 @@ async def populate_jsone_context(chain, parent_link, decision_link, tasks_for):
         else:
             raise CoTError('Unknown tasks_for "{}" for hg cot_product "{}"!'.format(tasks_for, chain.context.config["cot_product"]))
     else:
-        raise CoTError(
-            'Unknown cot_product_type "{}" for cot_product "{}"!'.format(chain.context.config["cot_product_type"], chain.context.config["cot_product"])
-        )
+        raise CoTError('Unknown cot_product_type "{}" for cot_product "{}"!'.format(cot_product_type, chain.context.config["cot_product"]))
 
     log.debug("{} json-e context:".format(parent_link.name))
     # format_json() breaks on lambda values; use pprint.pformat here.
@@ -1407,16 +1416,11 @@ def build_taskcluster_yml_url(link):
     """
     source_env_prefix = link.context.config["source_env_prefix"]
     repo_url = get_repo(link.task, source_env_prefix)
-    repo_url = repo_url.replace("git@github.com:", "ssh://github.com/", 1)
     revision = get_revision(link.task, source_env_prefix)
     repo_parts = urlparse(repo_url)
-    if repo_parts.netloc == "github.com":
-        user, repo_name = extract_github_repo_owner_and_name(repo_url)
-        url = f"https://raw.githubusercontent.com/{user}/{repo_name}/{revision}/.taskcluster.yml"
-    elif repo_parts.netloc == "hg.mozilla.org":
-        url = f"{repo_parts.scheme}://{repo_parts.netloc}{repo_parts.path}/raw-file/{revision}/.taskcluster.yml"
-    else:
+    if repo_parts.netloc != "hg.mozilla.org":
         raise CoTError("Unsupported VCS server!")
+    url = f"{repo_parts.scheme}://{repo_parts.netloc}{repo_parts.path}/raw-file/{revision}/.taskcluster.yml"
     log.debug(f"{link.name} .taskcluster.yml is at {url}")
     return url
 
@@ -1438,19 +1442,20 @@ async def get_in_tree_template(link):
 
     """
     context = link.context
-    source_url = build_taskcluster_yml_url(link)
-    repo_url = get_repo(link.task, link.context.config["source_env_prefix"])
+    source_env_prefix = context.config["source_env_prefix"]
+    repo_url = get_repo(link.task, source_env_prefix)
+    repo_url = repo_url.replace("git@github.com:", "ssh://github.com/", 1)
 
-    auth = None
-    if (
-        (repo_url.startswith(("ssh://", "git@github.com")) or any(vcs_rule.get("require_secret") for vcs_rule in context.config["trusted_vcs_rules"]))
-        and "github.com" in repo_url
-        and context.config.get("github_oauth_token")
-    ):
-        auth = aiohttp.BasicAuth(context.config["github_oauth_token"])
+    if is_github_url(repo_url):
+        revision = get_revision(link.task, source_env_prefix)
+        repo_owner, repo_name = extract_github_repo_owner_and_name(repo_url)
+        github_repo = GitHubRepository(context, repo_owner, repo_name)
+        content = await github_repo.get_file_contents(".taskcluster.yml", ref=revision)
+        return load_json_or_yaml(content, file_type="yaml")
+
+    source_url = build_taskcluster_yml_url(link)
     url_hash = hashlib.sha1(source_url.encode("ascii")).hexdigest()
-    tmpl = await load_json_or_yaml_from_url(context, source_url, os.path.join(context.config["work_dir"], "{}_taskcluster.yml".format(url_hash)), auth=auth)
-    return tmpl
+    return await load_json_or_yaml_from_url(context, source_url, os.path.join(context.config["work_dir"], "{}_taskcluster.yml".format(url_hash)))
 
 
 def _get_action_from_actions_json(all_actions, callback_name):

{scriptworker-63.5.0 → scriptworker-63.6.0}/src/scriptworker/github.py

@@ -1,12 +1,16 @@
 """GitHub helper functions."""
 
 import asyncio
+import hashlib
 import logging
 import re
+from pathlib import Path
 from urllib.parse import unquote, urlparse
 
 from github3 import GitHub
 from github3.exceptions import GitHubException
+from taskcluster.aio import Auth
+from taskcluster.exceptions import TaskclusterFailure
 
 from scriptworker.exceptions import ConfigError
 from scriptworker.utils import get_parts_of_url_path, get_single_item_from_sequence, retry_async_decorator, retry_request, retry_sync
@@ -24,30 +28,110 @@ log = logging.getLogger(__name__)
 class GitHubRepository:
     """Wrapper around GitHub API. Used to access public data."""
 
-    def __init__(self, owner, repo_name, token=""):
-        """Build the GitHub API URL which points to the definition of the repository.
+    GITHUB_PERMISSIONS = {"contents": "read", "metadata": "read", "pull_requests": "read"}
+
+    def __init__(self, context, owner, repo_name):
+        """Store the repository coordinates. The github3 repository object is built lazily.
 
         Args:
-            owner (str): the owner's GitHub username
+            context (scriptworker.context.Context): the scriptworker context
+            owner (str): the owner of the repository
             repo_name (str): the name of the repository
-            token (str): the GitHub API token
+
+        """
+        self._context = context
+        self._owner = owner
+        self._repo_name = repo_name
+        self._repository_cache = None
+        self._repository_lock = asyncio.Lock()
+
+    async def _get_repository(self):
+        """Build and cache the github3 repository object.
 
         Returns:
-            dict: a representation of the repo definition
+            github3.repos.repo.Repository: the github3 repository object
 
         """
-        github = retry_sync(GitHub, kwargs={"token": token}, sleeptime_kwargs=_GITHUB_LIBRARY_SLEEP_TIME_KWARGS)
-        self._github_repository = retry_sync(github.repository, args=(owner, repo_name), sleeptime_kwargs=_GITHUB_LIBRARY_SLEEP_TIME_KWARGS)
+        async with self._repository_lock:
+            if self._repository_cache is None:
+                token = await self._get_token(self._context, self._owner, self._repo_name)
+                github = retry_sync(GitHub, kwargs={"token": token}, sleeptime_kwargs=_GITHUB_LIBRARY_SLEEP_TIME_KWARGS)
+                self._repository_cache = retry_sync(github.repository, args=(self._owner, self._repo_name), sleeptime_kwargs=_GITHUB_LIBRARY_SLEEP_TIME_KWARGS)
+
+        return self._repository_cache
+
+    async def _get_token(self, context, owner, repo_name):
+        """Get a repository scoped GitHub token from Taskcluster's auth service.
+
+        Falls back to ``context.config["github_oauth_token"]`` if the auth service call
+        fails, e.g. because of missing scopes.
+
+        Args:
+            context (scriptworker.context.Context): the scriptworker context
+            owner (str): the owner of the repository
+            repo_name (str): the name of the repository
 
-    @property
-    def definition(self):
+        Returns:
+            str: the scoped GitHub token, or the fallback token
+
+        """
+        if not context.credentials:
+            return context.config.get("github_oauth_token", "")
+
+        try:
+            auth = Auth(options={"rootUrl": context.config["taskcluster_root_url"], "credentials": context.credentials})
+            response = await auth.githubRepoToken(
+                context.config["github_app_name"], owner, payload={"repositories": [repo_name], "permissions": self.GITHUB_PERMISSIONS}
+            )
+            return response["token"]
+        except TaskclusterFailure as e:
+            # TODO When opening a PR from a fork, we're guaranteed to hit this
+            # fallback as the task won't have auth service scopes for the repo
+            # fork. We'll need to improve this before we can stop depending on
+            # `github_oauth_token`.
+            log.warning(f"Could not obtain Github token from Taskcluster for {owner}/{repo_name}, falling back to `github_oauth_token`: {e}")
+            return context.config.get("github_oauth_token", "")
+
+    async def get_definition(self):
         """Fetch the definition of the repository, exposed by the GitHub API.
 
         Returns:
             dict: a representation of the repo definition
 
         """
-        return self._github_repository.as_dict()
+        repository = await self._get_repository()
+        return repository.as_dict()
+
+    @retry_async_decorator(retry_exceptions=GitHubException)
+    async def get_file_contents(self, path, ref=None):
+        """Fetch the decoded contents of a file in the repository.
+
+        Args:
+            path (str): the path to the file, relative to the repository root
+            ref (str, optional): the commit/branch/tag to read the file from.
+                Defaults to the repository's default branch. Results for full
+                commit hashes are cached on disk in `work_dir`.
+
+        Returns:
+            str: the decoded contents of the file
+
+        """
+        # Files at a full commit hash are immutable, so they're safe to cache on disk.
+        cache_path = None
+        if ref and _is_git_full_hash(ref):
+            cache_key = hashlib.sha1(f"{self._owner}/{self._repo_name}@{ref}:{path}".encode("utf-8")).hexdigest()
+            cache_path = Path(self._context.config["work_dir"]) / ".cache" / "github_files" / cache_key
+            if cache_path.exists():
+                return cache_path.read_text(encoding="utf-8")
+
+        repository = await self._get_repository()
+        contents = repository.file_contents(path, ref=ref)
+        decoded = contents.decoded.decode("utf-8")
+
+        if cache_path:
+            cache_path.parent.mkdir(parents=True, exist_ok=True)
+            cache_path.write_text(decoded, encoding="utf-8")
+        return decoded
 
     @retry_async_decorator(retry_exceptions=GitHubException)
     async def get_commit(self, commit_hash):
@@ -60,7 +144,8 @@ class GitHubRepository:
             dict: a representation of the commit
 
         """
-        return self._github_repository.commit(commit_hash).as_dict()
+        repository = await self._get_repository()
+        return repository.commit(commit_hash).as_dict()
 
     @retry_async_decorator(retry_exceptions=GitHubException)
     async def get_pull_request(self, pull_request_number):
@@ -73,7 +158,8 @@ class GitHubRepository:
             dict: a representation of the pull request
 
         """
-        return self._github_repository.pull_request(pull_request_number).as_dict()
+        repository = await self._get_repository()
+        return repository.pull_request(pull_request_number).as_dict()
 
     @retry_async_decorator(retry_exceptions=GitHubException)
     async def get_release(self, tag_name):
@@ -86,7 +172,8 @@ class GitHubRepository:
             dict: a representation of the tag
 
         """
-        return self._github_repository.release_from_tag(tag_name).as_dict()
+        repository = await self._get_repository()
+        return repository.release_from_tag(tag_name).as_dict()
 
     @retry_async_decorator(retry_exceptions=GitHubException)
     async def get_tag_hash(self, tag_name):
@@ -99,8 +186,9 @@ class GitHubRepository:
             str: the commit hash linked by the tag
 
         """
+        repository = await self._get_repository()
         tag_object = get_single_item_from_sequence(
-            sequence=self._github_repository.tags(),
+            sequence=repository.tags(),
             condition=lambda tag: tag.name == tag_name,
             no_item_error_message='No tag "{}" exist'.format(tag_name),
             too_many_item_error_message='Too many tags "{}" found'.format(tag_name),
@@ -129,7 +217,8 @@ class GitHubRepository:
         if not _is_git_full_hash(revision):
             revision = await self.get_tag_hash(tag_name=revision)
 
-        html_text = await _fetch_github_branch_commits_data(context, self._github_repository.html_url, revision)
+        repository = await self._get_repository()
+        html_text = await _fetch_github_branch_commits_data(context, repository.html_url, revision)
 
         # https://github.com/{repo_owner}/{repo_name}/branch_commits/{revision} just returns some \n
         # when the commit hasn't landed on the origin repo. Otherwise, some HTML data is returned - it

{scriptworker-63.5.0 → scriptworker-63.6.0}/src/scriptworker/task.py

@@ -550,7 +550,7 @@ async def is_pull_request(context, task):
         if not revision and can_skip:
             continue
 
-        github_repository = GitHubRepository(repo_owner, repo_name, context.config["github_oauth_token"])
+        github_repository = GitHubRepository(context, repo_owner, repo_name)
         conditions.append(not await github_repository.has_commit_landed_on_repository(context, revision))
 
     return any(conditions)