scriptworker 63.6.1__tar.gz → 64.0.0__tar.gz

{scriptworker-63.6.1 → scriptworker-64.0.0}/HISTORY.rst

@@ -4,6 +4,35 @@ Change Log
 All notable changes to this project will be documented in this file.
 This project adheres to `Semantic Versioning <http://semver.org/>`__.
 
+64.0.0 - 2026-11-05
+-------------------
+
+Fixed
+~~~~~
+
+- use the compare API to tell if a commit landed on a github repo (bug 2089531, #855)
+
+Removed
+~~~~~~~
+
+- {mozilla,comm}-esr140 support (bug 2081292)
+
+63.7.1 - 2026-10-27
+-------------------
+
+Fixed
+~~~~~
+
+- CoT verification crash due to long github CSP response header (bug 2085468)
+
+63.7.0 - 2026-10-13
+-------------------
+
+Added
+~~~~~
+
+- take a git project's scm level from the task branch (#839)
+
 63.6.1 - 2026-10-08
 -------------------
 

{scriptworker-63.6.1 → scriptworker-64.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: scriptworker
-Version: 63.6.1
+Version: 64.0.0
 Summary: Taskcluster scriptworker
 Author-email: Mozilla Release Engineering <releng@mozilla.com>
 License-File: LICENSE

{scriptworker-63.6.1 → scriptworker-64.0.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scriptworker"
-version = "63.6.1"
+version = "64.0.0"
 description = "Taskcluster scriptworker"
 authors = [
     { name = "Mozilla Release Engineering", email = "releng@mozilla.com" },

{scriptworker-63.6.1 → scriptworker-64.0.0}/src/scriptworker/constants.py

@@ -464,7 +464,6 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
                                 "/releases/mozilla-beta": "/mozilla-firefox/firefox",
                                 "/releases/mozilla-release": "/mozilla-firefox/firefox",
                                 "/releases/mozilla-esr115": "/mozilla-firefox/firefox",
-                                "/releases/mozilla-esr140": "/mozilla-firefox/firefox",
                                 "/releases/mozilla-esr153": "/mozilla-firefox/firefox",
                             }
                         ),
@@ -634,7 +633,6 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
                                 "project:releng:lando:repo:beta": "beta",
                                 "project:releng:lando:repo:release": "release",
                                 "project:releng:lando:repo:esr115": "esr115",
-                                "project:releng:lando:repo:esr140": "esr140",
                                 "project:releng:lando:repo:esr153": "esr153",
                             }
                         ),
@@ -730,7 +728,6 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
                                     "/releases/mozilla-beta",
                                     "/releases/mozilla-release",
                                     "/releases/mozilla-esr115",
-                                    "/releases/mozilla-esr140",
                                     "/releases/mozilla-esr153",
                                     "/projects/birch",
                                     # Bug 1821839 hneiva using to test hardened signing
@@ -740,7 +737,6 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
                                     "/releases/mozilla-beta",
                                     "/releases/mozilla-release",
                                     "/releases/mozilla-esr115",
-                                    "/releases/mozilla-esr140",
                                     "/releases/mozilla-esr153",
                                     "/projects/birch",
                                     "/projects/maple",
@@ -754,11 +750,9 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
                                     "/releases/mozilla-release",
                                 ),
                                 "esr115": ("/releases/mozilla-esr115",),
-                                "esr140": ("/releases/mozilla-esr140",),
                                 "esr153": ("/releases/mozilla-esr153",),
                                 "esr": (
                                     "/releases/mozilla-esr115",
-                                    "/releases/mozilla-esr140",
                                     "/releases/mozilla-esr153",
                                 ),
                                 "nightly": ("/mozilla-central",),
@@ -770,7 +764,6 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
                                     "/releases/mozilla-beta",
                                     "/releases/mozilla-release",
                                     "/releases/mozilla-esr115",
-                                    "/releases/mozilla-esr140",
                                     "/releases/mozilla-esr153",
                                     # bug 1845368: a permanent project branch for testing updates
                                     "/projects/pine",
@@ -783,7 +776,6 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
                                     "/releases/mozilla-beta",
                                     "/releases/mozilla-release",
                                     "/releases/mozilla-esr115",
-                                    "/releases/mozilla-esr140",
                                     "/releases/mozilla-esr153",
                                 ),
                                 "all-production-branches-and-autoland": (
@@ -791,7 +783,6 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
                                     "/releases/mozilla-beta",
                                     "/releases/mozilla-release",
                                     "/releases/mozilla-esr115",
-                                    "/releases/mozilla-esr140",
                                     "/releases/mozilla-esr153",
                                     "/integration/autoland",
                                 ),
@@ -820,7 +811,6 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
                                     "/releases/comm-beta",
                                     "/releases/comm-release",
                                     "/releases/comm-esr115",
-                                    "/releases/comm-esr140",
                                     "/releases/comm-esr153",
                                 ),
                                 "beta": ("/releases/comm-beta",),
@@ -828,16 +818,14 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
                                 "release-or-esr": (
                                     "/releases/comm-release",
                                     "/releases/comm-esr115",
-                                    "/releases/comm-esr140",
                                     "/releases/comm-esr153",
                                 ),
-                                "esr": ("/releases/comm-esr115", "/releases/comm-esr140", "/releases/comm-esr153"),
+                                "esr": ("/releases/comm-esr115", "/releases/comm-esr153"),
                                 "all-nightly-branches": (
                                     "/comm-central",
                                     "/releases/comm-beta",
                                     "/releases/comm-release",
                                     "/releases/comm-esr115",
-                                    "/releases/comm-esr140",
                                     "/releases/comm-esr153",
                                 ),
                                 "nightly": ("/comm-central",),

{scriptworker-63.6.1 → scriptworker-64.0.0}/src/scriptworker/cot/verify.py

@@ -151,12 +151,34 @@ class ChainOfTrust(object):
             bool: True if a task is a try task.
 
         """
-        tasks = [asyncio.ensure_future(link.is_try_or_pull_request()) for link in self.links]
-        tasks.insert(0, asyncio.ensure_future(is_try_or_pull_request(self.context, self.task)))
+        tasks = [asyncio.ensure_future(link.is_try_or_pull_request(fallback_ref=self._get_decision_ref(link.decision_task_id))) for link in self.links]
+        tasks.insert(0, asyncio.ensure_future(is_try_or_pull_request(self.context, self.task, fallback_ref=self._get_decision_ref(self.decision_task_id))))
 
         conditions = await raise_future_exceptions(tasks)
         return any(conditions)
 
+    def _get_decision_ref(self, decision_task_id):
+        """Get the ref (branch or tag) of a decision task in the chain.
+
+        Some tasks, like docker-image or scriptworker ones, don't define their
+        own ref. Their decision task does, and it's the push the task comes from.
+
+        Args:
+            decision_task_id (str): the task id of the decision task.
+
+        Returns:
+            str: the ref, or None if the decision task isn't in the chain or doesn't define one.
+
+        """
+        if decision_task_id == self.task_id:
+            task = self.task
+        else:
+            try:
+                task = self.get_link(decision_task_id).task
+            except CoTError:
+                return None
+        return get_branch(task, self.context.config["source_env_prefix"])
+
     def get_link(self, task_id):
         """Get a ``LinkOfTrust`` by task id.
 
@@ -293,9 +315,18 @@ class LinkOfTrust(object):
         self.parent_task_id = get_parent_task_id(self.task)
         self.worker_impl = guess_worker_impl(self)
 
-    async def is_try_or_pull_request(self):
-        """bool: the task is either a try or a pull request one."""
-        return await is_try_or_pull_request(self.context, self.task)
+    async def is_try_or_pull_request(self, fallback_ref=None):
+        """Determine if the task is either a try or a pull request one.
+
+        Args:
+            fallback_ref (str, optional): the ref to check commits against if the
+                task doesn't define one. See ``scriptworker.task.is_pull_request``.
+
+        Returns:
+            bool: True if the task is either a try or a pull request one.
+
+        """
+        return await is_try_or_pull_request(self.context, self.task, fallback_ref=fallback_ref)
 
     @property
     def cot(self):
@@ -1031,7 +1062,16 @@ async def get_pushlog_info(decision_link):
 
 
 # get_scm_level {{{1
-async def get_scm_level(context, project):
+def _branch_matches(pattern, branch):
+    """Whether ``branch`` matches a ``projects.yml`` branch name, which may end in ``*``."""
+    if pattern == "*":
+        return True
+    if pattern.endswith("*"):
+        return branch.startswith(pattern[:-1])
+    return branch == pattern
+
+
+async def get_scm_level(context, project, branch=None):
     """Get the scm level for a project from ``projects.yml``.
 
     We define all known projects in ``projects.yml``. Let's make sure we have
@@ -1043,6 +1083,8 @@ async def get_scm_level(context, project):
     Args:
         context (scriptworker.context.Context): the scriptworker context
         project (str): the project to get the scm level for.
+        branch (str, optional): the branch the task runs on. Required for git
+            projects, where each branch has its own level. Ignored for hg.
 
     Returns:
         str: the level of the project, as a string.
@@ -1053,14 +1095,16 @@ async def get_scm_level(context, project):
     if config["repo_type"] == "hg":
         return config["access"].replace("scm_level_", "")
     elif config["repo_type"] == "git":
-        # TODO: we should be using the branch that the task is actually
-        # being run on
-        default_branch = config.get("default_branch", "main")
-        for branch in config["branches"]:
-            if branch["name"] == default_branch:
-                return str(branch["level"])
+        if branch is None:
+            raise ValueError("{} is a git project, so a branch is required to find its level".format(project))
+        if branch.startswith("refs/heads/"):
+            branch = branch[len("refs/heads/") :]
+        # the first matching entry wins, like fxci-config's Project.get_branch()
+        for branch_config in config["branches"]:
+            if _branch_matches(branch_config["name"], branch):
+                return str(branch_config["level"])
 
-    raise ValueError("Can't find level for project {}".format(project))
+    raise ValueError("Can't find level for project {} on branch {}".format(project, branch))
 
 
 async def _get_additional_hg_action_jsone_context(parent_link, decision_link):
@@ -1190,7 +1234,7 @@ async def _get_additional_git_cron_jsone_context(decision_link):
             "sender": {"login": user},
         },
         # Taskgraph cron contexts mirror hg-push contexts
-        "repository": {"url": repo, "project": repo_name, "level": await get_scm_level(decision_link.context, repo_name), "type": "git"},
+        "repository": {"url": repo, "project": repo_name, "level": await get_scm_level(decision_link.context, repo_name, branch), "type": "git"},
         "push": {"revision": revision, "branch": branch},
     }
 

{scriptworker-63.6.1 → scriptworker-64.0.0}/src/scriptworker/github.py

@@ -8,14 +8,15 @@ from pathlib import Path
 from urllib.parse import unquote, urlparse
 
 from github3 import GitHub
-from github3.exceptions import GitHubException
+from github3.exceptions import GitHubException, NotFoundError
 from taskcluster.aio import Auth
 from taskcluster.exceptions import TaskclusterFailure
 
 from scriptworker.exceptions import ConfigError
-from scriptworker.utils import get_parts_of_url_path, get_single_item_from_sequence, retry_async_decorator, retry_request, retry_sync
+from scriptworker.utils import get_parts_of_url_path, get_single_item_from_sequence, retry_async_decorator, retry_sync
 
 _GIT_FULL_HASH_PATTERN = re.compile(r"^[0-9a-f]{40}$")
+_GIT_HASH_LIKE_PATTERN = re.compile(r"^[0-9a-f]{7,40}$")
 # The github3 library already retries requests. It gives a round of waiting of usually 15 seconds.
 # A delay factor of 7.5s means the second round of waiting will occur ~15s after the first one,
 # the third one ~30s and so on.
@@ -196,64 +197,92 @@ class GitHubRepository:
 
         return tag_object.commit.sha
 
-    async def has_commit_landed_on_repository(self, context, revision):
-        """Tell if a commit was landed on the repository or if it just comes from a pull request.
+    async def has_commit_landed_on_repository(self, context, revision, ref):
+        """Tell if a commit was landed on the repository or if it just comes from a fork or a pull request.
+
+        GitHub serves commits from forks through the upstream repository, so the
+        commit merely existing isn't enough: it must be reachable from a ref of
+        this repository.
 
         Args:
             context (scriptworker.context.Context): the scriptworker context.
             revision (str): the commit hash or the tag name.
+            ref (str): the branch or tag of this repository that ``revision`` is
+                expected to be on, e.g. ``refs/heads/main``. Ignored if ``revision``
+                is a tag name.
 
         Returns:
-            bool: True if the commit is present in one of the branches of the main repository
+            bool: True if the commit is reachable from ``ref`` in this repository
 
         """
         if any(vcs_rule.get("require_secret") for vcs_rule in context.config["trusted_vcs_rules"]):
-            # This check uses unofficial API on github, which we can't easily
-            # check for private repos, assume its true in the private case.
+            # XXX This check isn't implemented for private repositories, assume it's true.
             log.info("has_commit_landed_on_repository() not implemented for private repositories, assume True")
             return True
 
-        # Revision may be a tag name. `branch_commits` doesn't work on tags
         if not _is_git_full_hash(revision):
-            revision = await self.get_tag_hash(tag_name=revision)
+            # The tag is looked up in this repository, so the commit it points to
+            # is reachable from one of its refs. This raises if there's no such tag.
+            await self.get_tag_hash(tag_name=revision)
+            return True
 
-        repository = await self._get_repository()
-        html_text = await _fetch_github_branch_commits_data(context, repository.html_url, revision)
+        qualified_ref = _qualify_ref(ref)
+        if qualified_ref is None:
+            log.warning(f"Cannot tell if {revision} landed on {self._owner}/{self._repo_name}: no usable ref (got {ref!r})")
+            return False
+
+        status = await _fetch_github_compare_status(self, qualified_ref, revision)
+        # "behind" and "identical" mean that `revision` is an ancestor of, or equal to, `ref`.
+        # "ahead" and "diverged" mean that `revision` isn't reachable from `ref`.
+        landed = status in ("behind", "identical")
+        if not landed:
+            log.warning(f"{revision} is not on {ref} of {self._owner}/{self._repo_name} (comparison status: {status})")
+        return landed
+
+    @retry_async_decorator(retry_exceptions=GitHubException)
+    async def get_compare_status(self, base, head):
+        """Compare two commitishes of the repository.
 
-        # https://github.com/{repo_owner}/{repo_name}/branch_commits/{revision} just returns some \n
-        # when the commit hasn't landed on the origin repo. Otherwise, some HTML data is returned - it
-        # represents the branches on which the given revision is present.
-        return html_text != ""
+        Args:
+            base (str): the base ref or commit
+            head (str): the head ref or commit
+
+        Returns:
+            str: the status of ``head`` relative to ``base``: one of "ahead", "behind",
+                "diverged" or "identical", or None if either side doesn't exist
 
+        """
+        repository = await self._get_repository()
+        try:
+            return repository.compare_commits(base, head).status
+        except NotFoundError:
+            return None
 
-_BRANCH_COMMITS_CACHE_TTL_IN_SECONDS = 10 * 60  # 10 minutes
-_BRANCH_COMMITS_CACHE = {}
 
+_COMPARE_CACHE_TTL_IN_SECONDS = 10 * 60  # 10 minutes
+_COMPARE_CACHE = {}
 
-async def _fetch_github_branch_commits_data(context, repo_html_url, revision):
-    # Include context identity because different contexts carry
-    # different HTTP sessions that may be closed independently.
-    cache_key = (id(context), repo_html_url, revision)
 
-    if cache_key in _BRANCH_COMMITS_CACHE:
-        return await _BRANCH_COMMITS_CACHE[cache_key]
+async def _fetch_github_compare_status(github_repository, base, head):
+    cache_key = (github_repository._owner, github_repository._repo_name, base, head)
+
+    if cache_key in _COMPARE_CACHE:
+        return await _COMPARE_CACHE[cache_key]
 
     future = asyncio.get_running_loop().create_future()
-    _BRANCH_COMMITS_CACHE[cache_key] = future
+    _COMPARE_CACHE[cache_key] = future
 
     try:
-        url = "/".join((repo_html_url.rstrip("/"), "branch_commits", revision))
-        html_text = await retry_request(context, url)
-        result = html_text.strip()
+        result = await github_repository.get_compare_status(base, head)
         future.set_result(result)
         asyncio.get_running_loop().call_later(
-            _BRANCH_COMMITS_CACHE_TTL_IN_SECONDS,
-            _BRANCH_COMMITS_CACHE.pop,
+            _COMPARE_CACHE_TTL_IN_SECONDS,
+            _COMPARE_CACHE.pop,
             cache_key,
             None,
         )
     except BaseException as e:
-        _BRANCH_COMMITS_CACHE.pop(cache_key, None)
+        _COMPARE_CACHE.pop(cache_key, None)
         future.set_exception(e)
         raise
 
@@ -400,6 +429,37 @@ def _is_git_full_hash(revision):
     return _GIT_FULL_HASH_PATTERN.match(revision) is not None
 
 
+def _qualify_ref(ref):
+    """Turn ``ref`` into a branch or tag ref that can prove a commit is in the repository.
+
+    Only branches and tags count: GitHub also exposes refs like ``refs/pull/123/head``
+    for pull requests, including ones from forks. GitHub resolves short names
+    loosely (``pull/123/head`` matches that pull request ref), so bare names are
+    qualified as branches, e.g. ``main`` becomes ``refs/heads/main``.
+
+    A commit is trivially reachable from itself, so (possibly abbreviated) hashes
+    prove nothing. ``owner:ref`` would make GitHub look up the ref in another
+    repository of the network, like a fork.
+
+    Args:
+        ref (str): the ref, e.g. ``refs/heads/main``, ``refs/tags/v1.0.0`` or ``main``
+
+    Returns:
+        str: the fully qualified ref, or None if ``ref`` can't be used.
+
+    """
+    if not ref or not isinstance(ref, str) or ":" in ref or ".." in ref:
+        return None
+    if ref.startswith("refs/"):
+        prefix, _, name = ref.partition("/")[2].partition("/")
+        if prefix in ("heads", "tags") and name:
+            return ref
+        return None
+    if _GIT_HASH_LIKE_PATTERN.match(ref):
+        return None
+    return f"refs/heads/{ref}"
+
+
 def _check_github_url_is_supported(url):
     if not is_github_url(url):
         raise ValueError('"{}" is not a supported GitHub URL!'.format(url))

{scriptworker-63.6.1 → scriptworker-64.0.0}/src/scriptworker/task.py

@@ -505,7 +505,7 @@ def is_try(task, source_env_prefix):
     )
 
 
-async def is_pull_request(context, task):
+async def is_pull_request(context, task, fallback_ref=None):
     """Determine if a task is a pull-request-like task (restricted privs).
 
     This goes further than checking ``tasks_for``. We may or may not want
@@ -516,11 +516,15 @@ async def is_pull_request(context, task):
         * ``task.extra.env.tasks_for`` is "github-pull-request" or "github-pull-request-untrusted"
         * ``task.payload.env.MOBILE_HEAD_REPOSITORY`` doesn't come from an official repo
         * ``task.metadata.source`` doesn't come from an official repo, either
-        * The last 2 items are landed on the official repo
+        * The last 2 items are landed on the official repo, i.e. reachable from the
+          task's ref (``*_HEAD_REF``/``*_HEAD_BRANCH``), or from ``fallback_ref``
+          when the task doesn't define one
 
     Args:
         context (scriptworker.context.Context): the scriptworker context.
         task (dict): the task definition to check.
+        fallback_ref (str, optional): the ref to check commits against when the task
+            doesn't define one, typically the one of the task's decision task.
 
     Returns:
         bool: True if it's a pull-request. False if it either comes from the official repos or if
@@ -531,6 +535,7 @@ async def is_pull_request(context, task):
     tasks_for = task.get("extra", {}).get("tasks_for")
     repo_url_from_payload = get_repo(task, context.config["source_env_prefix"])
     revision_from_payload = get_revision(task, context.config["source_env_prefix"])
+    ref = get_branch(task, context.config["source_env_prefix"]) or fallback_ref
 
     metadata_source_url = task["metadata"].get("source", "")
     repo_from_source_url, revision_from_source_url = extract_github_repo_and_revision_from_source_url(metadata_source_url)
@@ -551,12 +556,12 @@ async def is_pull_request(context, task):
             continue
 
         github_repository = GitHubRepository(context, repo_owner, repo_name)
-        conditions.append(not await github_repository.has_commit_landed_on_repository(context, revision))
+        conditions.append(not await github_repository.has_commit_landed_on_repository(context, revision, ref))
 
     return any(conditions)
 
 
-async def is_try_or_pull_request(context, task):
+async def is_try_or_pull_request(context, task, fallback_ref=None):
     """Determine if a task is a try or a pull-request-like task (restricted privs).
 
     Checks are the ones done in ``is_try`` and ``is_pull_request``
@@ -564,13 +569,14 @@ async def is_try_or_pull_request(context, task):
     Args:
         context (scriptworker.context.Context): the scriptworker context.
         task (dict): the task definition to check.
+        fallback_ref (str, optional): passed to ``is_pull_request``.
 
     Returns:
         bool: True if it's a pull-request or a try task
 
     """
     if is_github_task(task):
-        return await is_pull_request(context, task)
+        return await is_pull_request(context, task, fallback_ref=fallback_ref)
     else:
         return is_try(task, context.config["source_env_prefix"])
 

{scriptworker-63.6.1 → scriptworker-64.0.0}/src/scriptworker/utils.py

@@ -44,6 +44,8 @@ log = logging.getLogger(__name__)
 def scriptworker_session(*args, **kwargs):
     """Return an ``aiohttp.ClientSession`` with the scriptworker User-Agent header set."""
     kwargs.setdefault("headers", {}).setdefault("User-Agent", f"scriptworker {scriptworker.__version__}")
+    kwargs.setdefault("max_line_size", 8190 * 2)
+    kwargs.setdefault("max_field_size", 8190 * 2)
     return aiohttp.ClientSession(*args, **kwargs)