scriptworker 63.4.5__tar.gz → 63.6.0__tar.gz

{scriptworker-63.4.5 → scriptworker-63.6.0}/HISTORY.rst

@@ -4,6 +4,31 @@ Change Log
 All notable changes to this project will be documented in this file.
 This project adheres to `Semantic Versioning <http://semver.org/>`__.
 
+63.6.0 - 2026-10-06
+-------------------
+
+Added
+~~~~~
+
+- Accept `github-push` decision tasks for the Firefox CoT product
+- Add `comm-autoland` as a trusted project
+- Obtain a repository scoped Github token from the Taskcluster Auth service
+
+63.5.0 - 2026-09-23
+-------------------
+
+Fixed
+~~~~~
+
+- Add `mozilla-firefox` as a Github owner for Firefox
+- Url parsing bug when Github <owner> has <repo> as a substring
+
+Added
+~~~~~
+
+- Allow tasks sourced from "Git mirrors" to run in graphs alongside tasks
+  sourced from their Mercurial equivalents
+
 63.4.5 - 2026-09-11
 -------------------
 

{scriptworker-63.4.5 → scriptworker-63.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: scriptworker
-Version: 63.4.5
+Version: 63.6.0
 Summary: Taskcluster scriptworker
 Author-email: Mozilla Release Engineering <releng@mozilla.com>
 License-File: LICENSE
@@ -21,7 +21,7 @@ Requires-Dist: json-e>=2.5.0
 Requires-Dist: jsonschema[format-nongpl]
 Requires-Dist: pyyaml
 Requires-Dist: taskcluster-taskgraph
-Requires-Dist: taskcluster>=40
+Requires-Dist: taskcluster>=106
 Description-Content-Type: text/x-rst
 
 ===================

{scriptworker-63.4.5 → scriptworker-63.6.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scriptworker"
-version = "63.4.5"
+version = "63.6.0"
 description = "Taskcluster scriptworker"
 authors = [
     { name = "Mozilla Release Engineering", email = "releng@mozilla.com" },
@@ -25,7 +25,7 @@ dependencies = [
     "jsonschema[format-nongpl]",
     "json-e>=2.5.0",
     "PyYAML",
-    "taskcluster>=40",
+    "taskcluster>=106",
     "taskcluster-taskgraph",
 ]
 

{scriptworker-63.4.5 → scriptworker-63.6.0}/scriptworker.yaml.tmpl

@@ -45,9 +45,15 @@ verify_cot_signature: false
 # Chain of Trust job type, e.g. signing
 cot_job_type: scriptworker
 cot_product: firefox
-# Calls to Github API are limited to 60 an hour. Using an API token allows to raise the limit to
-# 5000 per hour. https://developer.github.com/v3/#rate-limiting
-github_oauth_token: somegithubtoken
+
+# Scriptworker first tries to obtain a repository scoped token from Taskcluster's auth service,
+# using the Github App registered under this name.
+# github_app_name: read
+#
+# `github_oauth_token` is used as a fallback if that fails (e.g. missing scopes or app not
+# configured). Without either, calls to the Github API are unauthenticated and limited to 60 an
+# hour. See https://developer.github.com/v3/#rate-limiting
+# github_oauth_token: somegithubtoken
 
 
 #-----------------------------------------------------------------------------------------------

{scriptworker-63.4.5 → scriptworker-63.6.0}/src/scriptworker/constants.py

@@ -79,7 +79,12 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
         "max_chain_length": 20,
         # Calls to Github API are limited to 60 an hour. Using an API token allows to raise the limit to
         # 5000 per hour. https://developer.github.com/v3/#rate-limiting
+        # Scriptworker first tries to obtain a repository scoped token from Taskcluster's auth
+        # service, falling back to this token if that fails.
         "github_oauth_token": "",
+        # The name of the Github App registered with Taskcluster's auth service, used to obtain
+        # a repository scoped token via `auth.githubRepoToken`.
+        "github_app_name": "read",
         # ed25519 settings
         "ed25519_private_key_path": "...",
         "ed25519_public_keys": immutabledict(
@@ -142,7 +147,8 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
                         "adhoc": "github",
                         "app-services": "github",
                         "enterprise": "github",
-                        "firefox": "hg",
+                        # Firefox is migrating from hg.mozilla.org to Github (bug 2070846)
+                        "firefox": ("hg", "github"),
                         "glean": "github",
                         "mobile": "github",
                         "mozillavpn": "github",
@@ -408,6 +414,7 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
                                     "netlocs": ("hg.mozilla.org",),
                                     "path_regexes": (
                                         r"^(?P<path>/comm-central)(/|$)",
+                                        r"^(?P<path>/integration/comm-autoland)(/|$)",
                                         r"^(?P<path>/releases/comm-(beta|release|esr\d+))(/|$)",
                                     ),
                                 }
@@ -435,6 +442,43 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
                 )
             }
         ),
+        # XXX While migrating Firefox from hg.mozilla.org to GitHub, graphs are
+        # still generated by decision tasks from hg, but some tasks in them
+        # clone from the GitHub mirror instead. This maps an hg repo to its
+        # GitHub mirror. A task whose source is the mirror of its parent task's
+        # repo is treated as coming from the parent's repo; its definition has
+        # already been verified against the parent task's graph. Parent
+        # (decision/action) tasks themselves are never mapped.
+        # Remove this once the migration is complete.
+        "hg_git_mirrors": immutabledict(
+            {
+                "by-cot-product": immutabledict(
+                    {
+                        "adhoc": immutabledict({}),
+                        "app-services": immutabledict({}),
+                        "enterprise": immutabledict({}),
+                        "firefox": immutabledict(
+                            {
+                                "/mozilla-central": "/mozilla-firefox/firefox",
+                                "/integration/autoland": "/mozilla-firefox/firefox",
+                                "/releases/mozilla-beta": "/mozilla-firefox/firefox",
+                                "/releases/mozilla-release": "/mozilla-firefox/firefox",
+                                "/releases/mozilla-esr115": "/mozilla-firefox/firefox",
+                                "/releases/mozilla-esr140": "/mozilla-firefox/firefox",
+                                "/releases/mozilla-esr153": "/mozilla-firefox/firefox",
+                            }
+                        ),
+                        "glean": immutabledict({}),
+                        "mobile": immutabledict({}),
+                        "mozillavpn": immutabledict({}),
+                        "scriptworker": immutabledict({}),
+                        "thunderbird": immutabledict({}),
+                        "translations": immutabledict({}),
+                        "xpi": immutabledict({}),
+                    }
+                )
+            }
+        ),
         "valid_tasks_for": immutabledict(
             {
                 "by-cot-product": immutabledict(
@@ -455,7 +499,15 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
                             "github-release",
                             "pr-action",
                         ),
-                        "firefox": ("hg-push", "cron", "action"),
+                        "firefox": (
+                            "action",
+                            "cron",
+                            "github-pull-request",
+                            "github-pull-request-untrusted",
+                            "github-push",
+                            "hg-push",
+                            "pr-action",
+                        ),
                         "glean": (
                             "action",
                             "cron",
@@ -514,7 +566,7 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
                             "mozilla",
                             "thunderbird",
                         ),
-                        "firefox": "",
+                        "firefox": "mozilla-firefox",
                         "glean": "mozilla",
                         "mobile": "mozilla-mobile",
                         "mozillavpn": "mozilla-mobile",
@@ -789,6 +841,7 @@ DEFAULT_CONFIG: immutabledict[str, Any] = immutabledict(
                                     "/releases/comm-esr153",
                                 ),
                                 "nightly": ("/comm-central",),
+                                "autoland": ("/integration/comm-autoland",),
                             }
                         ),
                         "translations": immutabledict(

{scriptworker-63.4.5 → scriptworker-63.6.0}/src/scriptworker/cot/verify.py

@@ -20,7 +20,6 @@ import tempfile
 from copy import deepcopy
 from urllib.parse import urlparse
 
-import aiohttp
 import dictdiffer
 import jsone
 from immutabledict import immutabledict
@@ -40,7 +39,13 @@ from scriptworker.constants import DEFAULT_CONFIG
 from scriptworker.context import Context
 from scriptworker.ed25519 import ed25519_public_key_from_string, verify_ed25519_signature
 from scriptworker.exceptions import BaseDownloadError, CoTError, ScriptWorkerEd25519Error
-from scriptworker.github import GitHubRepository, extract_github_repo_full_name, extract_github_repo_owner_and_name, extract_github_repo_ssh_url
+from scriptworker.github import (
+    GitHubRepository,
+    extract_github_repo_full_name,
+    extract_github_repo_owner_and_name,
+    extract_github_repo_ssh_url,
+    is_github_url,
+)
 from scriptworker.log import contextual_log_handler, get_chain_of_trust_log_filename
 from scriptworker.task import (
     get_action_callback_name,
@@ -62,6 +67,7 @@ from scriptworker.task import (
     get_triggered_by,
     get_worker_pool_id,
     is_action,
+    is_github_task,
     is_try_or_pull_request,
     retry_get_task_definition,
 )
@@ -1121,7 +1127,7 @@ async def _get_additional_github_releases_jsone_context(decision_link):
     repo_owner, repo_name = extract_github_repo_owner_and_name(repo_url)
     tag_name = get_revision(task, source_env_prefix)
 
-    github_repo = GitHubRepository(repo_owner, repo_name, context.config["github_oauth_token"])
+    github_repo = GitHubRepository(context, repo_owner, repo_name)
     release_data = await github_repo.get_release(tag_name)
 
     # The release data expose by the API[1] is not the same as the original event[2]. That's why
@@ -1203,17 +1209,16 @@ async def _get_additional_github_pull_request_jsone_context(decision_link):
     repo_url = repo_url.replace("git@github.com:", "ssh://github.com/", 1)
     repo_owner, repo_name = extract_github_repo_owner_and_name(repo_url)
     pull_request_number = get_pull_request_number(task, source_env_prefix)
-    token = context.config["github_oauth_token"]
 
-    github_repo = GitHubRepository(repo_owner, repo_name, token)
-    repo_definition = github_repo.definition
+    github_repo = GitHubRepository(context, repo_owner, repo_name)
+    repo_definition = await github_repo.get_definition()
 
     # We need to query the repository where the pull request was made to extract
     # pull request data. The pull request could be created on the same repo as
     # the commit, or an upstream repo. We can compare the base and head repo URLs
     # to infer where the pull request lives.
     if repo_definition["fork"] and base_repo_url != repo_url:
-        github_repo = GitHubRepository(owner=repo_definition["parent"]["owner"]["login"], repo_name=repo_definition["parent"]["name"], token=token)
+        github_repo = GitHubRepository(context, repo_definition["parent"]["owner"]["login"], repo_definition["parent"]["name"])
 
     pull_request_data = await github_repo.get_pull_request(pull_request_number)
     # Even though pull_request_data['head']['repo']['pushed_at'] does exist,
@@ -1248,7 +1253,7 @@ async def _get_additional_github_push_jsone_context(decision_link):
     repo_owner, repo_name = extract_github_repo_owner_and_name(repo_url)
     commit_hash = get_revision(task, source_env_prefix)
 
-    github_repo = GitHubRepository(repo_owner, repo_name, context.config["github_oauth_token"])
+    github_repo = GitHubRepository(context, repo_owner, repo_name)
     commit_data = await github_repo.get_commit(commit_hash)
 
     committer = commit_data["committer"] or {}
@@ -1346,7 +1351,18 @@ async def populate_jsone_context(chain, parent_link, decision_link, tasks_for):
         "taskId": None,
     }
 
-    if chain.context.config["cot_product_type"] == "github":
+    cot_product_type = chain.context.config["cot_product_type"]
+    # A tuple becomes a list when the config round-trips through json/yaml.
+    if isinstance(cot_product_type, (tuple, list)):
+        # Products migrating between VCSes support several types (bug 2070846).
+        # tasks_for like `action` and `cron` exist on both hg and github, so
+        # determine the type from the decision task itself.
+        detected_type = "github" if is_github_task(decision_link.task) else "hg"
+        if detected_type not in cot_product_type:
+            raise CoTError('cot_product_type "{}" is not enabled for cot_product "{}"!'.format(detected_type, chain.context.config["cot_product"]))
+        cot_product_type = detected_type
+
+    if cot_product_type == "github":
         if tasks_for == "github-release":
             jsone_context.update(await _get_additional_github_releases_jsone_context(decision_link))
         elif tasks_for == "cron":
@@ -1359,7 +1375,7 @@ async def populate_jsone_context(chain, parent_link, decision_link, tasks_for):
             jsone_context.update(await _get_additional_github_push_jsone_context(decision_link))
         else:
             raise CoTError('Unknown tasks_for "{}" for github cot_product "{}"!'.format(tasks_for, chain.context.config["cot_product"]))
-    elif chain.context.config["cot_product_type"] == "hg":
+    elif cot_product_type == "hg":
         source_url = get_source_url(decision_link)
         project = await get_project(chain.context, source_url)
         jsone_context["repository"] = {
@@ -1379,9 +1395,7 @@ async def populate_jsone_context(chain, parent_link, decision_link, tasks_for):
         else:
             raise CoTError('Unknown tasks_for "{}" for hg cot_product "{}"!'.format(tasks_for, chain.context.config["cot_product"]))
     else:
-        raise CoTError(
-            'Unknown cot_product_type "{}" for cot_product "{}"!'.format(chain.context.config["cot_product_type"], chain.context.config["cot_product"])
-        )
+        raise CoTError('Unknown cot_product_type "{}" for cot_product "{}"!'.format(cot_product_type, chain.context.config["cot_product"]))
 
     log.debug("{} json-e context:".format(parent_link.name))
     # format_json() breaks on lambda values; use pprint.pformat here.
@@ -1402,16 +1416,11 @@ def build_taskcluster_yml_url(link):
     """
     source_env_prefix = link.context.config["source_env_prefix"]
     repo_url = get_repo(link.task, source_env_prefix)
-    repo_url = repo_url.replace("git@github.com:", "ssh://github.com/", 1)
     revision = get_revision(link.task, source_env_prefix)
     repo_parts = urlparse(repo_url)
-    if repo_parts.netloc == "github.com":
-        user, repo_name = extract_github_repo_owner_and_name(repo_url)
-        url = f"https://raw.githubusercontent.com/{user}/{repo_name}/{revision}/.taskcluster.yml"
-    elif repo_parts.netloc == "hg.mozilla.org":
-        url = f"{repo_parts.scheme}://{repo_parts.netloc}{repo_parts.path}/raw-file/{revision}/.taskcluster.yml"
-    else:
+    if repo_parts.netloc != "hg.mozilla.org":
         raise CoTError("Unsupported VCS server!")
+    url = f"{repo_parts.scheme}://{repo_parts.netloc}{repo_parts.path}/raw-file/{revision}/.taskcluster.yml"
     log.debug(f"{link.name} .taskcluster.yml is at {url}")
     return url
 
@@ -1433,19 +1442,20 @@ async def get_in_tree_template(link):
 
     """
     context = link.context
-    source_url = build_taskcluster_yml_url(link)
-    repo_url = get_repo(link.task, link.context.config["source_env_prefix"])
+    source_env_prefix = context.config["source_env_prefix"]
+    repo_url = get_repo(link.task, source_env_prefix)
+    repo_url = repo_url.replace("git@github.com:", "ssh://github.com/", 1)
 
-    auth = None
-    if (
-        (repo_url.startswith(("ssh://", "git@github.com")) or any(vcs_rule.get("require_secret") for vcs_rule in context.config["trusted_vcs_rules"]))
-        and "github.com" in repo_url
-        and context.config.get("github_oauth_token")
-    ):
-        auth = aiohttp.BasicAuth(context.config["github_oauth_token"])
+    if is_github_url(repo_url):
+        revision = get_revision(link.task, source_env_prefix)
+        repo_owner, repo_name = extract_github_repo_owner_and_name(repo_url)
+        github_repo = GitHubRepository(context, repo_owner, repo_name)
+        content = await github_repo.get_file_contents(".taskcluster.yml", ref=revision)
+        return load_json_or_yaml(content, file_type="yaml")
+
+    source_url = build_taskcluster_yml_url(link)
     url_hash = hashlib.sha1(source_url.encode("ascii")).hexdigest()
-    tmpl = await load_json_or_yaml_from_url(context, source_url, os.path.join(context.config["work_dir"], "{}_taskcluster.yml".format(url_hash)), auth=auth)
-    return tmpl
+    return await load_json_or_yaml_from_url(context, source_url, os.path.join(context.config["work_dir"], "{}_taskcluster.yml".format(url_hash)))
 
 
 def _get_action_from_actions_json(all_actions, callback_name):
@@ -1983,6 +1993,44 @@ def get_source_url(obj):
     return source
 
 
+# hg_git_mirrors {{{1
+def _get_parent_repo(chain, repos):
+    """Get the repo of ``chain``'s parent (decision/action) tasks.
+
+    These are the tasks whose definitions are rebuilt from the tree, and that
+    the other tasks in the graph derive their trust from.
+
+    Args:
+        chain (ChainOfTrust): the chain we're operating on
+        repos (dict): the repo path of each object in the chain
+
+    Returns:
+        str: the parent tasks' repo path. If they don't agree, or can't be
+            found, ``chain``'s repo; any mismatch is reported by the sibling check.
+
+    """
+    parent_repos = {repo for obj, repo in repos.items() if obj.decision_task_id == chain.decision_task_id and obj.task_type in PARENT_TASK_TYPES}
+    if len(parent_repos) == 1:
+        return parent_repos.pop()
+    return repos[chain]
+
+
+def _is_git_mirror(chain, repo, parent_repo):
+    """Determine whether ``repo`` is the git mirror of ``parent_repo``.
+
+    Args:
+        chain (ChainOfTrust): the chain we're operating on
+        repo (str): a non-parent task's repo path
+        parent_repo (str): the parent tasks' repo path
+
+    Returns:
+        bool: whether ``repo`` is the git mirror of ``parent_repo``.
+
+    """
+    mirrors = chain.context.config["hg_git_mirrors"] or {}
+    return repo is not None and mirrors.get(parent_repo) == repo
+
+
 # trace_back_to_tree {{{1
 async def trace_back_to_tree(chain):
     """Trace the chain back to the tree.
@@ -2011,6 +2059,17 @@ async def trace_back_to_tree(chain):
         source_url = get_source_url(obj)
         repo_path = match_url_regex(chain.context.config["trusted_vcs_rules"], source_url, match_url_path_callback)
         repos[obj] = repo_path
+    # XXX During the hg -> git migration, graphs generated from hg may contain
+    # tasks sourced from the git mirror. Treat those as coming from the parent
+    # tasks' repo. Parent tasks are never mapped: their source is what their
+    # definition gets rebuilt from, so it must be trusted in its own right.
+    parent_repo = _get_parent_repo(chain, repos)
+    for obj, repo in repos.items():
+        if obj.decision_task_id != chain.decision_task_id or obj.task_type in PARENT_TASK_TYPES:
+            continue
+        if _is_git_mirror(chain, repo, parent_repo):
+            log.info(f"{obj.name} {obj.task_id}: {repo} is the git mirror of {parent_repo}")
+            repos[obj] = parent_repo
     # check for restricted scopes.
     my_repo = repos[chain]
     for scope in chain.task["scopes"]:

{scriptworker-63.4.5 → scriptworker-63.6.0}/src/scriptworker/github.py

@@ -1,11 +1,16 @@
 """GitHub helper functions."""
 
 import asyncio
+import hashlib
 import logging
 import re
+from pathlib import Path
+from urllib.parse import unquote, urlparse
 
 from github3 import GitHub
 from github3.exceptions import GitHubException
+from taskcluster.aio import Auth
+from taskcluster.exceptions import TaskclusterFailure
 
 from scriptworker.exceptions import ConfigError
 from scriptworker.utils import get_parts_of_url_path, get_single_item_from_sequence, retry_async_decorator, retry_request, retry_sync
@@ -23,30 +28,110 @@ log = logging.getLogger(__name__)
 class GitHubRepository:
     """Wrapper around GitHub API. Used to access public data."""
 
-    def __init__(self, owner, repo_name, token=""):
-        """Build the GitHub API URL which points to the definition of the repository.
+    GITHUB_PERMISSIONS = {"contents": "read", "metadata": "read", "pull_requests": "read"}
+
+    def __init__(self, context, owner, repo_name):
+        """Store the repository coordinates. The github3 repository object is built lazily.
 
         Args:
-            owner (str): the owner's GitHub username
+            context (scriptworker.context.Context): the scriptworker context
+            owner (str): the owner of the repository
             repo_name (str): the name of the repository
-            token (str): the GitHub API token
+
+        """
+        self._context = context
+        self._owner = owner
+        self._repo_name = repo_name
+        self._repository_cache = None
+        self._repository_lock = asyncio.Lock()
+
+    async def _get_repository(self):
+        """Build and cache the github3 repository object.
 
         Returns:
-            dict: a representation of the repo definition
+            github3.repos.repo.Repository: the github3 repository object
 
         """
-        github = retry_sync(GitHub, kwargs={"token": token}, sleeptime_kwargs=_GITHUB_LIBRARY_SLEEP_TIME_KWARGS)
-        self._github_repository = retry_sync(github.repository, args=(owner, repo_name), sleeptime_kwargs=_GITHUB_LIBRARY_SLEEP_TIME_KWARGS)
+        async with self._repository_lock:
+            if self._repository_cache is None:
+                token = await self._get_token(self._context, self._owner, self._repo_name)
+                github = retry_sync(GitHub, kwargs={"token": token}, sleeptime_kwargs=_GITHUB_LIBRARY_SLEEP_TIME_KWARGS)
+                self._repository_cache = retry_sync(github.repository, args=(self._owner, self._repo_name), sleeptime_kwargs=_GITHUB_LIBRARY_SLEEP_TIME_KWARGS)
+
+        return self._repository_cache
+
+    async def _get_token(self, context, owner, repo_name):
+        """Get a repository scoped GitHub token from Taskcluster's auth service.
+
+        Falls back to ``context.config["github_oauth_token"]`` if the auth service call
+        fails, e.g. because of missing scopes.
+
+        Args:
+            context (scriptworker.context.Context): the scriptworker context
+            owner (str): the owner of the repository
+            repo_name (str): the name of the repository
+
+        Returns:
+            str: the scoped GitHub token, or the fallback token
 
-    @property
-    def definition(self):
+        """
+        if not context.credentials:
+            return context.config.get("github_oauth_token", "")
+
+        try:
+            auth = Auth(options={"rootUrl": context.config["taskcluster_root_url"], "credentials": context.credentials})
+            response = await auth.githubRepoToken(
+                context.config["github_app_name"], owner, payload={"repositories": [repo_name], "permissions": self.GITHUB_PERMISSIONS}
+            )
+            return response["token"]
+        except TaskclusterFailure as e:
+            # TODO When opening a PR from a fork, we're guaranteed to hit this
+            # fallback as the task won't have auth service scopes for the repo
+            # fork. We'll need to improve this before we can stop depending on
+            # `github_oauth_token`.
+            log.warning(f"Could not obtain Github token from Taskcluster for {owner}/{repo_name}, falling back to `github_oauth_token`: {e}")
+            return context.config.get("github_oauth_token", "")
+
+    async def get_definition(self):
         """Fetch the definition of the repository, exposed by the GitHub API.
 
         Returns:
             dict: a representation of the repo definition
 
         """
-        return self._github_repository.as_dict()
+        repository = await self._get_repository()
+        return repository.as_dict()
+
+    @retry_async_decorator(retry_exceptions=GitHubException)
+    async def get_file_contents(self, path, ref=None):
+        """Fetch the decoded contents of a file in the repository.
+
+        Args:
+            path (str): the path to the file, relative to the repository root
+            ref (str, optional): the commit/branch/tag to read the file from.
+                Defaults to the repository's default branch. Results for full
+                commit hashes are cached on disk in `work_dir`.
+
+        Returns:
+            str: the decoded contents of the file
+
+        """
+        # Files at a full commit hash are immutable, so they're safe to cache on disk.
+        cache_path = None
+        if ref and _is_git_full_hash(ref):
+            cache_key = hashlib.sha1(f"{self._owner}/{self._repo_name}@{ref}:{path}".encode("utf-8")).hexdigest()
+            cache_path = Path(self._context.config["work_dir"]) / ".cache" / "github_files" / cache_key
+            if cache_path.exists():
+                return cache_path.read_text(encoding="utf-8")
+
+        repository = await self._get_repository()
+        contents = repository.file_contents(path, ref=ref)
+        decoded = contents.decoded.decode("utf-8")
+
+        if cache_path:
+            cache_path.parent.mkdir(parents=True, exist_ok=True)
+            cache_path.write_text(decoded, encoding="utf-8")
+        return decoded
 
     @retry_async_decorator(retry_exceptions=GitHubException)
     async def get_commit(self, commit_hash):
@@ -59,7 +144,8 @@ class GitHubRepository:
             dict: a representation of the commit
 
         """
-        return self._github_repository.commit(commit_hash).as_dict()
+        repository = await self._get_repository()
+        return repository.commit(commit_hash).as_dict()
 
     @retry_async_decorator(retry_exceptions=GitHubException)
     async def get_pull_request(self, pull_request_number):
@@ -72,7 +158,8 @@ class GitHubRepository:
             dict: a representation of the pull request
 
         """
-        return self._github_repository.pull_request(pull_request_number).as_dict()
+        repository = await self._get_repository()
+        return repository.pull_request(pull_request_number).as_dict()
 
     @retry_async_decorator(retry_exceptions=GitHubException)
     async def get_release(self, tag_name):
@@ -85,7 +172,8 @@ class GitHubRepository:
             dict: a representation of the tag
 
         """
-        return self._github_repository.release_from_tag(tag_name).as_dict()
+        repository = await self._get_repository()
+        return repository.release_from_tag(tag_name).as_dict()
 
     @retry_async_decorator(retry_exceptions=GitHubException)
     async def get_tag_hash(self, tag_name):
@@ -98,8 +186,9 @@ class GitHubRepository:
             str: the commit hash linked by the tag
 
         """
+        repository = await self._get_repository()
         tag_object = get_single_item_from_sequence(
-            sequence=self._github_repository.tags(),
+            sequence=repository.tags(),
             condition=lambda tag: tag.name == tag_name,
             no_item_error_message='No tag "{}" exist'.format(tag_name),
             too_many_item_error_message='Too many tags "{}" found'.format(tag_name),
@@ -128,7 +217,8 @@ class GitHubRepository:
         if not _is_git_full_hash(revision):
             revision = await self.get_tag_hash(tag_name=revision)
 
-        html_text = await _fetch_github_branch_commits_data(context, self._github_repository.html_url, revision)
+        repository = await self._get_repository()
+        html_text = await _fetch_github_branch_commits_data(context, repository.html_url, revision)
 
         # https://github.com/{repo_owner}/{repo_name}/branch_commits/{revision} just returns some \n
         # when the commit hasn't landed on the origin repo. Otherwise, some HTML data is returned - it
@@ -202,6 +292,8 @@ def extract_github_repo_owner_and_name(url):
     _check_github_url_is_supported(url)
 
     parts = get_parts_of_url_path(url)
+    if len(parts) < 2:
+        raise ValueError(f"Could not extract repo owner and name from GitHub URL: {url}")
     repo_owner = parts[0]
     repo_name = parts[1]
 
@@ -257,17 +349,20 @@ def extract_github_repo_and_revision_from_source_url(url):
     """
     _check_github_url_is_supported(url)
 
-    parts = get_parts_of_url_path(url)
-    repo_name = parts[1]
+    parsed = urlparse(url)
+    parts = unquote(parsed.path).lstrip("/").split("/")
+    if len(parts) < 2:
+        raise ValueError(f"Could not extract repo owner and name from GitHub URL: {url}")
+    repo_owner = parts[0]
+    repo_name = _strip_trailing_dot_git(parts[1])
     try:
         revision = parts[3]
     except IndexError:
-        raise ValueError("Revision cannot be extracted from url: {}".format(url))
+        raise ValueError(f"Revision cannot be extracted from url: {url}")
 
-    end_index = url.index(repo_name) + len(repo_name)
-    repo_url = url[:end_index]
+    repo_url = f"{parsed.scheme}://{parsed.netloc}/{repo_owner}/{repo_name}"
 
-    return _strip_trailing_dot_git(repo_url), revision
+    return repo_url, revision
 
 
 def _strip_trailing_dot_git(url):

{scriptworker-63.4.5 → scriptworker-63.6.0}/src/scriptworker/task.py

@@ -550,7 +550,7 @@ async def is_pull_request(context, task):
         if not revision and can_skip:
             continue
 
-        github_repository = GitHubRepository(repo_owner, repo_name, context.config["github_oauth_token"])
+        github_repository = GitHubRepository(context, repo_owner, repo_name)
         conditions.append(not await github_repository.has_commit_landed_on_repository(context, revision))
 
     return any(conditions)