taskcluster-taskgraph 24.2.2__py3-none-any.whl → 24.3.0__py3-none-any.whl

{taskcluster_taskgraph-24.2.2.dist-info → taskcluster_taskgraph-24.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: taskcluster-taskgraph
-Version: 24.2.2
+Version: 24.3.0
 Summary: Build taskcluster taskgraphs
 Project-URL: Repository, https://github.com/taskcluster/taskgraph
 Project-URL: Issues, https://github.com/taskcluster/taskgraph/issues

{taskcluster_taskgraph-24.2.2.dist-info → taskcluster_taskgraph-24.3.0.dist-info}/RECORD

@@ -5,7 +5,7 @@ taskgraph/decision.py,sha256=V5TUWSeIlUbKcRFMBGhYd3t7wLmd2rwAgdx6jN4Yt1Q,11769
 taskgraph/docker.py,sha256=5k9ZjooqvtBDtb3AAZijUdUNzgRcm6CXM5CrmLO0KFA,24429
 taskgraph/filter_tasks.py,sha256=R7tYXiaVPGIkQ6O1c9-QJrKZ59m9pFXCloUlPraVnZU,866
 taskgraph/generator.py,sha256=zanO2GOb0HWEGOQkEZGmXP0JySql71xoeENjk-cKOc8,23170
-taskgraph/graph.py,sha256=GGNY9n44xVgz_LbnprbQDY1IFcNV9oxhrBavu3SW-V8,6202
+taskgraph/graph.py,sha256=Gnv4pC2sRn4rVPNegnXD1ni3eUb1ULju04zq-6bmH0A,6304
 taskgraph/main.py,sha256=miuQkXuxD9eAUa0MqORARdyCRVtsqobLA-vW6n1sf_Y,39424
 taskgraph/morph.py,sha256=pF9_bqM2aNpNfSH7xEFWQz83ohxxVdUUDqdv2yq7waA,9232
 taskgraph/parameters.py,sha256=dlEC048z06WKdIL8QhH1nu2hy9jF4AQ4KcVfJauIXyk,13867
@@ -26,10 +26,10 @@ taskgraph/loader/transform.py,sha256=_PPHSLdnqa8MUlXqIZSkRsGAndQBVTZjNLAaIUVU5ps
 taskgraph/optimize/__init__.py,sha256=Oqpq1RW8QzOcu7zaMlNQ3BHT9ws9e_93FWfCqzNcQps,123
 taskgraph/optimize/base.py,sha256=c2ETEJChLCFm6yoA-IcWepu7V0eOIA_e_tQfFwld-EA,20449
 taskgraph/optimize/strategies.py,sha256=SAzMXbp_oQ3EJ2pDLOvShh2RosJExCRTZR4PFsD-3l0,3776
-taskgraph/run-task/fetch-content,sha256=ZpjiX-SmosoKBGyG-GxWtAGvUCzspWMrmoFQXhbhw8s,36209
+taskgraph/run-task/fetch-content,sha256=0QsAbgf8O65N8kvfFc2d0YjOKl-oqATXhRLBeXwRwTE,39216
 taskgraph/run-task/hgrc,sha256=BybWLDR89bWi3pE5T05UqmDHs02CbLypE-omLZWU6Uk,896
-taskgraph/run-task/robustcheckout.py,sha256=DZ-MB68P7FmZY1vkj7o3Xp-d21aOiFnLMimv25S3FTs,30778
-taskgraph/run-task/run-task,sha256=yEQqTaw2u7hgFqlUmWsN10J89GFarZhQtIheFl6WI0I,47409
+taskgraph/run-task/robustcheckout.py,sha256=85tcHqlAlWre1RYJ5qu3CJtqAAsMnB9AlDeN4zfpOrQ,33021
+taskgraph/run-task/run-task,sha256=DRpXBcHXLmfLPwsz7S0IJwcWC0XFIDiE99Pu2yZCZrA,58070
 taskgraph/transforms/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 taskgraph/transforms/base.py,sha256=NULQNfwJfBTo0Bem1hp_6aDztUABwbnJrQ0MBpOgGDE,5125
 taskgraph/transforms/cached_tasks.py,sha256=Izxfl4_UG56Hk0W2vIj5OK8hji7adG2zfvtELJNaE6Q,2592
@@ -40,7 +40,7 @@ taskgraph/transforms/fetch.py,sha256=Rf8FGMDY84sLNKp1mw7MVQspiJvses5-njWmIBIi0AQ
 taskgraph/transforms/from_deps.py,sha256=hifdCsKY_kiBzUCxsOAfCR0VR525x-H0kWNutygTDCQ,8085
 taskgraph/transforms/matrix.py,sha256=anoUjnPnpvguQgHkj2HiVdHs3hDCROxQ6Wb0dLJ6l_8,3069
 taskgraph/transforms/notify.py,sha256=KTku8694xEicls5vhUC4Uvpg-aaSwDUsVizVlR5RObw,6360
-taskgraph/transforms/task.py,sha256=DhQBnEgqvJw_qdvXuwHirwS8oL08iIeDP_sO4a238R4,43139
+taskgraph/transforms/task.py,sha256=6OP9b-EWc4AZjfBF-Dh4h7nIyB5DLcwUaAMNAFt8h-g,43148
 taskgraph/transforms/task_context.py,sha256=diijTlofODLCkgc0ZDR02_VojGXDW8m54lAvEkUIpBc,5144
 taskgraph/transforms/run/__init__.py,sha256=s9GkP3rYtLUa8IbpfZQC-x5YiiQSQHjHVfgf8jaEOWg,18195
 taskgraph/transforms/run/common.py,sha256=ANiOZJ0sSl1lL09Qo3aTESFd1gK6SvwLX8cTj_J2SmU,6861
@@ -75,8 +75,8 @@ taskgraph/util/vcs.py,sha256=Z2sCJT0-kwP_AbSUMIzya9lk8SyjWPzMYtQPYJ2WtnA,22496
 taskgraph/util/verify.py,sha256=hY6xl4rEgXeVuPswG8HCGBHJNKHF2QXx2cTxrrRtDeE,14829
 taskgraph/util/workertypes.py,sha256=SpqQmf99kB9QxtgOdXeMUvzuVv1o0ZVB4lYahZH3jDY,2534
 taskgraph/util/yaml.py,sha256=29h6RE7JA4z0U2V3WCu-S39lPMBS9CEZEglv1delDvw,1075
-taskcluster_taskgraph-24.2.2.dist-info/METADATA,sha256=dgtz4W6pVH4-KRYuFfTfcs7rJyUUq6AnEVJ9uT56stk,5065
-taskcluster_taskgraph-24.2.2.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
-taskcluster_taskgraph-24.2.2.dist-info/entry_points.txt,sha256=2hxDzE3qq_sHh-J3ROqwpxgQgxO-196phWAQREl2-XA,50
-taskcluster_taskgraph-24.2.2.dist-info/licenses/LICENSE,sha256=HyVuytGSiAUQ6ErWBHTqt1iSGHhLmlC8fO7jTCuR8dU,16725
-taskcluster_taskgraph-24.2.2.dist-info/RECORD,,
+taskcluster_taskgraph-24.3.0.dist-info/METADATA,sha256=CNocuQbuMOJ1k24k2MTavl1rSVvRmBes4ny6OJyKYek,5065
+taskcluster_taskgraph-24.3.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+taskcluster_taskgraph-24.3.0.dist-info/entry_points.txt,sha256=2hxDzE3qq_sHh-J3ROqwpxgQgxO-196phWAQREl2-XA,50
+taskcluster_taskgraph-24.3.0.dist-info/licenses/LICENSE,sha256=HyVuytGSiAUQ6ErWBHTqt1iSGHhLmlC8fO7jTCuR8dU,16725
+taskcluster_taskgraph-24.3.0.dist-info/RECORD,,

{taskcluster_taskgraph-24.2.2.dist-info → taskcluster_taskgraph-24.3.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.31.0
+Generator: hatchling 1.32.0
 Root-Is-Purelib: true
 Tag: py3-none-any

taskgraph/graph.py

@@ -61,20 +61,24 @@ class Graph(_Graph):
                 f"Unknown nodes in transitive closure: {nodes - self.nodes}"
             )
 
-        # generate a new graph by expanding along edges until reaching a fixed
-        # point
-        new_nodes, new_edges = nodes, set()
-        nodes, edges = set(), set()
-        while (new_nodes, new_edges) != (nodes, edges):
-            nodes, edges = new_nodes, new_edges
-            add_edges = {
-                (left, right, name)
-                for (left, right, name) in self.edges
-                if (right if reverse else left) in nodes
-            }
-            add_nodes = {(left if reverse else right) for (left, right, _) in add_edges}
-            new_nodes = nodes | add_nodes
-            new_edges = edges | add_edges
+        # Build an adjacency map keyed by the node to expand from. This reduces
+        # traversal below from O(V·E) -> O(V+E).
+        adjacency = collections.defaultdict(list)
+        for edge in self.edges:
+            left, right, _ = edge
+            adjacency[right if reverse else left].append(edge)
+
+        new_nodes = set(nodes)
+        new_edges = set()
+        queue = collections.deque(nodes)
+        while queue:
+            node = queue.popleft()
+            for edge in adjacency.get(node, ()):
+                new_edges.add(edge)
+                neighbor = edge[0] if reverse else edge[1]
+                if neighbor not in new_nodes:
+                    new_nodes.add(neighbor)
+                    queue.append(neighbor)
         return Graph(new_nodes, new_edges)
 
     def _visit(self, reverse):

taskgraph/run-task/fetch-content

@@ -18,6 +18,7 @@ import os
 import pathlib
 import random
 import re
+import shutil
 import ssl
 import stat
 import subprocess
@@ -377,6 +378,70 @@ def archive_type(path: pathlib.Path):
         return None
 
 
+class TarFile(tarfile.TarFile):
+    # When a tarball contains symbolic links, the python tarfile module ends up
+    # trying to seek in the tar, which doesn't quite work when the tar is a
+    # stream (which it is when decompressing at the same time). We wrap TarFile
+    # so as to handle the symbolic link situation ourselves, and when we can't
+    # create symbolic links (e.g. on Windows), we instead create a copy of the
+    # file the link points to.
+    def _do_extract(
+        self, extract, member, path="", set_attrs=True, numeric_owner=False, **kwargs
+    ):
+        deferred_links = getattr(self, "_deferred_links", None)
+        if not isinstance(member, tarfile.TarInfo):
+            member = self.getmember(member)
+        targetpath = os.path.normcase(os.path.join(path, member.name))
+
+        if deferred_links is not None and member.issym():
+            if os.path.lexists(targetpath):
+                # Avoid FileExistsError on following os.symlink.
+                os.unlink(targetpath)
+            try:
+                os.symlink(member.linkname, targetpath)
+            except (NotImplementedError, OSError):
+                # On Windows, os.symlink can fail, in this case fallback to
+                # creating a copy. If the destination was not already created,
+                # defer the link creation.
+                source = os.path.normcase(
+                    os.path.join(os.path.dirname(targetpath), member.linkname)
+                )
+
+                if source in self._extracted_members:
+                    shutil.copy(source, targetpath)
+                    self.chown(member, targetpath, numeric_owner)
+                else:
+                    deferred_links.setdefault(source, []).append(
+                        (member, targetpath, numeric_owner)
+                    )
+            return
+
+        extract(member, path, set_attrs, numeric_owner=numeric_owner, **kwargs)
+        if deferred_links is not None:
+            for tarinfo, linkpath, numeric_owner in deferred_links.pop(targetpath, []):
+                shutil.copy(targetpath, linkpath)
+                self.chown(tarinfo, linkpath, numeric_owner)
+            self._extracted_members.add(targetpath)
+
+    def extract(self, *args, **kwargs):
+        self._do_extract(super(TarFile, self).extract, *args, **kwargs)
+
+    # extractall in versions for cpython that implement PEP 706 call _extract_one
+    # instead of extract.
+    def _extract_one(self, *args, **kwargs):
+        self._do_extract(super(TarFile, self)._extract_one, *args, **kwargs)
+
+    def extractall(self, *args, **kwargs):
+        self._deferred_links = {}
+        self._extracted_members = set()
+        super(TarFile, self).extractall(*args, **kwargs)
+        for links in self._deferred_links.values():
+            for tarinfo, linkpath, numeric_owner in links:
+                log(f"Cannot create dangling symbolic link: {linkpath}")
+        delattr(self, "_deferred_links")
+        delattr(self, "_extracted_members")
+
+
 def extract_archive(path, dest_dir):
     """Extract an archive to a destination directory."""
 
@@ -398,7 +463,7 @@ def extract_archive(path, dest_dir):
         # also not much slower on Windows, presumably because of the
         # notoriously bad I/O).
         if sys.platform == "win32":
-            tar = tarfile.open(fileobj=ifh, mode="r|")
+            tar = TarFile.open(fileobj=ifh, mode="r|")
             tar.extractall(str(dest_dir))
             args = []
         else:

taskgraph/run-task/robustcheckout.py

@@ -18,6 +18,7 @@ import random
 import re
 import socket
 import ssl
+import stat
 import time
 
 from mercurial.i18n import _
@@ -38,6 +39,22 @@ from mercurial import (
     vfs,
 )
 
+# TRACKING hg72 - `hg.repository`/`hg.peer` moved to `mercurial.repo.factory`.
+try:
+    from mercurial.repo import factory as _repofactory
+
+    _repository = _repofactory.repository
+    _peer = _repofactory.peer
+except ImportError:
+    _repository = hg.repository
+    _peer = hg.peer
+
+# TRACKING hg72 - `hg.clone` moved to `mercurial.cmd_impls.clone.clone`.
+try:
+    from mercurial.cmd_impls.clone import clone as _clone
+except ImportError:
+    _clone = hg.clone
+
 # Causes worker to purge caches on process exit and for task to retry.
 EXIT_PURGE_CACHE = 72
 
@@ -67,6 +84,57 @@ def peerlookup(remote, v):
         return e.callcommand(b"lookup", {b"key": v}).result()
 
 
+def remove_dangling_links(ui, path):
+    """On Windows, remove dangling symlinks and junctions under ``path``.
+
+    npm's ``file:`` protocol dependencies create directory junctions in
+    ``node_modules/`` with absolute targets.  When a Taskcluster cache
+    is restored to a different task directory those targets no longer
+    exist, and Mercurial's purge crashes in ``vfs.listdir()`` with
+    ``FileNotFoundError``.  Removing them first lets purge proceed.
+    """
+    if os.name != "nt":
+        return
+
+    ui.write(b"windows detected, removing dangling links\n")
+
+    stack = [path]
+    while stack:
+        dirpath = stack.pop()
+        try:
+            entries = os.scandir(dirpath)
+        except OSError:
+            continue
+
+        with entries:
+            for entry in entries:
+                p = entry.path
+                try:
+                    attrs = entry.stat(follow_symlinks=False).st_file_attributes
+                except OSError:
+                    continue
+
+                if attrs & stat.FILE_ATTRIBUTE_REPARSE_POINT:
+                    if not os.path.exists(p):
+                        ui.write(
+                            b"(removing dangling link %s)\n"
+                            % os.fsencode(os.path.relpath(p, path))
+                        )
+                        try:
+                            os.rmdir(p)
+                        except OSError:
+                            os.remove(p)
+                    continue
+
+                try:
+                    is_dir = entry.is_dir(follow_symlinks=False)
+                except OSError:
+                    continue
+
+                if is_dir:
+                    stack.append(p)
+
+
 @command(
     b"robustcheckout",
     [
@@ -427,7 +495,7 @@ def _docheckout(
     def handlerepoerror(e):
         if pycompat.bytestr(e) == _(b"abandoned transaction found"):
             ui.warn(b"(abandoned transaction found; trying to recover)\n")
-            repo = hg.repository(ui, dest)
+            repo = _repository(ui, dest)
             if not repo.recover():
                 ui.warn(b"(could not recover repo state; deleting shared store)\n")
                 with timeit("remove_unrecovered_shared_store", "remove-store"):
@@ -535,7 +603,7 @@ def _docheckout(
     cloneurl = upstream or url
 
     try:
-        clonepeer = hg.peer(ui, {}, cloneurl)
+        clonepeer = _peer(ui, {}, cloneurl)
         rootnode = peerlookup(clonepeer, b"0")
     except error.RepoLookupError:
         raise error.Abort(b"unable to resolve root revision from clone source")
@@ -623,12 +691,12 @@ def _docheckout(
             ui.write(b"(cloning from upstream repo %s)\n" % upstream)
 
         if not storevfs.exists():
-            behaviors.add(b"create-store")
+            behaviors.add("create-store")
 
         try:
             with timeit("clone", "clone"):
                 shareopts = {b"pool": sharebase, b"mode": b"identity"}
-                res = hg.clone(
+                res = _clone(
                     ui,
                     {},
                     clonepeer,
@@ -667,7 +735,7 @@ def _docheckout(
     # The destination .hg directory should exist. Now make sure we have the
     # wanted revision.
 
-    repo = hg.repository(ui, dest)
+    repo = _repository(ui, dest)
 
     # We only pull if we are using symbolic names or the requested revision
     # doesn't exist.
@@ -694,7 +762,7 @@ def _docheckout(
 
         remote = None
         try:
-            remote = hg.peer(repo, {}, url)
+            remote = _peer(repo, {}, url)
             pullrevs = [peerlookup(remote, revision or branch)]
             checkoutrevision = hex(pullrevs[0])
             if branch:
@@ -741,6 +809,10 @@ def _docheckout(
     # guaranteed to not have conflicts on `hg update`.
     if purge and not created:
         ui.write(b"(purging working directory)\n")
+
+        with timeit("purge", "dangling_link_remove"):
+            remove_dangling_links(ui, dest)
+
         purge = getattr(commands, "purge", None)
         if not purge:
             purge = extensions.find(b"purge").purge

taskgraph/run-task/run-task

@@ -27,12 +27,13 @@ import socket
 import stat
 import subprocess
 import sys
+import tempfile
 import threading
 import time
 import urllib.error
 import urllib.request
 from pathlib import Path
-from typing import Dict, Optional
+from typing import Dict, List, Literal, NamedTuple, Optional
 
 SECRET_BASEURL_TPL = "{}/secrets/v1/secret/{{}}".format(
     os.environ.get("TASKCLUSTER_PROXY_URL", "http://taskcluster").rstrip("/")
@@ -234,8 +235,8 @@ def remove(path):
         _call_windows_retry(shutil.rmtree, (path,))
 
 
-def run_required_command(prefix, args, *, extra_env=None, cwd=None):
-    res = run_command(prefix, args, extra_env=extra_env, cwd=cwd)
+def run_required_command(prefix, args, *, extra_env=None, cwd=None, stdin_data=None):
+    res = run_command(prefix, args, extra_env=extra_env, cwd=cwd, stdin_data=stdin_data)
     if res:
         sys.exit(res)
 
@@ -253,7 +254,7 @@ def retry_required_command(prefix, args, *, extra_env=None, cwd=None, retries=2)
         time.sleep(backoff)
 
 
-def run_command(prefix, args, *, extra_env=None, cwd=None):
+def run_command(prefix, args, *, extra_env=None, cwd=None, stdin_data=None):
     """Runs a process and prefixes its output with the time.
 
     Returns the process exit code.
@@ -274,6 +275,12 @@ def run_command(prefix, args, *, extra_env=None, cwd=None):
     # we manually install a latin1 stream wrapper. This allows us to readline()
     # and preserves bytes, without losing any data.
 
+    stdin_file = None
+    if stdin_data is not None:
+        stdin_file = tempfile.TemporaryFile()
+        stdin_file.write(stdin_data)
+        stdin_file.seek(0)
+
     p = subprocess.Popen(
         args,
         # Disable buffering because we want to receive output
@@ -282,11 +289,14 @@ def run_command(prefix, args, *, extra_env=None, cwd=None):
         bufsize=0,
         stdout=subprocess.PIPE,
         stderr=subprocess.STDOUT,
-        stdin=sys.stdin.fileno(),
+        stdin=stdin_file if stdin_file is not None else sys.stdin.fileno(),
         cwd=cwd,
         env=env,
     )
 
+    if stdin_file is not None:
+        stdin_file.close()
+
     stdout = io.TextIOWrapper(p.stdout, encoding="latin1")
 
     if os.getpid() == 1:
@@ -571,8 +581,9 @@ def configure_volume_posix(volume, user, group, running_as_root):
 class PerfRecorder:
     """Collects operation timings and emits them as PERFHERDER_DATA."""
 
-    def __init__(self, framework: str):
+    def __init__(self, framework: str, extra_options: Optional[List[str]] = None):
         self.framework = framework
+        self.extra_options = list(extra_options or [])
         self.optimes = []
         self._stack = []
 
@@ -610,7 +621,7 @@ class PerfRecorder:
                     "value": duration,
                     "lowerIsBetter": True,
                     "shouldAlert": False,
-                    "extraOptions": [instance_type],
+                    "extraOptions": [instance_type, *self.extra_options],
                     "subtests": [],
                 }
             )
@@ -673,6 +684,176 @@ def _clean_git_checkout(destination_path):
     print_line(b"vcs", b"successfully cleaned git checkout!\n")
 
 
+def _git_config(repo_path: str, key: str) -> Optional[str]:
+    p = subprocess.run(
+        ["git", "config", "--get", key],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        cwd=repo_path,
+        env=os.environ,
+        encoding="utf-8",
+    )
+    if p.returncode == 1:
+        return None
+    if p.returncode != 0:
+        raise RuntimeError(f"git config --get {key} failed: {p.stderr.strip()}")
+    return p.stdout.strip()
+
+
+def _git_lines(repo_path: str, *args: str) -> List[str]:
+    return subprocess.check_output(
+        ["git", *args], cwd=repo_path, env=os.environ, encoding="utf-8"
+    ).splitlines()
+
+
+_PATTERN_METACHARS_RE = re.compile(r"([*?\[\\])")
+
+
+def _escape_pattern(path: str) -> str:
+    escaped = _PATTERN_METACHARS_RE.sub(r"\\\1", path)
+    if escaped.endswith(" "):
+        escaped = escaped[:-1] + "\\ "
+    return escaped
+
+
+def _cone_ancestor_files(
+    repo_path: str, rev: str, directories: List[str]
+) -> List[str]:
+    patterns = []
+    seen = set()
+    for directory in directories:
+        parts = directory.split("/")
+        for depth in range(len(parts)):
+            ancestor = "/".join(parts[:depth])
+            if ancestor in seen:
+                continue
+            seen.add(ancestor)
+            args = ["ls-tree", "-z", rev]
+            if ancestor:
+                args.append(f"{ancestor}/")
+            out = subprocess.check_output(
+                ["git", *args], cwd=repo_path, env=os.environ, encoding="utf-8"
+            )
+            for entry in out.split("\0"):
+                if not entry:
+                    continue
+                meta, path = entry.split("\t", 1)
+                if meta.split(" ")[1] == "blob" and "\n" not in path and "\r" not in path:
+                    patterns.append(f"/{_escape_pattern(path)}")
+    return patterns
+
+
+def _cone_directories(patterns: List[str]) -> Optional[List[str]]:
+    directories = []
+    for pattern in patterns:
+        body = pattern[1:].rstrip("/")
+        if not pattern.startswith("/") or not body or _PATTERN_METACHARS_RE.search(body):
+            return None
+        directories.append(body)
+    return directories
+
+
+class SparsePlan(NamedTuple):
+    """The one `git sparse-checkout` command a checkout needs, if any."""
+
+    cone: bool
+    op: Optional[Literal["sparse_set", "sparse_add", "sparse_convert", "sparse_disable"]]
+    args: List[str]
+    entries: List[str]
+    message: str
+
+
+def _plan_sparse_checkout(
+    repo_path: str,
+    has_repo: bool,
+    is_sparse: bool,
+    sparse_profile: Optional[str],
+    sparse_patterns: Optional[List[str]],
+    rev: str,
+) -> SparsePlan:
+    profile = sparse_profile or ""
+    if not sparse_patterns:
+        if is_sparse:
+            return SparsePlan(
+                False,
+                "sparse_disable",
+                ["git", "sparse-checkout", "disable"],
+                [],
+                "widening the sparse checkout to the full tree",
+            )
+        return SparsePlan(False, None, [], [], "")
+
+    if has_repo and not is_sparse:
+        message = (
+            f"the cached checkout is a full checkout, sparse profile "
+            f"{profile} is not applied"
+        )
+        return SparsePlan(False, None, [], [], message)
+
+    cone_directories = _cone_directories(sparse_patterns)
+    if not has_repo:
+        if cone_directories is not None:
+            args = [
+                "git",
+                "sparse-checkout",
+                "set",
+                "--cone",
+                "--sparse-index",
+                "--stdin",
+            ]
+            cone, entries = True, cone_directories
+        else:
+            args = ["git", "sparse-checkout", "set", "--no-cone", "--stdin"]
+            cone, entries = False, sparse_patterns
+        op = "sparse_set"
+    else:
+        is_cone = _git_config(repo_path, "core.sparseCheckoutCone") == "true"
+        current = _git_lines(repo_path, "sparse-checkout", "list")
+        # Newly checked out patterns are compatible with a `cone` sparse
+        # checkout and the existing sparse checkout is `cone`: simply add the
+        # new directories to it.
+        if cone_directories is not None and is_cone:
+            args = ["git", "sparse-checkout", "add", "--stdin"]
+            cone, op = True, "sparse_add"
+            entries = [d for d in cone_directories if d not in set(current)]
+        # Newly checked out patterns are not compatible with a `cone` sparse
+        # checkout and the existing sparse checkout is `cone`: perform a new
+        # sparse checkout with `cone` disabled, including the existing
+        # directories, the parent directory files that `cone` mode included
+        # implicitly, and the new patterns.
+        elif is_cone:
+            args = [
+                "git",
+                "sparse-checkout",
+                "set",
+                "--no-cone",
+                "--no-sparse-index",
+                "--stdin",
+            ]
+            cone, op = False, "sparse_convert"
+            entries = list(
+                dict.fromkeys(
+                    [f"/{_escape_pattern(d)}" for d in current]
+                    + _cone_ancestor_files(repo_path, rev, current)
+                    + sparse_patterns
+                )
+            )
+        # Newly checked out patterns are not compatible with a `cone` sparse
+        # checkout and the existing sparse checkout is not `cone`: simply add the
+        # new patterns to it.
+        else:
+            args = ["git", "sparse-checkout", "add", "--stdin"]
+            cone, op = False, "sparse_add"
+            entries = [p for p in sparse_patterns if p not in set(current)]
+        if not entries:
+            return SparsePlan(
+                cone, None, [], [], f"sparse profile {profile} already applied"
+            )
+
+    message = f"{op}: sparse profile {profile}, {len(entries)} entries"
+    return SparsePlan(cone, op, args, entries, message)
+
+
 def shortref(ref: str) -> str:
     """Normalize a git ref to its short form.
 
@@ -696,6 +877,8 @@ def git_checkout(
     ssh_key_file: Optional[Path],
     ssh_known_hosts_file: Optional[Path],
     shallow: bool = False,
+    sparse_profile: Optional[str] = None,
+    sparse_patterns: Optional[List[str]] = None,
 ):
     assert head_ref or head_rev
 
@@ -725,20 +908,32 @@ def git_checkout(
         )
 
     has_repo = os.path.exists(destination_path)
-    perf = PerfRecorder("vcs")
-    with perf.record("overall", "overall_pull" if has_repo else "overall_clone"):
-        # Bypass Git's "safe directory" feature as the destination could be
-        # coming from a cache and therefore cloned by a different user.
-        args = [
-            "git",
-            "config",
-            "--global",
-            "--add",
-            "safe.directory",
-            Path(destination_path).as_posix(),
-        ]
-        retry_required_command(b"vcs", args, extra_env=env)
 
+    # Bypass Git's "safe directory" feature as the destination could be
+    # coming from a cache and therefore cloned by a different user.
+    args = [
+        "git",
+        "config",
+        "--global",
+        "--add",
+        "safe.directory",
+        Path(destination_path).as_posix(),
+    ]
+    retry_required_command(b"vcs", args, extra_env=env)
+
+    workers = os.environ.get("RUN_TASK_GIT_CHECKOUT_WORKERS", "0")
+    args = ["git", "config", "--global", "checkout.workers", workers]
+    retry_required_command(b"vcs", args, extra_env=env)
+
+    is_sparse = (
+        has_repo and _git_config(destination_path, "core.sparseCheckout") == "true"
+    )
+    sparse = bool(sparse_patterns) and (not has_repo or is_sparse)
+
+    action = "pull" if has_repo else "clone"
+    mode = "sparsecheckout" if sparse else "fullcheckout"
+    perf = PerfRecorder("vcs")
+    with perf.record("overall", f"overall_{action}", f"overall_{action}_{mode}"):
         if not has_repo:
             # Repository doesn't already exist, needs to be cloned
             args = [
@@ -747,7 +942,13 @@ def git_checkout(
             ]
 
             if shallow:
-                args.extend(["--depth=1", "--no-checkout"])
+                args.append("--depth=1")
+
+            if sparse:
+                args.append("--filter=blob:none")
+
+            if shallow or sparse:
+                args.append("--no-checkout")
 
             args.extend(
                 [
@@ -758,6 +959,11 @@ def git_checkout(
 
             with perf.record("clone"):
                 retry_required_command(b"vcs", args, extra_env=env)
+        else:
+            # Fetches go through origin by name, so a cached clone has to
+            # point origin at the repository this task wants.
+            args = ["git", "remote", "set-url", "origin", base_repo or head_repo]
+            run_required_command(b"vcs", args, cwd=destination_path)
 
         # For Github based repos, base_rev often doesn't refer to an ancestor of
         # head_rev simply due to Github not providing that information in their
@@ -786,16 +992,42 @@ def git_checkout(
             # shallow clone, head_rev needs to be fetched independently regardless.
             targets.append(head_rev)
 
+        # A partial clone keeps its blob filter on the origin remote, and a
+        # fetch only inherits the filter when it names that remote. Fetching
+        # the same repository by URL would download every blob of the commit.
+        remote = "origin" if head_repo == (base_repo or head_repo) else head_repo
+
         with perf.record("pull"):
             git_fetch(
                 destination_path,
                 *targets,
-                remote=head_repo,
+                remote=remote,
                 tags=tags,
                 shallow=shallow,
                 env=env,
             )
 
+        # `git fetch` set `FETCH_HEAD` reference to the last commit of the desired branch
+        rev = head_rev if head_rev else "FETCH_HEAD"
+
+        plan = _plan_sparse_checkout(
+            destination_path, has_repo, is_sparse, sparse_profile, sparse_patterns, rev
+        )
+        if sparse:
+            perf.extra_options.append("cone" if plan.cone else "no-cone")
+            if sparse_profile:
+                perf.extra_options.append(os.path.basename(sparse_profile))
+        if plan.message:
+            print_line(b"vcs", f"{plan.message}\n".encode("utf-8"))
+        if plan.op:
+            stdin_data = None
+            if plan.entries:
+                stdin_data = ("\n".join(plan.entries) + "\n").encode("utf-8")
+            with perf.record(plan.op):
+                run_required_command(
+                    b"vcs", plan.args, cwd=destination_path, stdin_data=stdin_data
+                )
+
         args = [
             "git",
             "checkout",
@@ -805,8 +1037,7 @@ def git_checkout(
         if head_ref:
             args.extend(["-B", shortref(head_ref)])
 
-        # `git fetch` set `FETCH_HEAD` reference to the last commit of the desired branch
-        args.append(head_rev if head_rev else "FETCH_HEAD")
+        args.append(rev)
 
         with perf.record("update"):
             run_required_command(b"vcs", args, cwd=destination_path)
@@ -893,6 +1124,7 @@ def hg_checkout(
     store_path: str,
     branch: Optional[str],
     revision: Optional[str],
+    sparse_profile: Optional[str] = None,
 ):
     if IS_MACOSX or IS_POSIX:
         hg_bin = "hg"
@@ -916,6 +1148,9 @@ def hg_checkout(
     if base_repo:
         args.extend(["--upstream", base_repo])
 
+    if sparse_profile:
+        args.extend(["--sparseprofile", sparse_profile])
+
     # Specify method to checkout a revision. This defaults to revisions as
     # SHA-1 strings, but also supports symbolic revisions like `tip` via the
     # branch flag.
@@ -983,13 +1218,35 @@ def add_vcs_arguments(parser, project, name):
         action="store_true",
         help=f"Use shallow clone for {name}",
     )
+    parser.add_argument(
+        f"--{project}-sparse-profile",
+        help=f"Path to sparse profile for {name} checkout. For git checkouts "
+        f"the patterns come from the {project.upper()}_SPARSE_PATTERNS "
+        "environment variable, a JSON list of git sparse-checkout patterns.",
+    )
 
 
 def collect_vcs_options(args, project, name):
     checkout = getattr(args, f"{project}_checkout")
     shallow_clone = getattr(args, f"{project}_shallow_clone")
+    sparse_profile = getattr(args, f"{project}_sparse_profile", None)
 
     env_prefix = project.upper()
+    sparse_patterns = None
+    if f"{env_prefix}_SPARSE_PATTERNS" in os.environ:
+        try:
+            sparse_patterns = json.loads(os.environ[f"{env_prefix}_SPARSE_PATTERNS"])
+        except ValueError as e:
+            raise RuntimeError(
+                f"{env_prefix}_SPARSE_PATTERNS is not a JSON list of patterns: {e}"
+            ) from None
+        if not isinstance(sparse_patterns, list) or not all(
+            isinstance(p, str) and p.strip() for p in sparse_patterns
+        ):
+            raise RuntimeError(
+                f"{env_prefix}_SPARSE_PATTERNS must be a JSON list of non-empty "
+                "pattern strings"
+            )
 
     repo_type = os.environ.get(f"{env_prefix}_REPOSITORY_TYPE")
     base_repo = os.environ.get(f"{env_prefix}_BASE_REPOSITORY")
@@ -1032,6 +1289,8 @@ def collect_vcs_options(args, project, name):
         "ssh-secret-name": private_key_secret,
         "pip-requirements": pip_requirements,
         "shallow-clone": shallow_clone,
+        "sparse-profile": sparse_profile,
+        "sparse-patterns": sparse_patterns,
     }
 
 
@@ -1040,6 +1299,10 @@ def vcs_checkout_from_args(options):
         if options["head-ref"] and not options["head-rev"]:
             print("task should be defined in terms of non-symbolic revision")
             sys.exit(1)
+        if options["sparse-profile"] or options["sparse-patterns"] is not None:
+            raise RuntimeError(
+                f"--{options['project']}-sparse-profile needs --{options['project']}-checkout"
+            )
         return
 
     head_ref = options["head-ref"]
@@ -1071,6 +1334,17 @@ def vcs_checkout_from_args(options):
             if not head_ref:
                 print("Providing a ref will improve the performance of this checkout")
 
+            if bool(options["sparse-profile"]) != (options["sparse-patterns"] is not None):
+                raise RuntimeError(
+                    f"--{options['project']}-sparse-profile and the "
+                    f"{options['env-prefix']}_SPARSE_PATTERNS environment variable "
+                    "must be given together for a git checkout"
+                )
+            if options["sparse-patterns"] is not None and not options["sparse-patterns"]:
+                raise RuntimeError(
+                    f"{options['env-prefix']}_SPARSE_PATTERNS holds no patterns"
+                )
+
             revision = git_checkout(
                 options["checkout"],
                 options["head-repo"],
@@ -1081,6 +1355,8 @@ def vcs_checkout_from_args(options):
                 ssh_key_file,
                 ssh_known_hosts_file,
                 shallow=options.get("shallow-clone", False),
+                sparse_profile=options["sparse-profile"],
+                sparse_patterns=options["sparse-patterns"],
             )
         elif options["repo-type"] == "hg":
             revision = hg_checkout(
@@ -1090,6 +1366,7 @@ def vcs_checkout_from_args(options):
                 options["store-path"],
                 head_ref,
                 head_rev,
+                sparse_profile=options["sparse-profile"],
             )
         else:
             raise RuntimeError('Type of VCS must be either "git" or "hg"')

taskgraph/transforms/task.py

@@ -870,6 +870,8 @@ def add_generic_index_routes(config, task):
 
 @transforms.add
 def process_treeherder_metadata(config, tasks):
+    group_names = config.graph_config.get("treeherder", {}).get("group-names", {})
+
     for task in tasks:
         routes = task.get("routes", [])
         extra = task.get("extra", {})
@@ -891,7 +893,6 @@ def process_treeherder_metadata(config, tasks):
             treeherder["machine"] = {"platform": machine_platform}
             treeherder["collection"] = {collection: True}
 
-            group_names = config.graph_config["treeherder"]["group-names"]
             groupSymbol, symbol = split_symbol(merged_th["symbol"])
             if groupSymbol != "?":
                 treeherder["groupSymbol"] = groupSymbol