cmem-plugin-git 1.0.0__py3-none-any.whl

cmem_plugin_git/remote.py

@@ -0,0 +1,774 @@
+"""Object level access to a remote git repository
+
+The tasks in this package never create a working tree. They fetch the commit and
+the trees of a revision, walk them, and read only the blobs they actually need;
+an upload builds a tree and a commit object on the fetched tip and sends that.
+"""
+
+import posixpath
+import re
+import time
+from collections.abc import Callable, Iterator
+from collections.abc import Set as AbstractSet
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from fnmatch import fnmatch
+from io import BytesIO
+from urllib.parse import urlparse
+
+from dulwich.client import GitClient, HTTPUnauthorized, SendPackResult, get_transport_and_path
+from dulwich.errors import GitProtocolError, HangupException
+from dulwich.ignore import IgnoreFilter, read_ignore_patterns
+from dulwich.index import commit_tree
+from dulwich.objects import Blob, Commit, ObjectID, ShaFile, Tree
+from dulwich.pack import UnpackedObject, pack_objects_to_data
+from dulwich.refs import Ref
+from dulwich.repo import MemoryRepo
+
+DEFAULT_USERNAME = "gitlab-ci-token"
+"""Basic auth user name sent along with an access token.
+
+GitHub ignores the user name when the password is a token, while GitLab checks
+it and answers a mismatch with 401 or 500 rather than with an explanation. This
+value is the one GitLab accepts for a repository token, so it is the default
+that reaches both.
+"""
+
+DEFAULT_AUTHOR = "eccenca Corporate Memory <cmem@eccenca.com>"
+"""Commit identity used when the executing user cannot be determined."""
+
+DEFAULT_MESSAGE = "Update files from an eccenca Corporate Memory workflow"
+"""Commit message an upload uses when it is not given one."""
+
+LFS_POINTER_PREFIX = b"version https://git-lfs.github.com/spec/v1"
+MAX_SYMLINK_HOPS = 10
+MODE_TREE = 0o40000
+MODE_SYMLINK = 0o120000
+MODE_SUBMODULE = 0o160000
+BRANCH_PREFIX = "refs/heads/"
+TAG_PREFIX = "refs/tags/"
+COMMIT_ID_PATTERN = re.compile(r"^[0-9a-f]{40}$")
+WEB_URL_MARKERS = ("/-/", "/tree/", "/blob/", "/raw/")
+MIN_REPOSITORY_DEPTH = 2
+"""A repository path holds at least a namespace and a project name."""
+
+
+class PushRejectedError(RuntimeError):
+    """Raised when the remote branch moved between reading and writing it."""
+
+
+@dataclass(frozen=True)
+class RemoteRef:
+    """A branch or a tag, as advertised by the remote."""
+
+    name: str
+    kind: str
+    commit_id: str
+
+
+@dataclass(frozen=True)
+class ResolvedRef:
+    """The revision a task works on."""
+
+    commit_id: str
+    name: str
+    is_branch: bool
+
+
+@dataclass(frozen=True)
+class RepositoryFile:
+    """A file of the repository at one revision.
+
+    For a symbolic link, ``path`` and ``name`` are the link's own, while
+    ``size``, ``mode`` and ``blob_id`` are those of the file it points at.
+
+    ``size`` is None whenever the content has not been transferred. A git tree
+    records a name, a mode and an object id but not a length, so the size of a
+    file is only known once its content has been fetched - which is exactly what
+    the metadata fetch avoids.
+    """
+
+    path: str
+    name: str
+    size: int | None
+    mode: str
+    blob_id: str
+    commit_id: str
+
+
+@dataclass
+class Selection:
+    """The outcome of applying folder, expression and subfolder options."""
+
+    files: list[RepositoryFile] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+
+
+@dataclass
+class UploadRequest:
+    """Everything an upload needs to know, so that no function grows an argument list."""
+
+    branch: str
+    directory: str
+    contents: dict[str, bytes]
+    message: str
+    author: str
+    remove_obsolete: bool = False
+    honor_gitignore: bool = False
+
+
+@dataclass
+class UploadResult:
+    """What an upload did."""
+
+    commit_id: str | None = None
+    branch: str = ""
+    branch_created: bool = False
+    written: list[str] = field(default_factory=list)
+    removed: list[str] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+
+
+def normalize_url(url: str) -> str:
+    """Turn what a user pasted into a URL the git client can use
+
+    Accepts a clone URL as well as the address of the repository's web page,
+    with or without a trailing ``.git``.
+    """
+    candidate = url.strip()
+    if not candidate:
+        raise ValueError("The repository URL is empty.")
+    parsed = urlparse(candidate)
+    if parsed.scheme not in {"http", "https"}:
+        raise ValueError(
+            f"'{candidate}' is not an HTTP(S) repository URL. "
+            "These tasks talk to a repository over HTTP(S) only, so an SSH address such as "
+            "git@example.org:group/project.git cannot be used."
+        )
+    path = _without_web_suffix(parsed.path)
+    return f"{parsed.scheme}://{parsed.netloc}{path.rstrip('/')}"
+
+
+def _without_web_suffix(path: str) -> str:
+    """Cut what a forge appends to a repository path when it shows a page
+
+    A marker only counts once a namespace and a project name precede it. Without
+    that check a repository living in a group called 'raw' or 'tree' would be cut
+    down to the bare server address.
+    """
+    segments = [segment for segment in path.split("/") if segment]
+    for index in range(MIN_REPOSITORY_DEPTH, len(segments)):
+        if f"/{segments[index]}/" in WEB_URL_MARKERS:
+            return "/" + "/".join(segments[:index])
+    return path
+
+
+def _oid(value: str) -> ObjectID:
+    """Turn a hexadecimal object id into what dulwich expects."""
+    return ObjectID(value.encode())
+
+
+def _ref(name: str) -> Ref:
+    """Turn a fully qualified reference name into what dulwich expects."""
+    return Ref(name.encode())
+
+
+def _wants(objects: list[ObjectID]) -> Callable[..., list[ObjectID]]:
+    """Build a determine_wants callback asking for exactly these objects
+
+    The callback is invoked with the depth positionally by one transport and by
+    keyword by another, so it accepts whatever it is handed.
+    """
+
+    def determine(_refs: dict[Ref, ObjectID], *_args: object, **_kwargs: object) -> list[ObjectID]:
+        return objects
+
+    return determine
+
+
+def _lfs_patterns(gitattributes: bytes) -> list[str]:
+    """Read the path patterns that .gitattributes hands to the LFS filter."""
+    patterns = []
+    for raw_line in gitattributes.splitlines():
+        line = raw_line.decode("utf-8", errors="replace").strip()
+        if not line or line.startswith("#"):
+            continue
+        fields = line.split()
+        if len(fields) > 1 and "filter=lfs" in fields[1:]:
+            patterns.append(fields[0])
+    return patterns
+
+
+def _matches_pattern(path: str, pattern: str) -> bool:
+    """Check a repository path against a .gitattributes pattern."""
+    if "/" in pattern.rstrip("/"):
+        return fnmatch(path, pattern.lstrip("/"))
+    return fnmatch(posixpath.basename(path), pattern)
+
+
+def _blob_of(store: MemoryRepo, sha: ObjectID) -> Blob:
+    """Read one object as a blob."""
+    obj = store[sha]
+    if not isinstance(obj, Blob):
+        raise TypeError(f"Object {sha.decode()} is a {obj.type_name.decode()}, not a file.")
+    return obj
+
+
+def _tree_of(store: MemoryRepo, sha: ObjectID) -> Tree:
+    """Read one object as a tree."""
+    obj = store[sha]
+    if not isinstance(obj, Tree):
+        raise TypeError(f"Object {sha.decode()} is a {obj.type_name.decode()}, not a folder.")
+    return obj
+
+
+def _commit_of(store: MemoryRepo, sha: ObjectID) -> Commit:
+    """Read one object as a commit."""
+    obj = store[sha]
+    if not isinstance(obj, Commit):
+        raise TypeError(f"Object {sha.decode()} is a {obj.type_name.decode()}, not a commit.")
+    return obj
+
+
+def _holds(store: MemoryRepo, sha: ObjectID) -> bool:
+    """Check whether an object has already been fetched."""
+    try:
+        store[sha]
+    except KeyError:
+        return False
+    return True
+
+
+class GitRemote:
+    """A remote git repository, reachable over HTTP(S)"""
+
+    def __init__(self, url: str, token: str = "", username: str = DEFAULT_USERNAME) -> None:
+        self.url = normalize_url(url)
+        self.username = username.strip() or DEFAULT_USERNAME
+        self.token = token
+        if token:
+            client, path = get_transport_and_path(self.url, username=self.username, password=token)
+        else:
+            client, path = get_transport_and_path(self.url)
+        self.client: GitClient = client
+        self.path: bytes = path.encode() if isinstance(path, str) else path
+        self._advertised: dict[Ref, ObjectID] | None = None
+        self._symrefs: dict[Ref, Ref] = {}
+
+    @contextmanager
+    def _explaining(self, action: str) -> Iterator[None]:
+        """Turn a transport failure into something a workflow designer can act on
+
+        A refused credential does not always arrive as 401. Some servers answer
+        the reference advertisement with a 500 when the user name does not suit
+        the token, and dulwich passes that on as the bare status code.
+        """
+        try:
+            yield
+        except HTTPUnauthorized as error:
+            raise ValueError(
+                f"{self.url} refused these credentials while {action}. Check the access token, "
+                "and that it carries the right to read - or, for an upload, to write - this "
+                "repository."
+            ) from error
+        except GitProtocolError as error:
+            raise ValueError(
+                f"{self.url} did not answer while {action}: {error}. When the status is 401 or "
+                "500, the user name is worth a look: GitLab checks it against the token and "
+                f"accepts '{DEFAULT_USERNAME}', while GitHub ignores it. The user name "
+                f"currently sent is '{self.username}'."
+            ) from error
+
+    def _refs(self) -> dict[Ref, ObjectID]:
+        """Ask the remote what it advertises, once per instance."""
+        if self._advertised is None:
+            with self._explaining("listing its branches and tags"):
+                result = self.client.get_refs(self.path)
+            self._advertised = {name: sha for name, sha in result.refs.items() if sha is not None}
+            self._symrefs = dict(result.symrefs)
+        return self._advertised
+
+    def list_refs(self) -> list[RemoteRef]:
+        """List the branches and tags the remote advertises."""
+        refs = []
+        for name, commit_id in self._refs().items():
+            decoded = name.decode()
+            if decoded.endswith("^{}"):
+                continue
+            if decoded.startswith(BRANCH_PREFIX):
+                refs.append(RemoteRef(decoded[len(BRANCH_PREFIX) :], "branch", commit_id.decode()))
+            elif decoded.startswith(TAG_PREFIX):
+                refs.append(RemoteRef(decoded[len(TAG_PREFIX) :], "tag", commit_id.decode()))
+        return sorted(refs, key=lambda ref: (ref.kind, ref.name))
+
+    def default_branch(self) -> str:
+        """Name the branch the remote's HEAD points at."""
+        self._refs()
+        head = self._symrefs.get(Ref(b"HEAD"), Ref(b"")).decode()
+        if head.startswith(BRANCH_PREFIX):
+            return head[len(BRANCH_PREFIX) :]
+        branches = [ref.name for ref in self.list_refs() if ref.kind == "branch"]
+        if not branches:
+            raise ValueError(f"The repository at {self.url} advertises no branch.")
+        if len(branches) > 1:
+            raise ValueError(
+                f"The repository at {self.url} does not say which of its branches is the "
+                f"default one, so the revision has to be named. It offers: "
+                f"{', '.join(branches)}."
+            )
+        return branches[0]
+
+    def resolve(self, ref: str) -> ResolvedRef:
+        """Turn a branch name, tag name or commit id into the revision to read."""
+        wanted = ref.strip()
+        if not wanted:
+            wanted = self.default_branch()
+        advertised = self._refs()
+        branch = advertised.get(_ref(f"{BRANCH_PREFIX}{wanted}"))
+        if branch is not None:
+            return ResolvedRef(branch.decode(), wanted, is_branch=True)
+        tag = advertised.get(_ref(f"{TAG_PREFIX}{wanted}"))
+        if tag is not None:
+            peeled = advertised.get(_ref(f"{TAG_PREFIX}{wanted}^{{}}"), tag)
+            return ResolvedRef(peeled.decode(), wanted, is_branch=False)
+        if COMMIT_ID_PATTERN.match(wanted):
+            return ResolvedRef(wanted, wanted, is_branch=False)
+        known = ", ".join(sorted(reference.name for reference in self.list_refs())) or "none"
+        raise ValueError(
+            f"'{wanted}' is neither a branch, a tag nor a commit id of {self.url}. "
+            f"The repository advertises: {known}."
+        )
+
+    def fetch_objects(
+        self,
+        target: MemoryRepo,
+        objects: list[ObjectID],
+        filter_spec: bytes | None = None,
+        depth: int | None = 1,
+    ) -> None:
+        """Fetch objects into a store, retrying without the filter when it is refused."""
+        with self._explaining("fetching from it"):
+            try:
+                self.client.fetch(
+                    self.path,
+                    target,
+                    determine_wants=_wants(objects),
+                    depth=depth,
+                    filter_spec=filter_spec,
+                    protocol_version=2,
+                )
+            except (GitProtocolError, HangupException):
+                if filter_spec is None:
+                    raise
+                self.client.fetch(self.path, target, determine_wants=_wants(objects), depth=depth)
+
+    def snapshot(self, ref: str) -> "Snapshot":
+        """Fetch the commit and the trees of a revision, without any file content."""
+        resolved = self.resolve(ref)
+        store = MemoryRepo()
+        self.fetch_objects(store, [_oid(resolved.commit_id)], filter_spec=b"blob:none")
+        return Snapshot(remote=self, store=store, revision=resolved)
+
+    def fetch_blobs(self, blob_ids: list[str]) -> dict[str, bytes]:
+        """Fetch the given blobs and nothing else
+
+        The objects go into an empty store on purpose: a store that already holds
+        the commit would make the client claim it has everything reachable from it,
+        and the remote would answer with an empty pack.
+        """
+        if not blob_ids:
+            return {}
+        store = MemoryRepo()
+        with self._explaining("reading file content from it"):
+            self.client.fetch(
+                self.path,
+                store,
+                determine_wants=_wants([_oid(blob_id) for blob_id in blob_ids]),
+                protocol_version=2,
+            )
+        return {blob_id: _blob_of(store, _oid(blob_id)).data for blob_id in blob_ids}
+
+    def check_write_access(self) -> None:
+        """Ask the remote for a write handshake and hang up again
+
+        The advertisement of git-receive-pack is only served to somebody who may
+        write, so reaching it is the check. Returning no reference update makes
+        the client stop before it sends anything.
+        """
+
+        def nothing(_refs: dict[Ref, ObjectID]) -> dict[Ref, ObjectID]:
+            return {}
+
+        def no_pack(
+            have: AbstractSet[ObjectID],
+            want: AbstractSet[ObjectID],
+            *,
+            ofs_delta: bool = True,
+            progress: Callable[[bytes], None] | None = None,
+        ) -> tuple[int, Iterator[UnpackedObject]]:
+            _ = (have, want, ofs_delta, progress)
+            return 0, iter([])
+
+        with self._explaining("asking whether it may be written to"):
+            self.client.send_pack(self.path, nothing, no_pack)
+
+    def upload(self, request: UploadRequest) -> UploadResult:
+        """Commit the given contents onto a branch and push it, once retrying a moved branch
+
+        Re-applying the same contents to the new tip is safe because an upload
+        sets paths to contents rather than replaying a difference.
+        """
+        try:
+            return self._build_and_push(request)
+        except PushRejectedError:
+            return self._build_and_push(request)
+
+    def _base_of(self, branch: str) -> tuple[str | None, bool]:
+        """Find the commit an upload builds on, and whether the branch is new."""
+        advertised = self._refs()
+        tip = advertised.get(_ref(f"{BRANCH_PREFIX}{branch}"))
+        if tip is not None:
+            return tip.decode(), False
+        if not advertised:
+            return None, True
+        return self.resolve(self.default_branch()).commit_id, True
+
+    def _build_and_push(self, request: UploadRequest) -> UploadResult:
+        """Assemble the new tree, create the commit object and send the pack."""
+        self._advertised = None
+        base_commit_id, branch_created = self._base_of(request.branch)
+        store = MemoryRepo()
+        entries: dict[str, tuple[int, ObjectID]] = {}
+        if base_commit_id is not None:
+            self.fetch_objects(store, [_oid(base_commit_id)], filter_spec=b"blob:none")
+            entries = _flatten(store, _commit_of(store, _oid(base_commit_id)).tree)
+        result = UploadResult(branch=request.branch, branch_created=branch_created)
+        known = set(store.object_store)
+        new_entries = _apply(store, entries, request, result)
+        tree_id = commit_tree(
+            store.object_store,
+            [(path.encode(), sha, mode) for path, (mode, sha) in sorted(new_entries.items())],
+        )
+        base_tree = _commit_of(store, _oid(base_commit_id)).tree if base_commit_id else None
+        if tree_id == base_tree:
+            result.warnings.append(
+                "The repository already holds exactly these files, so nothing was committed."
+            )
+            return result
+        commit = _commit_object(tree_id, base_commit_id, request)
+        store.object_store.add_object(commit)
+        created = [store[ObjectID(sha)] for sha in set(store.object_store) - known]
+        self._send(
+            request.branch,
+            commit.id,
+            None if branch_created else base_commit_id,
+            created,
+        )
+        result.commit_id = commit.id.decode()
+        return result
+
+    def _send(
+        self,
+        branch: str,
+        commit_id: ObjectID,
+        expected_tip: str | None,
+        objects: list[ShaFile],
+    ) -> None:
+        """Push one branch, refusing to overwrite work that arrived meanwhile
+
+        The protocol itself would happily replace the branch, so the check that
+        the branch still points where it did when the commit was built happens
+        here. ``expected_tip`` is None for a branch that did not exist yet, which
+        also catches somebody else creating it in the meantime.
+
+        Only the objects this upload created are packed. Everything else the new
+        commit refers to is already on the remote, and asking the local store to
+        work that out would need the blobs the metadata fetch deliberately left
+        behind.
+        """
+        ref = _ref(f"{BRANCH_PREFIX}{branch}")
+        expected = _oid(expected_tip) if expected_tip else None
+
+        def update_refs(refs: dict[Ref, ObjectID]) -> dict[Ref, ObjectID]:
+            if refs.get(ref) != expected:
+                raise PushRejectedError(
+                    f"The branch '{branch}' moved while this task was working on it."
+                )
+            return {ref: commit_id}
+
+        def generate_pack_data(
+            have: AbstractSet[ObjectID],
+            want: AbstractSet[ObjectID],
+            *,
+            ofs_delta: bool = True,
+            progress: Callable[[bytes], None] | None = None,
+        ) -> tuple[int, Iterator[UnpackedObject]]:
+            # The negotiation arguments are not consulted: what to send was
+            # decided while the commit was built. The names are dictated by the
+            # callback signature dulwich expects.
+            _ = (have, want, progress)
+            return pack_objects_to_data(objects, ofs_delta=ofs_delta)
+
+        with self._explaining("pushing to it"):
+            result = self.client.send_pack(self.path, update_refs, generate_pack_data)
+        _check_accepted(result, ref, branch)
+
+
+def _check_accepted(result: SendPackResult, ref: Ref, branch: str) -> None:
+    """Fail unless the remote actually took the branch update
+
+    A remote that refuses one reference - a protected branch, a hook that
+    declines, a rule about who may push where - answers with a per reference
+    status rather than with an error, so a push that was thrown away looks like a
+    successful one until this is read.
+    """
+    status = (result.ref_status or {}).get(ref)
+    if status is None:
+        return
+    if "fast forward" in status or "fast-forward" in status:
+        raise PushRejectedError(f"The remote refused to move '{branch}': {status}")
+    raise ValueError(
+        f"The remote refused to update the branch '{branch}': {status}. Nothing was committed. "
+        "A branch that is protected, or a hook that declines the push, reports itself this way."
+    )
+
+
+def _flatten(
+    store: MemoryRepo, tree_id: ObjectID, prefix: str = ""
+) -> dict[str, tuple[int, ObjectID]]:
+    """Turn a tree and its subtrees into a flat path to (mode, object id) mapping."""
+    entries: dict[str, tuple[int, ObjectID]] = {}
+    for entry in _tree_of(store, tree_id).items():
+        path = posixpath.join(prefix, entry.path.decode())
+        if entry.mode == MODE_TREE:
+            entries.update(_flatten(store, entry.sha, path))
+        else:
+            entries[path] = (entry.mode, entry.sha)
+    return entries
+
+
+def _apply(
+    store: MemoryRepo,
+    entries: dict[str, tuple[int, ObjectID]],
+    request: UploadRequest,
+    result: UploadResult,
+) -> dict[str, tuple[int, ObjectID]]:
+    """Write the incoming files into the flattened tree, and remove what was asked for."""
+    updated: dict[str, tuple[int, ObjectID]] = dict(entries)
+    directory = request.directory.strip("/")
+    ignored = _ignore_filter(store, entries) if request.honor_gitignore else None
+    keep = set()
+    for name, content in sorted(request.contents.items()):
+        path = posixpath.join(directory, name) if directory else name
+        keep.add(path)
+        if ignored is not None and ignored.is_ignored(path):
+            result.warnings.append(f"'{path}' is excluded by .gitignore and was not committed.")
+            continue
+        blob = Blob.from_string(content)
+        store.object_store.add_object(blob)
+        updated[path] = (0o100644, blob.id)
+        result.written.append(path)
+    if request.remove_obsolete:
+        for path in sorted(updated):
+            if path not in keep and posixpath.dirname(path) == directory:
+                del updated[path]
+                result.removed.append(path)
+    return updated
+
+
+def _ignore_filter(
+    store: MemoryRepo, entries: dict[str, tuple[int, ObjectID]]
+) -> IgnoreFilter | None:
+    """Read the repository's root .gitignore, when it has one."""
+    entry = entries.get(".gitignore")
+    if entry is None:
+        return None
+    return IgnoreFilter(read_ignore_patterns(BytesIO(_blob_of(store, entry[1]).data)))
+
+
+def clean_identity_part(value: str) -> str:
+    """Strip what would break out of a commit header
+
+    A commit records its author on one line, framed by angle brackets, and
+    dulwich writes what it is given. A name carrying a newline would end that
+    line early and let the rest be read as further header fields.
+    """
+    return "".join(character for character in value if character not in "\r\n<>").strip()
+
+
+def _commit_object(tree_id: ObjectID, parent: str | None, request: UploadRequest) -> Commit:
+    """Create the commit object an upload pushes."""
+    commit = Commit()
+    commit.tree = tree_id
+    commit.parents = [_oid(parent)] if parent else []
+    identity = request.author.encode()
+    commit.author = identity
+    commit.committer = identity
+    commit.commit_time = commit.author_time = int(time.time())
+    commit.commit_timezone = commit.author_timezone = 0
+    commit.encoding = b"UTF-8"
+    message = request.message.strip() or DEFAULT_MESSAGE
+    commit.message = f"{message}\n".encode()
+    return commit
+
+
+class Snapshot:
+    """One revision of a repository: its trees, and access to its blobs"""
+
+    def __init__(self, remote: GitRemote, store: MemoryRepo, revision: ResolvedRef) -> None:
+        self.remote = remote
+        self.store = store
+        self.revision = revision
+        self._entries: dict[str, tuple[int, ObjectID]] | None = None
+
+    def entries(self) -> dict[str, tuple[int, ObjectID]]:
+        """Map every non directory path of the revision to its mode and object id."""
+        if self._entries is None:
+            commit = _commit_of(self.store, _oid(self.revision.commit_id))
+            self._entries = _flatten(self.store, commit.tree)
+        return self._entries
+
+    def folders(self) -> list[str]:
+        """List the folders of the revision, for parameter autocompletion."""
+        folders = set()
+        for path in self.entries():
+            parts = path.split("/")[:-1]
+            for depth in range(1, len(parts) + 1):
+                folders.add("/".join(parts[:depth]))
+        return sorted(folders)
+
+    def read(self, blob_ids: list[str]) -> dict[str, bytes]:
+        """Read blobs, fetching the ones the metadata fetch did not bring along."""
+        contents: dict[str, bytes] = {}
+        missing: list[str] = []
+        for blob_id in dict.fromkeys(blob_ids):
+            if _holds(self.store, _oid(blob_id)):
+                contents[blob_id] = _blob_of(self.store, _oid(blob_id)).data
+            else:
+                missing.append(blob_id)
+        if not missing:
+            return contents
+        try:
+            contents.update(self.remote.fetch_blobs(missing))
+        except (GitProtocolError, HangupException, KeyError):
+            self.remote.fetch_objects(self.store, [_oid(self.revision.commit_id)])
+            for blob_id in missing:
+                contents[blob_id] = _blob_of(self.store, _oid(blob_id)).data
+        return contents
+
+    def select(self, path: str, regex: str, no_subfolder: bool) -> Selection:
+        """Apply the folder, expression and subfolder options to the revision."""
+        folder = path.strip("/")
+        expression = re.compile(regex) if regex else re.compile("^.*$")
+        selection = Selection()
+        candidates = [
+            (candidate, mode, sha)
+            for candidate, (mode, sha) in sorted(self.entries().items())
+            if _within(candidate, folder, no_subfolder)
+            and expression.search(posixpath.basename(candidate))
+        ]
+        lfs = self._lfs_matcher()
+        targets = self._resolve_symlinks(candidates, selection)
+        for candidate, mode, sha in candidates:
+            resolved = targets.get(candidate) if mode == MODE_SYMLINK else (candidate, mode, sha)
+            if resolved is None:
+                continue
+            target_path, target_mode, target_sha = resolved
+            if target_mode == MODE_SUBMODULE:
+                selection.warnings.append(f"'{candidate}' is a submodule and was skipped.")
+                continue
+            if lfs(target_path):
+                selection.warnings.append(
+                    f"'{candidate}' is stored with Git LFS, so only a pointer is in the "
+                    "repository, and it was skipped."
+                )
+                continue
+            selection.files.append(self._file(candidate, target_mode, target_sha))
+        return selection
+
+    def _file(self, path: str, mode: int, sha: ObjectID) -> RepositoryFile:
+        """Describe one selected file."""
+        return RepositoryFile(
+            path=path,
+            name=posixpath.basename(path),
+            size=self.store[sha].raw_length() if _holds(self.store, sha) else None,
+            mode=f"{mode:o}",
+            blob_id=sha.decode(),
+            commit_id=self.revision.commit_id,
+        )
+
+    def _lfs_matcher(self) -> Callable[[str], bool]:
+        """Build a test for whether a path is tracked by Git LFS."""
+        entry = self.entries().get(".gitattributes")
+        if entry is None:
+            return lambda _path: False
+        content = self.read([entry[1].decode()])[entry[1].decode()]
+        patterns = _lfs_patterns(content)
+        if not patterns:
+            return lambda _path: False
+        return lambda path: any(_matches_pattern(path, pattern) for pattern in patterns)
+
+    def _resolve_symlinks(
+        self, candidates: list[tuple[str, int, ObjectID]], selection: Selection
+    ) -> dict[str, tuple[str, int, ObjectID]]:
+        """Follow the symbolic links among the candidates to the file they point at."""
+        links = [candidate for candidate, mode, _ in candidates if mode == MODE_SYMLINK]
+        if not links:
+            return {}
+        entries = self.entries()
+        contents = self.read([entries[link][1].decode() for link in links])
+        resolved: dict[str, tuple[str, int, ObjectID]] = {}
+        for link in links:
+            target = self._follow(link, entries, contents, selection)
+            if target is not None:
+                resolved[link] = target
+        return resolved
+
+    def _follow(
+        self,
+        link: str,
+        entries: dict[str, tuple[int, ObjectID]],
+        contents: dict[str, bytes],
+        selection: Selection,
+    ) -> tuple[str, int, ObjectID] | None:
+        """Walk one chain of symbolic links, reporting where it does not end well."""
+        current = link
+        for _ in range(MAX_SYMLINK_HOPS):
+            blob_id = entries[current][1].decode()
+            raw = contents.get(blob_id) or self.read([blob_id])[blob_id]
+            contents[blob_id] = raw
+            target = posixpath.normpath(
+                posixpath.join(posixpath.dirname(current), raw.decode().strip())
+            )
+            if target.startswith("..") or posixpath.isabs(target):
+                selection.warnings.append(
+                    f"'{link}' points outside the repository and was skipped."
+                )
+                return None
+            entry = entries.get(target)
+            if entry is None:
+                selection.warnings.append(
+                    f"'{link}' points at '{target}', which the repository does not hold, "
+                    "and was skipped."
+                )
+                return None
+            if entry[0] != MODE_SYMLINK:
+                return (target, entry[0], entry[1])
+            current = target
+        selection.warnings.append(f"'{link}' is a loop of symbolic links and was skipped.")
+        return None
+
+
+def _within(path: str, folder: str, no_subfolder: bool) -> bool:
+    """Check whether a repository path sits in the selected folder."""
+    if folder and not path.startswith(f"{folder}/"):
+        return False
+    remainder = path[len(folder) + 1 :] if folder else path
+    return "/" not in remainder if no_subfolder else True
+
+
+def iter_warnings(warnings: list[str], limit: int = 10) -> Iterator[str]:
+    """Yield at most a handful of warnings, and say how many were left out."""
+    yield from warnings[:limit]
+    if len(warnings) > limit:
+        yield f"{len(warnings) - limit} further messages of this kind were left out."