gitpane 1.0.0__py3-none-any.whl

gitpane/submodule/checkout_refusal.py

@@ -0,0 +1,102 @@
+"""Read Git's refusal to check out a path of a submodule.
+
+Git writes two whole lines when the checkout of a submodule refuses a path: the refused path, then the submodule it
+could not check out. Only those whole lines count, never the same words inside another sentence, so the output of a
+server or of a hook cannot make GitPane believe a checkout was refused.
+
+Git writes the same fatal line for any checkout of a submodule that fails, on a locked index, a name too long for the
+file system or a full disk; such a fatal line counts only after the start of a line that cites a refused path. A failure
+of another kind adds no folder to the refused ones and stays a failure that Retry submodules takes up again.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from dataclasses import dataclass
+
+from gitpane.config import GIT_PATH_SEPARATOR
+from gitpane.system.path_identity import canonical_path
+
+# Git's own two lines when the checkout of a submodule refuses a path: the refused path, then the submodule that could
+# not be checked out. A fatal line counts only after the start of a refused path's line, since Git writes it for any
+# failed checkout of a submodule. Only a whole line counts, so a server's sentence that quotes these words does not. A
+# file name that holds a line break can still make these two lines; it adds at most one folder GitPane leaves unwritten,
+# so nothing is ever written because of them. Git writes the line break of a refused path as it is, which cuts the first
+# line in two: that path gets no sentence of GitPane's, but the start of its line stays at the start of a line, and its
+# submodule is still recognised by the fatal line that follows, which stays whole. The path is matched up to the last
+# quote, since a file name may itself hold a quote.
+INVALID_PATH_START = "error: invalid path '"
+INVALID_PATH_PATTERN = re.compile("^" + re.escape(INVALID_PATH_START) + r"(?P<path>.*)'$")
+SUBMODULE_CHECKOUT_FAILED_PATTERN = re.compile(r"^fatal: Unable to checkout '(?P<oid>[0-9a-f]+)' in submodule path '(?P<submodule>.*)'$")
+
+
+def _git_lines(stderr_text: str) -> list[str]:
+    """Split Git's error output into lines, Windows line ends and the carriage returns of progress lines becoming plain line ends."""
+    return stderr_text.replace("\r\n", "\n").replace("\r", "\n").split("\n")
+
+
+@dataclass(frozen=True)
+class CheckoutRefusal:
+    """Name a submodule Git could not check out and the paths it refused there, in the order Git printed them."""
+
+    submodule_path: str
+    refused_paths: tuple[str, ...]
+
+
+def checkout_refusals(stderr_text: str) -> tuple[CheckoutRefusal, ...]:
+    """Pair every refused path Git printed with the submodule checkout failure that follows it.
+
+    Windows line ends and the carriage returns that end Git's progress lines become plain line ends first, so each line
+    is matched whole on every system. A checkout failure with no refused path before it says nothing about a file name
+    and gives no refusal.
+    """
+    refusals: list[CheckoutRefusal] = []
+    pending_paths: list[str] = []
+    for line in _git_lines(stderr_text):
+        invalid = INVALID_PATH_PATTERN.match(line)
+        if invalid is not None:
+            pending_paths.append(invalid.group("path"))
+            continue
+        failed = SUBMODULE_CHECKOUT_FAILED_PATTERN.match(line)
+        if failed is not None and pending_paths:
+            refusals.append(CheckoutRefusal(failed.group("submodule"), tuple(pending_paths)))
+            pending_paths = []
+    return tuple(refusals)
+
+
+def refused_checkout_submodule_paths(stderr_text: str) -> tuple[str, ...]:
+    """Return the path of every submodule whose checkout Git refused on a file name, in the order Git printed them.
+
+    A submodule counts when a line that starts as Git's refused path line comes before its whole fatal line, even when a
+    line break in the refused path cut that line in two: its submodule must then be left as Git left it. The fatal line
+    alone is not enough, since Git writes it for every checkout of a submodule that fails, on a locked index, a name too
+    long for the file system or a full disk; such a submodule is not refused and stays to be checked out again. Each
+    fatal line ends what was read for its submodule, so a refusal never carries over to the next one.
+    """
+    submodule_paths: list[str] = []
+    refusal_started = False
+    for line in _git_lines(stderr_text):
+        if line.startswith(INVALID_PATH_START):
+            refusal_started = True
+            continue
+        failed = SUBMODULE_CHECKOUT_FAILED_PATTERN.match(line)
+        if failed is None:
+            continue
+        if refusal_started:
+            submodule_paths.append(failed.group("submodule"))
+        refusal_started = False
+    return tuple(submodule_paths)
+
+
+def refused_submodule_dirs(submodule_paths: tuple[str, ...], base_dir: str) -> list[str]:
+    """Return the folders of the submodules whose checkout Git refused, so a later checkout of the others can leave them alone.
+
+    ``submodule_paths`` are the submodules as Git names them in its refusal, by their path from the repository it ran in,
+    nested submodules included, so ``base_dir`` is the folder of that repository. The folders are canonical, to compare
+    with the folders a submodule read gives.
+    """
+    directories = []
+    for submodule_path in submodule_paths:
+        directories.append(canonical_path(os.path.join(base_dir, *submodule_path.split(GIT_PATH_SEPARATOR))))
+    return directories

gitpane/submodule/checkout_refusal_notice.py

@@ -0,0 +1,111 @@
+"""Say in plain words why Git refused to check out a path of a submodule.
+
+A submodule whose checkout Git refused is explained from the lines Git wrote, naming only the paths Git cited, with no
+file name rule of GitPane's own: the rules of Windows are read by Python's ntpath, and only to pick which reason a path
+Git already refused gets.
+"""
+
+from __future__ import annotations
+
+import ntpath
+import unicodedata
+from typing import TYPE_CHECKING
+
+from gitpane.config import CONTROL_CHARACTER_CATEGORY, GIT_PATH_SEPARATOR, WINDOWS_PLATFORM, SharedText
+from gitpane.process.git_output_text import GIT_ERROR_DECODE_ERRORS
+
+if TYPE_CHECKING:
+    from gitpane.submodule.checkout_refusal import CheckoutRefusal
+
+# A control character left in a path Git cited is written as \x and two hexadecimal digits, so it can neither move the
+# console's cursor nor vanish from the text. Every control character of Unicode fits in two digits.
+CONTROL_CHARACTER_ESCAPE_FORMAT = r"\x{code:02x}"
+# A byte that was not UTF-8 in Git's error output is kept as one character from U+DC80 to U+DCFF, which no console or
+# log could write: it is written back as the byte it stands for, in the same \x form.
+UNDECODED_BYTE_FIRST_CODE = 0xDC80
+UNDECODED_BYTE_LAST_CODE = 0xDCFF
+UNDECODED_BYTE_INDEX = 0
+# The folder names Git never writes inside a repository, on every system, compared without case as Git does. git~1 is
+# the short name Windows can give a .git folder.
+PROTECTED_FOLDER_NAME_SET = frozenset({".git", "git~1"})
+# The file names Git refuses to write as a symbolic link, on every system. Git refuses them only as a link, so a refusal
+# of one of these names means a link.
+PROTECTED_LINK_NAME_SET = frozenset({".gitmodules", ".gitattributes", ".gitignore", ".mailmap"})
+# The characters Windows drops from the end of a name, so ".git." and ".git " name the .git folder there; Git compares
+# without them on every system.
+WINDOWS_DROPPED_NAME_END = ". "
+LAST_COMPONENT_INDEX = -1
+# What Git writes in its error lines for each control character of a path, except the tab and the line break.
+GIT_CONTROL_CHARACTER_MARK = "?"
+CHECKOUT_REFUSAL_PART_SEPARATOR = " "
+CHECKOUT_REFUSAL_LINE_SEPARATOR = "\n"
+
+
+def displayed_git_path(path_text: str) -> str:
+    r"""Return a path Git cited as plain text that shows every character, for the console and the forms.
+
+    Each control character, and each byte that was not UTF-8 and stayed undecoded, is written as \x and two hexadecimal
+    digits. Everything else is kept, including a \xNN text the path really holds and the replacement character a
+    repository name may really hold.
+    """
+    characters: list[str] = []
+    for character in path_text:
+        if UNDECODED_BYTE_FIRST_CODE <= ord(character) <= UNDECODED_BYTE_LAST_CODE:
+            byte = character.encode("utf-8", GIT_ERROR_DECODE_ERRORS)[UNDECODED_BYTE_INDEX]
+            characters.append(CONTROL_CHARACTER_ESCAPE_FORMAT.format(code=byte))
+            continue
+        if unicodedata.category(character) == CONTROL_CHARACTER_CATEGORY:
+            characters.append(CONTROL_CHARACTER_ESCAPE_FORMAT.format(code=ord(character)))
+            continue
+        characters.append(character)
+    return "".join(characters)
+
+
+def _protected_name(component: str) -> str:
+    """Return a folder name as Git compares it with the folder names it protects: without case, without the dots and spaces Windows drops at its end."""
+    return component.rstrip(WINDOWS_DROPPED_NAME_END).lower()
+
+
+def checkout_refusal_reason(path_text: str, platform: str) -> SharedText:
+    """Say why Git refused to write a path on the given system, named by the caller as Python names the running one.
+
+    A .git component, or a name Windows reads as one, is refused by Git itself on every system, so it is never put on
+    the file system. Under Windows, a path with a part Windows refuses is then put on Windows, even when it ends with a
+    name Git protects. A path whose last part is exactly .gitmodules, .gitattributes, .gitignore or .mailmap, without
+    case, is refused by Git too, as a symbolic link. Otherwise Windows is named only on Windows, where its file name
+    rules are the usual cause; elsewhere the reason stays neutral.
+    """
+    components = path_text.split(GIT_PATH_SEPARATOR)
+    for component in components:
+        if _protected_name(component) in PROTECTED_FOLDER_NAME_SET:
+            return SharedText.CHECKOUT_REFUSED_PROTECTED_PATH
+    # Python's own reading of the Windows name rules: dots and spaces at the end of a part, forbidden characters such as
+    # : and ?, device names such as CON. ntpath reads a start like "x:" as a drive and a backslash as a separator, so those
+    # two forms keep the reason of the link; no repository has a reason to hold them.
+    if platform == WINDOWS_PLATFORM and ntpath.isreserved(path_text):
+        return SharedText.CHECKOUT_REFUSED_WINDOWS_NAME
+    if components[LAST_COMPONENT_INDEX].lower() in PROTECTED_LINK_NAME_SET:
+        return SharedText.CHECKOUT_REFUSED_PROTECTED_LINK
+    if platform == WINDOWS_PLATFORM:
+        return SharedText.CHECKOUT_REFUSED_WINDOWS_NAME
+    return SharedText.CHECKOUT_REFUSED_OTHER_NAME
+
+
+def checkout_refusal_text(refusals: tuple[CheckoutRefusal, ...], platform: str) -> str:
+    """Return the sentence that explains each submodule Git could not check out, one line per submodule, or an empty text.
+
+    Each line names the submodule, gives one reason per path Git cited, in Git's order, and ends by saying that other
+    files may be in the same case and that the rest of the clone is kept. Only what Git wrote is named. Git replaces
+    each control character of its error lines by ?, except the tab and the line break, on every system: a cited path
+    that holds ? gets a sentence that says so after its reason, and no other path does.
+    """
+    lines: list[str] = []
+    for refusal in refusals:
+        reasons: list[str] = []
+        for path in refusal.refused_paths:
+            reasons.append(checkout_refusal_reason(path, platform).value.format(path=displayed_git_path(path)))
+            if GIT_CONTROL_CHARACTER_MARK in path:
+                reasons.append(SharedText.CHECKOUT_REFUSED_QUESTION_MARK.value)
+        start = SharedText.SUBMODULE_CHECKOUT_REFUSED_START.value.format(submodule=displayed_git_path(refusal.submodule_path))
+        lines.append(start + CHECKOUT_REFUSAL_PART_SEPARATOR.join(reasons) + CHECKOUT_REFUSAL_PART_SEPARATOR + SharedText.SUBMODULE_CHECKOUT_REFUSED_END.value)
+    return CHECKOUT_REFUSAL_LINE_SEPARATOR.join(lines)

gitpane/submodule/cloned_without_checkout.py

@@ -0,0 +1,65 @@
+"""Tell from the disk alone whether Git cloned a submodule but never checked out its files.
+
+Git clones every submodule of a recursive clone first, then checks them out one by one, and stops at the first
+checkout it cannot finish: the submodules after that one keep their Git folder, with the history and a HEAD, but no
+index, and their folder holds nothing but the ``.git`` file that points to that Git folder. Only that exact state is
+recognised here, so a write GitPane starts on such a submodule can never reach a file the user put there. No Git
+process is started; any doubt answers no.
+"""
+
+from __future__ import annotations
+
+import os
+import stat
+
+from gitpane.sequencer.replay_state import HEAD_BASENAME, LIVE_INDEX_BASENAME
+from gitpane.submodule.submodule_loss import GIT_ENTRY_BASENAME
+
+# What Git writes at the start of the .git file of a submodule whose Git folder lives in its parent's Git folder.
+GIT_FILE_PREFIX = "gitdir:"
+# A .git file holds one short line; anything larger is not one Git wrote, and is never read whole.
+GIT_FILE_SIZE_LIMIT_BYTES = 4096
+
+
+def linked_git_dir(git_filename: str) -> str:
+    """Return the Git folder a submodule's ``.git`` file names, made absolute, or an empty text when it is not such a file."""
+    metadata = os.lstat(git_filename)
+    if not stat.S_ISREG(metadata.st_mode) or metadata.st_size > GIT_FILE_SIZE_LIMIT_BYTES:
+        return ""
+    with open(git_filename, "rb") as stream:
+        line = stream.read(GIT_FILE_SIZE_LIMIT_BYTES).decode("utf-8").strip()
+    if not line.startswith(GIT_FILE_PREFIX):
+        return ""
+    git_dir = line.removeprefix(GIT_FILE_PREFIX).strip()
+    if not git_dir:
+        return ""
+    if not os.path.isabs(git_dir):
+        git_dir = os.path.join(os.path.dirname(git_filename), git_dir)
+    return os.path.normpath(git_dir)
+
+
+def is_cloned_without_checkout(submodule_dir: str) -> bool:
+    """Tell whether Git cloned this submodule but never wrote its files, so checking it out cannot overwrite anything.
+
+    True only when the folder is a real folder (not a link or a junction) whose only entry is a ``.git`` file, and the
+    Git folder that file names has a HEAD and no index. A folder that holds any other file, a ``.git`` folder, a Git
+    folder with an index, or anything that cannot be read answers False.
+    """
+    try:
+        metadata = os.lstat(submodule_dir)
+        if not stat.S_ISDIR(metadata.st_mode) or getattr(metadata, "st_reparse_tag", 0):
+            return False
+        basenames = []
+        with os.scandir(submodule_dir) as entries:
+            for entry in entries:
+                basenames.append(entry.name)
+                if entry.name != GIT_ENTRY_BASENAME:
+                    return False
+        if basenames != [GIT_ENTRY_BASENAME]:
+            return False
+        git_dir = linked_git_dir(os.path.join(submodule_dir, GIT_ENTRY_BASENAME))
+        if not git_dir:
+            return False
+        return os.path.isfile(os.path.join(git_dir, HEAD_BASENAME)) and not os.path.lexists(os.path.join(git_dir, LIVE_INDEX_BASENAME))
+    except (OSError, UnicodeDecodeError, ValueError):
+        return False