gitpane 1.0.0__py3-none-any.whl

gitpane/operation/diagnostic.py

@@ -0,0 +1,954 @@
+"""Prepare one secret-filtered diagnostic for every place that shows, copies or logs it."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import threading
+import uuid
+from contextlib import ExitStack
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from enum import Enum
+from typing import TYPE_CHECKING
+from urllib.parse import unquote, urlsplit, urlunsplit
+
+from gitpane.config import AUTHORIZATION_CODE_FIELD_NAME, DIAGNOSTIC_DECODE_LIMIT, LINE_SEPARATOR, NON_SECRET_ENVIRONMENT_NAME_SET, SENSITIVE_FIELD_NAMES, SENSITIVE_VALUE_MIN_LENGTH, TRACE_UNIT_LIMIT_BYTES
+
+if TYPE_CHECKING:
+    from collections.abc import Callable, Mapping
+    from typing import TextIO
+
+SENSITIVE_NAME_PATTERN = re.compile(r"password|passwd|token|secret|authorization|api[_-]?key|credential|cookie|verifier|nonce", re.IGNORECASE)
+URL_USERINFO_PATTERN = re.compile(r"(\b[a-z][a-z0-9+.-]*://)[^/\s@]+@", re.IGNORECASE)
+MASKED_VALUE = "[hidden]"
+# A "credential.username" or "credential.<url>.username" setting given on the command line: the key stays readable, the
+# login after "=" is hidden. Git reads configuration names without regard to case.
+CREDENTIAL_USERNAME_SETTING_PATTERN = re.compile(r"(credential\.(?:.+\.)?username=)(.+)", re.IGNORECASE | re.DOTALL)
+FIELD_PATTERN = "|".join(re.escape(name).replace("_", "[-_]") for name in SENSITIVE_FIELD_NAMES)
+# Words that make the "code" after them the result of a process or a request, never an OAuth authorization code. The
+# word must start a word of its own, but may follow an underscore ("process_exit_code"), and only blanks of the same
+# line, "_" or "-" may separate it from "code": a word that ends one line says nothing about a field on the next one.
+RESULT_CODE_QUALIFIER_PATTERN = r"(?<![A-Za-z0-9])(?:exit|status|error|return)[ \t_-]*"
+# Characters that, written right before a field name, make it the end of a longer name such as "passcode" or "otp_code".
+WORD_CHARACTER_PATTERN = re.compile(r"[A-Za-z0-9_-]")
+FIELD_ASSIGNMENT_PATTERN = re.compile(
+    r"(?P<prefix>(?P<qualifier>" + RESULT_CODE_QUALIFIER_PATTERN + r")?[\"']?(?P<name>" + FIELD_PATTERN + r")[\"']?\s*[:=]\s*)"
+    r"(?P<value>\[hidden\]|\[REDACTED\]|\"(?:\\.|[^\"\\])*\"|'(?:\\.|[^'\\])*'|[^\s&;,}\]]+)",
+    re.IGNORECASE,
+)
+# A whole number, with the minus sign of a signal exit (-15) or of a Windows status read as signed (-1073741819).
+NUMERIC_CODE_VALUE_PATTERN = re.compile(r"-?(?P<digits>[0-9]+)")
+# Longest number a bare "code" field may show: exit codes 0 to 255, signal exits, HTTP statuses and Git's 128 all fit,
+# while a six-digit one-time code, the numeric authorization code a server can send, stays hidden.
+BARE_CODE_DIGIT_LIMIT = 3
+# Longest number an "exit code", "status code", "error code" or "return code" may show: a Windows exit status is a
+# 32-bit value, up to 4294967295 (10 digits), such as 3221225477 for an access violation.
+QUALIFIED_CODE_DIGIT_LIMIT = 10
+SECRET_HEADER_PATTERN = re.compile(r"((?:proxy-authorization|authorization|set-cookie|cookie)\s*:\s*)([^\r\n]+)", re.IGNORECASE)
+URL_PATTERN = re.compile(r"\b[a-z][a-z0-9+.-]*://[^\s\"'<>]+", re.IGNORECASE)
+ANSI_PATTERN = re.compile(r"\x1b(?:\[[0-?]*[ -/]*[@-~]|\][^\x07]*(?:\x07|\x1b\\))")
+SENSITIVE_NORMALIZED_NAME_SET = frozenset(name.replace("-", "").replace("_", "") for name in SENSITIVE_FIELD_NAMES)
+CURL_DATA_PATTERN = re.compile(rb"(?:<= Recv|=> Send) data: (.*)")
+CURL_DATA_SIZE_PATTERN = re.compile(rb"(?:<= Recv|=> Send) data, [0-9]+ bytes")
+DIAGNOSTIC_OPERATION_PREFIX = "Operation: "
+DIAGNOSTIC_CAUSE_PREFIX = "Cause: "
+DIAGNOSTIC_STDERR_HEADER = "Standard error:"
+DIAGNOSTIC_TECHNICAL_DETAIL_HEADER = "Technical details:"
+# Start of the one-line description that replaces a binary output; the console leaves it out, the records and --log keep it.
+OPAQUE_BODY_PREFIX = "Opaque body: "
+# A text unit is read as JSON when its first non-blank byte opens an object or an array. On a bytes pattern, \s covers
+# exactly the six ASCII blanks bytes.lstrip() removes, and, like a strip of the rest of the output, it crosses line
+# breaks: a blank line followed by a JSON line is read together with it.
+JSON_UNIT_START_PATTERN = re.compile(rb"\s*[\[{]")
+LEADING_BLANK_PATTERN = re.compile(rb"\s*")
+# JSON tokens, read the way json.loads reads them (NaN and Infinity included). A string cannot hold a raw line break and
+# numbers and literals end at one, so no token straddles the end of a line: each complete line can be scanned on its own.
+JSON_BLANK_PATTERN = re.compile(rb"[ \t\n\r]*")
+JSON_TOKEN_PATTERN = re.compile(
+    rb'(?P<punctuation>[\[\]{},:])|(?P<string>"(?:[^"\\\x00-\x1f]|\\["\\/bfnrt]|\\u[0-9a-fA-F]{4})*")'
+    rb"|(?P<scalar>-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][-+]?[0-9]+)?|true|false|null|NaN|-?Infinity)"
+)
+JSON_ARRAY_OPENING = b"["
+JSON_OBJECT_OPENING = b"{"
+JSON_OPENING_BY_CLOSING = {b"]": JSON_ARRAY_OPENING, b"}": JSON_OBJECT_OPENING}
+JSON_NAME_SEPARATOR = b":"
+JSON_VALUE_SEPARATOR = b","
+INNERMOST_BRACKET_INDEX = -1
+EXIT_STATUS_RUNNING = "running"
+EXIT_STATUS_NOT_STARTED = "not started"
+# The status of a failure that came after its process had already ended normally: the process is not the cause.
+EXIT_STATUS_ENDED_BEFORE_FAILURE = "ended before the failure"
+TERMINAL_REDRAW_BREAK = "\r"
+
+
+@dataclass(frozen=True)
+class MaskedArguments:
+    """A command with its sensitive words replaced, and the raw secrets it replaced.
+
+    ``secret_values`` are values written after a secret-named option, such as a password or a token: every text of the
+    command, Git's own error output included, must hide them.
+
+    The login of a ``credential.<url>.username=<login>`` setting is hidden in ``arguments`` only, as the value of its
+    setting, and is not returned: it is no secret, and replacing that word anywhere else would damage the text around
+    it. On GitHub a login is also the owner part of that user's repository URLs, often the user's local account name in
+    every path, and a short one is part of ordinary words.
+    """
+
+    arguments: list[str]
+    secret_values: list[str]
+
+
+def mask_sensitive_arguments(arguments: list[str]) -> MaskedArguments:
+    """Return the command with every value written after a secret-named option replaced, plus those raw values.
+
+    This rule lives here alone: every surface that shows a command — error diagnostics, the request published to
+    subscribers, the process log line, the console command line — must hide exactly the same words, and separate
+    copies have already drifted apart. A second sensitive option in a row is hidden too, because hiding one word too
+    many is the safe reading.
+
+    The value of every ``credential.<url>.username=<login>`` setting is hidden in the command as well: commands of a
+    repository with submodules carry the login of each saved default account in such a setting. Only that value is
+    hidden, in the command's own words; the login is not returned, so no filter replaces it elsewhere.
+    """
+    masked_arguments: list[str] = []
+    secret_values: list[str] = []
+    hide_next = False
+    for argument in arguments:
+        username_setting = CREDENTIAL_USERNAME_SETTING_PATTERN.fullmatch(argument)
+        if hide_next:
+            masked_arguments.append(MASKED_VALUE)
+            secret_values.append(argument)
+        elif username_setting is not None:
+            masked_arguments.append(username_setting.group(1) + MASKED_VALUE)
+        else:
+            masked_arguments.append(argument)
+        hide_next = argument.startswith("-") and SENSITIVE_NAME_PATTERN.fullmatch(argument.lstrip("-")) is not None
+    return MaskedArguments(masked_arguments, secret_values)
+
+
+class SensitiveTextFilter:
+    """Filter complete text units and remember discovered secrets for later echoes."""
+
+    def __init__(self, environment_by_name: Mapping[str, str] | None = None) -> None:
+        """Accept an explicit environment for isolated tests and credential consumers."""
+        self.sensitive_value_set: set[str] = set()
+        for name, value in (os.environ if environment_by_name is None else environment_by_name).items():
+            if value and name.upper() not in NON_SECRET_ENVIRONMENT_NAME_SET and SENSITIVE_NAME_PATTERN.search(name):
+                self.sensitive_value_set.add(value)
+
+    @staticmethod
+    def decode(value: str) -> str:
+        """Recognize repeatedly encoded query keys without an unbounded decoding loop."""
+        for _ in range(DIAGNOSTIC_DECODE_LIMIT):
+            decoded = unquote(value)
+            if decoded == value:
+                break
+            value = decoded
+        return value
+
+    @classmethod
+    def normalized_name(cls, name: str) -> str:
+        """Bring a field name to the form the sensitive names are compared in: decoded, lowercase, without - or _."""
+        return cls.decode(name).lower().replace("-", "").replace("_", "")
+
+    @classmethod
+    def is_sensitive_name(cls, name: str) -> bool:
+        """Use one policy for nested JSON fields, query keys and protocol assignments."""
+        return cls.normalized_name(name) in SENSITIVE_NORMALIZED_NAME_SET
+
+    @classmethod
+    def is_readable_code_value(cls, name: str, value: object, qualified: bool = False) -> bool:
+        """Tell whether a "code" field holds a short whole number, an exit, HTTP or Git error code that stays readable.
+
+        The OAuth authorization code shares that field name, but it is a long string of letters and digits: any value
+        that is not a short number, and every value of the other sensitive fields, stays hidden. ``qualified`` says the
+        field was written after a word such as "exit" or "status", which allows the longer Windows exit statuses.
+        """
+        if cls.normalized_name(name) != AUTHORIZATION_CODE_FIELD_NAME:
+            return False
+        # JSON true and false are ints in Python, but no exit code: their text is not a number and they stay hidden.
+        if isinstance(value, int):
+            value = str(value)
+        if not isinstance(value, str):
+            return False
+        number = NUMERIC_CODE_VALUE_PATTERN.fullmatch(value)
+        if number is None:
+            return False
+        digit_limit = QUALIFIED_CODE_DIGIT_LIMIT if qualified else BARE_CODE_DIGIT_LIMIT
+        return len(number.group("digits")) <= digit_limit
+
+    def redact(self, text: str) -> str:
+        """Retain safe response text while hiding structured, encoded and known secrets."""
+        text = ANSI_PATTERN.sub("", text).replace("\r\n", "\n").replace("\r", "\n")
+        candidate = text.strip()
+        if candidate.startswith(("{", "[")):
+            try:
+                value = json.loads(candidate)
+                self._collect_json(value)
+                return json.dumps(self._redact_json(value), ensure_ascii=False)
+            except (ValueError, RecursionError):
+                pass
+        text = URL_PATTERN.sub(self._redact_url, text)
+        text = SECRET_HEADER_PATTERN.sub(self._redact_header, text)
+        text = FIELD_ASSIGNMENT_PATTERN.sub(self._redact_assignment, text)
+        return self.hide_known_values(text)
+
+    def hide_known_values(self, text: str) -> str:
+        """Replace every remembered value in already-parsed text, without learning anything new from it."""
+        for value in self.masked_values():
+            text = text.replace(value, MASKED_VALUE)
+        return text
+
+    def remember_values(self, values: list[str]) -> None:
+        """Hide these raw values in every later text, such as the words a command hid; an empty value is skipped."""
+        for value in values:
+            # An empty value would make the replacement insert the masking text between every single character.
+            if value:
+                self.sensitive_value_set.add(value)
+
+    def masked_values(self) -> list[str]:
+        """Return the remembered values that are long enough to hide, longest first so a value never splits a longer one.
+
+        The length rule lives here, where values are used, rather than where they are added: environment variables,
+        JSON fields, URLs and callers that add a value directly all end up in the same set, and none of them can bring
+        back a one-character value that would hide every occurrence of that character in the output.
+        """
+        long_enough_values: list[str] = []
+        for value in self.sensitive_value_set:
+            if len(value) >= SENSITIVE_VALUE_MIN_LENGTH:
+                long_enough_values.append(value)
+        return sorted(long_enough_values, key=len, reverse=True)
+
+    def _remember(self, value: object) -> None:
+        """Collect nested secret values before processing any public echo in the same unit."""
+        if isinstance(value, dict):
+            for nested in value.values():
+                self._remember(nested)
+        elif isinstance(value, list):
+            for nested in value:
+                self._remember(nested)
+        elif isinstance(value, str) and value and value not in {MASKED_VALUE, "[REDACTED]"}:
+            self.sensitive_value_set.add(value)
+            self.sensitive_value_set.add(self.decode(value))
+
+    def _collect_json(self, value: object) -> None:
+        """Discover secret fields before rendering sibling fields that can echo them."""
+        if isinstance(value, dict):
+            for name, nested in value.items():
+                if self.is_sensitive_name(name) and not self.is_readable_code_value(name, nested):
+                    self._remember(nested)
+                else:
+                    self._collect_json(nested)
+        elif isinstance(value, list):
+            for nested in value:
+                self._collect_json(nested)
+
+    def _redact_json(self, value: object) -> object:
+        """Preserve nested JSON structure while masking protected fields and echoed values."""
+        if isinstance(value, dict):
+            result_by_name: dict[str, object] = {}
+            for name, nested in value.items():
+                hidden = self.is_sensitive_name(name) and not self.is_readable_code_value(name, nested)
+                result_by_name[name] = MASKED_VALUE if hidden else self._redact_json(nested)
+            return result_by_name
+        if isinstance(value, list):
+            results: list[object] = []
+            for nested in value:
+                results.append(self._redact_json(nested))
+            return results
+        if isinstance(value, str):
+            return self.redact(value)
+        return value
+
+    def _redact_url(self, match: re.Match[str]) -> str:
+        """Keep the destination path while masking userinfo and secret query values.
+
+        Only a userinfo that carries a password is remembered, to hide it wherever it is echoed. A user name alone, such
+        as the account login GitPane writes in the URL of a command, is hidden in its URL only: it is no secret, and the
+        same word is often the owner of the user's repositories and the user's own folder name, which must stay readable.
+        """
+        value = match.group()
+        try:
+            parsed = urlsplit(value)
+            authority = parsed.netloc
+            if "@" in authority:
+                userinfo, _, destination = authority.rpartition("@")
+                _user, password_separator, password = userinfo.partition(":")
+                if password_separator:
+                    self._remember(userinfo)
+                    self._remember(password)
+                authority = MASKED_VALUE + "@" + destination
+            components: list[str] = []
+            for component in parsed.query.split("&"):
+                name, separator, content = component.partition("=")
+                if self.is_sensitive_name(name) and not self.is_readable_code_value(name, self.decode(content)):
+                    self._remember(content)
+                    component = name + separator + MASKED_VALUE
+                components.append(component)
+            return urlunsplit((parsed.scheme, authority, parsed.path, "&".join(components), parsed.fragment))
+        except ValueError:
+            return URL_USERINFO_PATTERN.sub(r"\1" + MASKED_VALUE + "@", value)
+
+    def _redact_header(self, match: re.Match[str]) -> str:
+        value = match.group(2)
+        self._remember(value)
+        for part in value.split(";"):
+            if "=" in part:
+                self._remember(part.partition("=")[-1].strip())
+        if value.lower().startswith(("bearer ", "basic ")):
+            self._remember(value.partition(" ")[-1])
+        return match.group(1) + MASKED_VALUE + ("\ufffd" if "\ufffd" in value else "")
+
+
+    @staticmethod
+    def name_ends_another_word(match: re.Match[str]) -> bool:
+        """Tell whether the field name found is only the end of a longer name, such as "code" in "passcode" or "otp_code".
+
+        The pattern finds a field name anywhere in a word, which keeps "my_token=..." hidden; but the readable short
+        number is allowed only to a "code" written as a word of its own, or right after "exit", "status", "error" or
+        "return".
+        """
+        name_start = match.start("name")
+        if match.group("qualifier") is not None or name_start == 0:
+            return False
+        return WORD_CHARACTER_PATTERN.fullmatch(match.string[name_start - 1]) is not None
+
+    def _redact_assignment(self, match: re.Match[str]) -> str:
+        """Mask protected assignments and remember their values for later echoes; a short exit or error code stays as is."""
+        value = match.group("value").strip("\"'")
+        if not self.name_ends_another_word(match) and self.is_readable_code_value(match.group("name"), value, match.group("qualifier") is not None):
+            return match.group()
+        self._remember(value)
+        return match.group("prefix") + MASKED_VALUE
+
+
+class JsonScanStatus(Enum):
+    """Where a JSON unit stands at the end of a scanned line."""
+
+    UNFINISHED = "unfinished"
+    COMPLETE = "complete"
+    INVALID = "invalid"
+
+
+class JsonExpectation(Enum):
+    """The token kinds the JSON grammar accepts next."""
+
+    VALUE = "value"
+    VALUE_OR_CLOSE = "value or close"
+    KEY = "key"
+    KEY_OR_CLOSE = "key or close"
+    COLON = "colon"
+    COMMA_OR_CLOSE = "comma or close"
+
+
+JSON_VALUE_EXPECTATION_SET = frozenset({JsonExpectation.VALUE, JsonExpectation.VALUE_OR_CLOSE})
+JSON_KEY_EXPECTATION_SET = frozenset({JsonExpectation.KEY, JsonExpectation.KEY_OR_CLOSE})
+JSON_CLOSE_EXPECTATION_SET = frozenset({JsonExpectation.VALUE_OR_CLOSE, JsonExpectation.KEY_OR_CLOSE, JsonExpectation.COMMA_OR_CLOSE})
+
+
+class JsonUnitScanner:
+    """Follow one JSON value line by line, so a stream knows at each line end whether to wait for more lines.
+
+    Re-parsing the whole unit with json.loads at every line cost the unit's size times its line count, and any line
+    that merely starts with a bracket, like Git's "[main 1a2b3c4] subject", kept failing and pulled in the rest of the
+    output. The scan reads each byte once and rejects such a line on the line where it appears. It only checks the
+    grammar; SensitiveTextFilter.redact still parses the finished unit itself.
+    """
+
+    def __init__(self) -> None:
+        """Start before the first token of the value."""
+        self.open_brackets: list[bytes] = []
+        self.expectation = JsonExpectation.VALUE
+        self.finished = False
+
+    def scan_line(self, content: bytearray, position: int, end: int) -> JsonScanStatus:
+        """Read the complete line content[position:end] and tell whether the value ended, can continue, or is not JSON."""
+        while True:
+            position = JSON_BLANK_PATTERN.match(content, position, end).end()
+            if position == end:
+                return JsonScanStatus.COMPLETE if self.finished else JsonScanStatus.UNFINISHED
+            if self.finished:
+                return JsonScanStatus.INVALID
+            match = JSON_TOKEN_PATTERN.match(content, position, end)
+            if match is None or not self._accept(match):
+                return JsonScanStatus.INVALID
+            position = match.end()
+
+    def _accept(self, match: re.Match[bytes]) -> bool:
+        """Apply one token to the grammar state, or return False when the grammar does not allow it here."""
+        punctuation = match.group("punctuation")
+        if punctuation is None:
+            if match.group("string") is not None and self.expectation in JSON_KEY_EXPECTATION_SET:
+                self.expectation = JsonExpectation.COLON
+                return True
+            if self.expectation not in JSON_VALUE_EXPECTATION_SET:
+                return False
+            self._end_value()
+            return True
+        if punctuation in (JSON_ARRAY_OPENING, JSON_OBJECT_OPENING):
+            if self.expectation not in JSON_VALUE_EXPECTATION_SET:
+                return False
+            self.open_brackets.append(punctuation)
+            self.expectation = JsonExpectation.VALUE_OR_CLOSE if punctuation == JSON_ARRAY_OPENING else JsonExpectation.KEY_OR_CLOSE
+            return True
+        if punctuation == JSON_NAME_SEPARATOR:
+            if self.expectation is not JsonExpectation.COLON:
+                return False
+            self.expectation = JsonExpectation.VALUE
+            return True
+        if punctuation == JSON_VALUE_SEPARATOR:
+            if self.expectation is not JsonExpectation.COMMA_OR_CLOSE:
+                return False
+            self.expectation = JsonExpectation.VALUE if self.open_brackets[INNERMOST_BRACKET_INDEX] == JSON_ARRAY_OPENING else JsonExpectation.KEY
+            return True
+        if not self.open_brackets or self.open_brackets[INNERMOST_BRACKET_INDEX] != JSON_OPENING_BY_CLOSING[punctuation]:
+            return False
+        if self.expectation not in JSON_CLOSE_EXPECTATION_SET:
+            return False
+        self.open_brackets.pop()
+        self._end_value()
+        return True
+
+    def _end_value(self) -> None:
+        """After a complete value, expect a separator or a closing bracket, or mark the whole unit finished."""
+        if self.open_brackets:
+            self.expectation = JsonExpectation.COMMA_OR_CLOSE
+            return
+        self.finished = True
+
+
+class TextUnitEnd(Enum):
+    """Say whether a filtered unit completes a line or only redraws its current value."""
+
+    LINE = "line"
+    REDRAW = "redraw"
+    HIDDEN_TAIL = "hidden tail"
+
+
+class TextValueSource(Enum):
+    """Distinguish a newly received visible value from an already published preview."""
+
+    NEW = "new"
+    PUBLISHED = "published"
+
+
+@dataclass(frozen=True)
+class TextUnit:
+    """Keep a complete filtered record separate from the value the console shows."""
+
+    text: str
+    shown_text: str | None
+    end: TextUnitEnd = TextUnitEnd.LINE
+    source: TextValueSource = TextValueSource.NEW
+
+
+class SensitiveTextStream:
+    """Hold incomplete UTF-8 and JSON units until the common filter can protect them."""
+
+    def __init__(self, text_filter: SensitiveTextFilter | None = None, hides_value: Callable[[str], bool] | None = None) -> None:
+        """Share a filter when several streams can repeat the same operation secret."""
+        self.filter = text_filter or SensitiveTextFilter()
+        self.hides_value = hides_value
+        self.buffer = bytearray()
+        self.complete = True
+        self.discarded = False
+        self.secret_header = False
+        # A JSON unit still open at the end of the previous call: it starts at the head of the buffer and its lines are
+        # scanned up to json_scan_end, so the next call resumes there instead of reading those lines again.
+        self.json_scanner: JsonUnitScanner | None = None
+        self.json_scan_end = 0
+        self.published_end = 0
+        self.redraw_end = 0
+        self.redraw_value = ""
+        self.redraw_value_start: int | None = None
+
+    def feed(self, data: bytes, final: bool = False) -> list[str]:
+        """Return complete filtered records without the console-only redraws."""
+        texts: list[str] = []
+        for unit in self.feed_units(data, final):
+            if unit.end != TextUnitEnd.REDRAW:
+                texts.append(unit.text)
+        return texts
+
+    def feed_units(self, data: bytes, final: bool = False) -> list[TextUnit]:
+        """Return complete records with safe previews of the currently open line."""
+        return self._feed_units(data, final)
+
+    def feed_pieces(self, data: bytes, final: bool = False) -> list[str]:
+        """Return filtered pieces a downstream trace can read with the original carriage returns.
+
+        Complete values of an open line are filtered separately so progress can arrive before the newline without
+        exposing a partial secret. Protected units stay whole; completed records still use the shared unit scanner.
+        """
+        pieces: list[str] = []
+        self._feed_units(data, final, pieces)
+        return pieces
+
+    def _feed_units(self, data: bytes, final: bool, pieces: list[str] | None = None) -> list[TextUnit]:
+        """Return only safe complete units; a quota failure withholds the remaining stream.
+
+        A unit is one line, or, when its first non-blank byte opens JSON, the lines up to the end of that JSON value; an
+        unfinished JSON value waits for more output unless the stream ends. A line that cannot continue the value, such
+        as Git's "[main 1a2b3c4] subject", ends its unit at once, so it never holds back the lines after it. The buffer
+        is walked with an index and cut once per call, and each line is scanned once even when a JSON unit spans several
+        calls, so the cost grows with the output instead of with the output times its line count.
+        """
+        if self.discarded:
+            return []
+        if len(self.buffer) + len(data) > TRACE_UNIT_LIMIT_BYTES:
+            self.buffer = bytearray()
+            self.json_scanner = None
+            self.complete = False
+            self.discarded = True
+            self.published_end = 0
+            self.reset_redraw()
+            text = f"Trace incomplete: text unit exceeds {TRACE_UNIT_LIMIT_BYTES} bytes; remaining stream withheld."
+            if pieces is not None:
+                pieces.append(text + "\n")
+            return [TextUnit(text, text)]
+        content = self.buffer
+        content.extend(data)
+        units: list[TextUnit] = []
+        if b"\n" not in data and not final:
+            # No unit can end without a new line break, and every earlier line break in the buffer was already tried.
+            if b"\r" in data:
+                self._publish_redraw(bytes(content), units, pieces)
+            return units
+        start = 0
+        scanner = self.json_scanner
+        scan_end = self.json_scan_end
+        # Where the blank run that begins at or after start ends: a run of blank lines is scanned once, not once per line.
+        blank_end = -1
+        while True:
+            line_start = start if scanner is None else scan_end
+            newline = content.find(b"\n", line_start)
+            if newline < 0:
+                break
+            end = newline + 1
+            if scanner is None:
+                if start > blank_end:
+                    blank_end = LEADING_BLANK_PATTERN.match(content, start).end()
+                if not JSON_UNIT_START_PATTERN.match(content, blank_end):
+                    units.append(self._finish_unit(bytes(content[start:end]), pieces))
+                    start = end
+                    continue
+                scanner = JsonUnitScanner()
+            if scanner.scan_line(content, line_start, end) is JsonScanStatus.UNFINISHED:
+                scan_end = end
+                continue
+            # The value is complete, or this line proves the unit is not JSON after all: either way the unit ends here,
+            # kept whole so a secret split across its lines is still filtered together.
+            units.append(self._finish_unit(bytes(content[start:end]), pieces))
+            scanner = None
+            start = end
+        del content[:start]
+        if final and content:
+            units.append(self._finish_unit(bytes(content), pieces))
+            content.clear()
+            scanner = None
+        self.json_scanner = scanner
+        self.json_scan_end = scan_end - start if scanner is not None else 0
+        if not final and b"\r" in data:
+            self._publish_redraw(bytes(content), units, pieces)
+        if final:
+            self.reset_redraw()
+        return units
+
+    def _allows_redraw(self, content: bytes, *, complete_unit: bool = False) -> bool:
+        """Keep protected units whole before either previewing or forwarding any value."""
+        if (not complete_unit and self.json_scanner is not None) or b"\0" in content or JSON_UNIT_START_PATTERN.match(content):
+            return False
+        return not (self.secret_header and content.startswith((b" ", b"\t")))
+
+    def _publish_redraw(self, content: bytes, units: list[TextUnit], pieces: list[str] | None) -> None:
+        """Forward every new complete value, or provide just the last console preview."""
+        if pieces is None:
+            redraw = self.open_line_redraw(content)
+            if redraw is not None:
+                units.append(redraw)
+            return
+        if not self._allows_redraw(content):
+            return
+        end = content.rfind(b"\r") + 1
+        if end <= self.published_end:
+            return
+        try:
+            text = content[self.published_end:end].decode("utf-8")
+        except UnicodeDecodeError:
+            return
+        for value in text.removesuffix("\r").split("\r"):
+            pieces.append(self.filter.redact(value) + "\r")
+        self.published_end = end
+
+    def _finish_unit(self, content: bytes, pieces: list[str] | None) -> TextUnit:
+        """Finish a shared record while forwarding only values not already published."""
+        can_reuse_redraw = self.redraw_value_start is not None and self._allows_redraw(content, complete_unit=True) and b"\n" not in content.removesuffix(b"\n")
+        if pieces is None:
+            unit = self._prepare_unit(content)
+        elif self.published_end:
+            # A later binary value cannot retract earlier progress. Filter each remaining value so its opaque
+            # description does not discard the readable values after it or repeat the already forwarded prefix.
+            secret_header = self.secret_header
+            texts: list[str] = []
+            for value in content[self.published_end:].removesuffix(b"\n").split(b"\r"):
+                texts.append(self._prepare_unit(value).text)
+            self.secret_header = secret_header
+            pieces.append("\r".join(texts) + "\n")
+            unit = self._prepare_unit(content)
+        else:
+            preserves_redraw = self._allows_redraw(content, complete_unit=True) and b"\n" not in content.removesuffix(b"\n")
+            try:
+                content.decode("utf-8")
+            except UnicodeDecodeError:
+                preserves_redraw = False
+            unit = self._prepare_unit(content)
+            # In a single ordinary line, every newline introduced by the filter came from a carriage return.
+            text = unit.text.replace("\n", "\r") if preserves_redraw else unit.text
+            pieces.append(text + "\n")
+        if can_reuse_redraw:
+            end = len(content.removesuffix(b"\n"))
+            source = self._learn_redraw(content, end) or TextValueSource.NEW
+            unit = TextUnit(unit.text, unit.shown_text, unit.end, source)
+        self.reset_redraw()
+        self.published_end = 0
+        return unit
+
+    def open_line_redraw(self, open_line: bytes) -> TextUnit | None:
+        """Learn new complete redraw values before choosing the last visible filtered preview.
+
+        The preview cursor is separate from the buffered record and forwarded pieces, which retain every value.
+        A quoted value split by a carriage return is filtered piece by piece, just as it is at a line feed. JSON,
+        binary text and folded secret headers must instead stay private until their whole unit can be filtered.
+        """
+        if b"\r" not in open_line or not self._allows_redraw(open_line):
+            return None
+        end = open_line.rfind(b"\r") + 1
+        source = self._learn_redraw(open_line, end)
+        if source is None:
+            return None
+        # Another segment or stream may have taught the shared filter a secret contained in the retained value.
+        shown = terminal_line(self.filter.redact(self.redraw_value), LINE_SEPARATOR, self.hides_value)
+        if not shown:
+            return None
+        self.redraw_value = shown
+        return TextUnit(shown, shown, TextUnitEnd.REDRAW, source)
+
+    def _learn_redraw(self, content: bytes, end: int) -> TextValueSource | None:
+        """Learn new segments while identifying their visible value by byte position, never by text equality."""
+        try:
+            text = content[self.redraw_end:end].decode("utf-8")
+        except UnicodeDecodeError:
+            return None
+        previous_start = self.redraw_value_start
+        start = self.redraw_end
+        for value in text.removesuffix("\r").split("\r"):
+            shown = terminal_line(self.filter.redact(value), LINE_SEPARATOR, self.hides_value)
+            if shown:
+                self.redraw_value = shown
+                self.redraw_value_start = start
+            start += len(value.encode("utf-8")) + len(TERMINAL_REDRAW_BREAK)
+        self.redraw_end = end
+        return TextValueSource.NEW if self.redraw_value_start != previous_start else TextValueSource.PUBLISHED
+
+    def reset_redraw(self) -> None:
+        """Forget the completed line's preview without forgetting operation secrets."""
+        self.redraw_end = 0
+        self.redraw_value = ""
+        self.redraw_value_start = None
+
+    def _prepare_unit(self, content: bytes) -> TextUnit:
+        """Remove terminal control sequences and folded secret headers before publication."""
+        if b"\0" in content:
+            text = OPAQUE_BODY_PREFIX + f"type=application/octet-stream; length={len(content)} bytes; result=opaque."
+            return TextUnit(text, text)
+        valid_utf8 = True
+        try:
+            text = content.decode("utf-8").rstrip("\n")
+        except UnicodeDecodeError:
+            valid_utf8 = False
+            self.complete = False
+            text = "Trace incomplete: invalid UTF-8 text. " + content.decode("utf-8", "replace").rstrip("\n")
+        if self.secret_header and text.startswith((" ", "\t")):
+            return TextUnit(MASKED_VALUE, MASKED_VALUE)
+        self.secret_header = bool(SECRET_HEADER_PATTERN.search(text))
+        text = self.filter.redact(text)
+        if not valid_utf8 or b"\n" in content.removesuffix(b"\n"):
+            return TextUnit(text, text)
+        # The shared filter has already turned each carriage return into a line feed, so those breaks are redraws here.
+        shown_text = terminal_line(text, LINE_SEPARATOR, self.hides_value)
+        last_value = terminal_line(text, LINE_SEPARATOR)
+        end = TextUnitEnd.LINE
+        if shown_text is not None and last_value and self.hides_value is not None and self.hides_value(last_value):
+            end = TextUnitEnd.HIDDEN_TAIL
+        return TextUnit(text, shown_text, end)
+
+class NetworkTextStream:
+    """Reconstruct curl display fragments before filtering complete HTTP bodies."""
+
+    def __init__(self, text_filter: SensitiveTextFilter, hides_value: Callable[[str], bool] | None = None) -> None:
+        """Share discovered secrets with ordinary output and final error publication."""
+        self.filter = text_filter
+        self.stream = SensitiveTextStream(text_filter, hides_value)
+        self.buffer = b""
+        self.body = bytearray()
+        self.complete = True
+        self.discarded = False
+
+    def feed(self, data: bytes, final: bool = False) -> list[tuple[str, str]]:
+        """Return complete filtered network records without console-only redraws."""
+        records: list[tuple[str, str]] = []
+        for kind, unit in self.feed_units(data, final):
+            if unit.end != TextUnitEnd.REDRAW:
+                records.append((kind, unit.text))
+        return records
+
+    def feed_units(self, data: bytes, final: bool = False) -> list[tuple[str, TextUnit]]:
+        """Return safe ordinary records and reconstructed bodies, withholding oversized units."""
+        if self.discarded:
+            return []
+        content = self.buffer + data
+        self.buffer = b""
+        if len(content) > TRACE_UNIT_LIMIT_BYTES:
+            return self._discard()
+        records: list[tuple[str, TextUnit]] = []
+        while b"\n" in content or final and content:
+            line, separator, content = content.partition(b"\n")
+            match = CURL_DATA_PATTERN.search(line)
+            if match is not None:
+                self.stream.reset_redraw()
+                self.body.extend(match.group(1).rstrip(b"\r"))
+                if len(self.body) > TRACE_UNIT_LIMIT_BYTES:
+                    return records + self._discard()
+                continue
+            if not CURL_DATA_SIZE_PATTERN.search(line):
+                records.extend(self._flush_body())
+            # Finish the unterminated last line directly: a discarded intermediate preview was never published.
+            for unit in self.stream.feed_units(line + separator, final=final and not separator):
+                records.append(("stderr", unit))
+            self.stream.reset_redraw()
+        self.buffer = content
+        if not final and b"\r" in data and not self.body and not CURL_DATA_PATTERN.search(content) and not CURL_DATA_SIZE_PATTERN.search(content):
+            redraw = self.stream.open_line_redraw(content)
+            if redraw is not None:
+                records.append(("stderr", redraw))
+        if final:
+            records.extend(self._flush_body())
+            for unit in self.stream.feed_units(b"", final=True):
+                records.append(("stderr", unit))
+        self.complete = self.complete and self.stream.complete
+        return records
+
+    def _flush_body(self) -> list[tuple[str, TextUnit]]:
+        """Filter a complete body before any consumer receives its fragmented fields."""
+        if not self.body:
+            return []
+        stream = SensitiveTextStream(self.filter)
+        records = []
+        for unit in stream.feed_units(bytes(self.body), final=True):
+            records.append(("http-body", unit))
+        self.body.clear()
+        self.complete = self.complete and stream.complete
+        return records
+
+    def _discard(self) -> list[tuple[str, TextUnit]]:
+        """Fail closed when a body or display line exceeds the bounded capture size."""
+        self.complete = False
+        self.discarded = True
+        self.body.clear()
+        self.buffer = b""
+        self.stream.reset_redraw()
+        text = "Trace incomplete: a network text unit exceeded the capture limit; remaining stream withheld."
+        return [("stderr", TextUnit(text, text))]
+
+
+def terminal_line(line: str, redraw_break: str = TERMINAL_REDRAW_BREAK, hides_value: Callable[[str], bool] | None = None) -> str | None:
+    """Keep the last nonempty visible value of a redrawn line without trailing padding.
+
+    A break at the very end does not redraw anything, so interrupted progress keeps its value. Return None when
+    every nonempty value is hidden, or an empty string when the line never held a nonempty value.
+    """
+    hidden = False
+    for value in reversed(line.split(redraw_break)):
+        value = value.rstrip()
+        if not value:
+            continue
+        if hides_value is not None and hides_value(value):
+            hidden = True
+            continue
+        return value
+    return None if hidden else ""
+
+
+def terminal_text(text: str) -> str:
+    """Return the final visible value of each terminal line without trailing padding.
+
+    Git redraws its progress lines with carriage returns ("Receiving objects:  10%", then "100%, done."), so only the
+    final value of each progress line is worth showing. A Windows line ending (carriage return then line feed) is an
+    ordinary line break, not a redraw. A final carriage return leaves the last nonempty value intact.
+    """
+    lines: list[str] = []
+    for line in text.replace("\r\n", "\n").split("\n"):
+        lines.append(terminal_line(line))
+    return "\n".join(lines)
+
+
+@dataclass(frozen=True)
+class PreparedDiagnostic:
+    """Hold already-filtered text so every output exposes the same details.
+
+    Only the "Arguments:" line of ``detail`` differs from Git's own text: it hides the value of each
+    ``credential.<url>.username`` setting, while ``stderr``, ``cause`` and the rest of ``detail`` keep the login
+    wherever Git wrote it, such as in the URL of a refused repository.
+
+    ``stderr`` is the filtered standard error alone, as a terminal would show it, so a form can show what Git said
+    without the command line and the rest of the detail.
+
+    ``return_code`` and ``process_id`` say which process failed and how, when one did: an operation console that
+    already ended that process with its own line does not repeat the failure, and one that hid the process gives its
+    exit code instead.
+    """
+
+    operation: str
+    cause: str
+    detail: str
+    masked: bool
+    stderr: str = ""
+    return_code: int | None = None
+    process_id: str = ""
+
+
+@dataclass(frozen=True)
+class Diagnostic:
+    """Describe a failure without recording command stdout or the whole environment.
+
+    ``process_started`` tells that the process was launched and has not exited yet, as for the record written when a
+    command starts: without an exit code, its status reads "running" instead of "not started".
+
+    ``technical_detail`` holds facts meant for the details and the --log file only, such as object IDs: they follow
+    the cause line in the detail, filtered like the rest, and never enter the cause shown on screen.
+
+    ``process_id`` names the process the failure comes from, when there is one, so an operation console can tell that
+    it already showed this failure and must not repeat it.
+
+    ``ended_before_failure`` says that the process named had already ended normally when the failure happened, such
+    as the cleanup of the copies a comparison tool used: the status line then does not read as that process's failure
+    nor as a process that never started.
+    """
+
+    operation: str
+    repository_dir: str
+    program_filename: str
+    arguments: list[str]
+    cause: str
+    stderr: bytes = b""
+    return_code: int | None = None
+    process_started: bool = False
+    technical_detail: str = ""
+    process_id: str = ""
+    ended_before_failure: bool = False
+
+    def prepare(self, environment_by_name: Mapping[str, str] | None = None) -> PreparedDiagnostic:
+        """Decode complete byte output before redacting secrets across fragment boundaries."""
+        text_filter = SensitiveTextFilter(environment_by_name)
+        redact = text_filter.redact
+        masked = mask_sensitive_arguments(self.arguments)
+        safe_arguments = masked.arguments
+        text_filter.remember_values(masked.secret_values)
+        safe_stderr = self._normalize_network(self.stderr, text_filter)
+        safe_cause = self._normalize_network(self.cause.encode("utf-8", "replace"), text_filter)
+
+        if self.ended_before_failure:
+            return_text = EXIT_STATUS_ENDED_BEFORE_FAILURE
+        elif self.return_code is not None:
+            return_text = str(self.return_code)
+        elif self.process_started:
+            return_text = EXIT_STATUS_RUNNING
+        else:
+            return_text = EXIT_STATUS_NOT_STARTED
+        technical_text = ""
+        safe_technical_text = ""
+        if self.technical_detail:
+            technical_text = f"{DIAGNOSTIC_TECHNICAL_DETAIL_HEADER}\n{self.technical_detail}\n"
+            safe_technical_text = f"{DIAGNOSTIC_TECHNICAL_DETAIL_HEADER}\n{self._normalize_network(self.technical_detail.encode('utf-8', 'replace'), text_filter)}\n"
+        detail = (
+            f"{DIAGNOSTIC_OPERATION_PREFIX}{self.operation}\nRepository: {self.repository_dir}\nExecutable: {self.program_filename}\n"
+            f"Arguments: {self.arguments!r}\nExit status: {return_text}\n{DIAGNOSTIC_CAUSE_PREFIX}{self.cause}\n{technical_text}"
+            f"{DIAGNOSTIC_STDERR_HEADER}\n{self.stderr.decode('utf-8', 'replace')}"
+        )
+        safe_detail = (
+            f"{DIAGNOSTIC_OPERATION_PREFIX}{self.operation}\nRepository: {self.repository_dir}\nExecutable: {self.program_filename}\n"
+            f"Arguments: {safe_arguments!r}\nExit status: {return_text}\n{DIAGNOSTIC_CAUSE_PREFIX}{safe_cause}\n{safe_technical_text}{DIAGNOSTIC_STDERR_HEADER}\n{safe_stderr}"
+        )
+        redacted_detail = redact(safe_detail)
+        return PreparedDiagnostic(redact(self.operation), redact(safe_cause), redacted_detail, redacted_detail != detail, redact(terminal_text(safe_stderr)),
+                                  self.return_code, self.process_id)
+
+    @staticmethod
+    def _normalize_network(content: bytes, text_filter: SensitiveTextFilter) -> str:
+        """Reuse live trace parsing when complete diagnostics contain curl body display lines."""
+        if not CURL_DATA_PATTERN.search(content):
+            return content.decode("utf-8", "replace")
+        records = NetworkTextStream(text_filter).feed(content, final=True)
+        return "\n".join(text for _kind, text in records)
+
+
+class DiagnosticLog:
+    """Write UTF-8 diagnostics into a unique session directory only when requested."""
+
+    def __init__(self, enabled: bool, base_dir: str, now: datetime | None = None) -> None:
+        """Create an exclusive timestamped file without colliding with another launch."""
+        self.log_filename: str | None = None
+        self.stream: TextIO | None = None
+        self.warning = ""
+        self.resource = ExitStack()
+        # Lines also come from background tasks, such as the search of the credential manager: one writer at a time
+        # keeps lines whole. Reentrant, since a write that fails closes the log under the same lock.
+        self.lock = threading.RLock()
+        if not enabled:
+            return
+        try:
+            session_dir = os.path.join(base_dir, "diagnostic", uuid.uuid4().hex)
+            os.makedirs(session_dir, exist_ok=False)
+            basename = (now or datetime.now(UTC).astimezone()).strftime("%Y%m%d-%H%M%S_gitpane.log")
+            self.log_filename = os.path.join(session_dir, basename)
+            # The file context spans the application lifetime and closes with the owning ExitStack.
+            self.stream = self.resource.enter_context(open(self.log_filename, "x", encoding="utf-8", newline="\n"))  # noqa: SIM115
+        except OSError as error:
+            self.warning = self._warning(error)
+
+    def write(self, diagnostic: PreparedDiagnostic) -> None:
+        """Write only the prepared details, making a disk error non-fatal to Git reads."""
+        self.write_line(diagnostic.detail)
+
+    def write_line(self, text: str) -> None:
+        """Write one already-filtered line, for records that are not error diagnostics; a disk error stays non-fatal.
+
+        Safe from any thread: a background task may write while the window's thread writes process lines.
+        """
+        with self.lock:
+            if self.stream is None:
+                return
+            try:
+                self.stream.write(text + "\n")
+                self.stream.flush()
+            except OSError as error:
+                self.warning = self._warning(error)
+                self.close()
+
+    def close(self) -> None:
+        """Flush and release this session's handler without touching another log; safe from any thread."""
+        with self.lock:
+            stream = self.stream
+            self.stream = None
+            if stream is None:
+                return
+            try:
+                self.resource.close()
+            except OSError as error:
+                self.warning = self._warning(error)
+
+    def _warning(self, error: OSError) -> str:
+        safe_error = Diagnostic("Log", "", "", [], str(error)).prepare().cause
+        return f"Could not write the log : {safe_error}. Error details remain available in the dialogs."