gitfence 0.1.0__py3-none-any.whl

fence/eval/backtest.py

@@ -0,0 +1,271 @@
+"""Removal-Regret Backtest (D1): replay history and score Fence against what happened.
+
+For each first-parent commit C that removes code, a removal block is **regretted**
+when, within the next `window` commits, C is reverted or the removed code (by
+fingerprint) is added back. Fence then checks C's diff with only the history
+visible at C's parent (Context.as_of), so no future knowledge leaks into the
+verdict. A block is **warned** when Fence's verdict is KEEP or VERIFY.
+
+Regret labels are a lower bound: a warned removal that nobody restored may still
+have been risky. So precision here is conservative, and the report says so.
+"""
+
+from __future__ import annotations
+
+import math
+import secrets
+import statistics
+import time
+from dataclasses import dataclass, field
+
+from fence.anchors.fingerprint import containment, shingles
+from fence.check.diff import DiffError, FileDiff, parse_diff
+from fence.check.engine import KEEP, VERIFY, check
+from fence.context import Context
+from fence.decisions.build import TEST_PATH
+from fence.git.runner import GitError
+from fence.lang import plugin_for, tokens_for
+
+MIN_TOKENS = 8
+RESTORE_CONTAINMENT = 0.8
+MAX_FILES_PER_COMMIT = 50
+MAX_DIFF_BYTES = 1_000_000
+
+
+@dataclass
+class _Commit:
+    sha: str
+    parent: str | None
+    time: int
+    diff: str
+    files: list[FileDiff] = field(default_factory=list)
+    added: dict[str, frozenset[int]] = field(default_factory=dict)  # per path
+
+    @property
+    def all_added(self) -> frozenset[int]:
+        return frozenset().union(*self.added.values()) if self.added else frozenset()
+
+
+@dataclass
+class BlockOutcome:
+    sha: str
+    path: str
+    start: int
+    regretted: bool
+    regret_kind: str | None  # "reverted" | "restored"
+    lead_days: float | None
+    verdict: str  # KEEP | VERIFY | CLEAR
+    risk: int
+    confidence: float | None
+    reason: str | None
+    kind: str | None = None
+    grade: str | None = None
+    dtype: str | None = None
+    strength: int = 0
+    lost: int = 0
+    factors: tuple[str, ...] = ()  # risk factor ids, for error analysis
+
+
+@dataclass
+class BacktestReport:
+    repo: str
+    commits_evaluated: int = 0
+    commits_with_removals: int = 0
+    blocks: int = 0
+    regretted: int = 0
+    warned: int = 0
+    keep: int = 0
+    tp_warned: int = 0
+    tp_keep: int = 0
+    commits_warned: int = 0
+    latencies: list[float] = field(default_factory=list)
+    leads: list[float] = field(default_factory=list)
+    outcomes: list[BlockOutcome] = field(default_factory=list)
+    errors: int = 0
+    seconds: float = 0.0
+
+    @staticmethod
+    def _ratio(a: int, b: int) -> float | None:
+        return round(a / b, 3) if b else None
+
+    def summary(self) -> dict:
+        return {
+            "repo": self.repo,
+            "commits_evaluated": self.commits_evaluated,
+            "removal_blocks": self.blocks,
+            "regretted_blocks": self.regretted,
+            "warned_blocks": self.warned,
+            "keep_blocks": self.keep,
+            "precision_warn": self._ratio(self.tp_warned, self.warned),
+            "recall_warn": self._ratio(self.tp_warned, self.regretted),
+            "precision_keep": self._ratio(self.tp_keep, self.keep),
+            "recall_keep": self._ratio(self.tp_keep, self.regretted),
+            "warned_commits_per_1k": (round(1000 * self.commits_warned / self.commits_evaluated, 1)
+                                      if self.commits_evaluated else None),
+            "warning_blocks_per_1k_commits": (round(1000 * self.warned / self.commits_evaluated, 1)
+                                              if self.commits_evaluated else None),
+            "median_check_seconds": (round(statistics.median(self.latencies), 3)
+                                     if self.latencies else None),
+            # Nearest-rank percentile: never below the median, even for small samples.
+            "p90_check_seconds": (round(sorted(self.latencies)[
+                min(len(self.latencies) - 1, math.ceil(0.9 * len(self.latencies)) - 1)], 3)
+                                  if self.latencies else None),
+            "median_lead_days": round(statistics.median(self.leads), 1) if self.leads else None,
+            "calibration": self.calibration(),
+            "calibration_monotonic": self.calibration_monotonic(),
+            "errors": self.errors,
+            "seconds": round(self.seconds, 1),
+        }
+
+    def calibration(self) -> list[dict]:
+        """Observed regret rate of ALL blocks per bin of the shipped calibration table.
+
+        On repos the table was not fitted on, this is the held-out check: rates should
+        rise with the bins, and the observed rate should be near the table's.
+        """
+        from fence.scoring import calibration
+
+        bins, _ = calibration.load()
+        out = []
+        for b in bins:
+            xs = [o for o in self.outcomes if b.risk_min <= o.risk <= b.risk_max]
+            k = sum(o.regretted for o in xs)
+            out.append({"risk": f"{b.risk_min}-{b.risk_max}", "table_rate": round(b.rate, 4),
+                        "blocks": len(xs), "regretted": k,
+                        "observed_rate": self._ratio(k, len(xs))})
+        return out
+
+    def calibration_monotonic(self) -> bool | None:
+        rates = [c["observed_rate"] for c in self.calibration() if c["blocks"]]
+        if len(rates) < 2:
+            return None
+        return all(a < b for a, b in zip(rates, rates[1:], strict=False))
+
+
+def _stream_commits(ctx: Context, rev: str, limit: int) -> list[_Commit]:
+    nonce = secrets.token_hex(6)
+    marker = f"\x1e{nonce}"
+    args = ["log", "--first-parent", "--diff-merges=first-parent", "-p", "--unified=0",
+            "--no-color", "--no-ext-diff", "--no-textconv", "-M50%", "--src-prefix=a/",
+            "--dst-prefix=b/", f"--max-count={limit}", f"--format=%x1e{nonce}%H %P%n%ct",
+            ctx.repo.verify_ref(rev), "--"]
+    text = "".join(ctx.repo.stream(args, timeout=ctx.cfg.performance.log_timeout_seconds))
+    commits = []
+    for rec in text.split(marker):
+        if not rec.strip():
+            continue
+        head, _, rest = rec.partition("\n")
+        ts, _, diff = rest.partition("\n")
+        parts = head.split()
+        if not parts or not ts.strip().isdigit():
+            continue
+        commits.append(_Commit(parts[0], parts[1] if len(parts) > 1 else None, int(ts),
+                               diff if len(diff) <= MAX_DIFF_BYTES else ""))
+    commits.reverse()  # oldest first
+    for c in commits:
+        try:
+            c.files = parse_diff(c.diff) if c.diff else []
+        except DiffError:
+            c.files = []
+        for f in c.files:
+            if f.added_lines:
+                c.added[f.path] = shingles(tokens_for(f.path, "\n".join(f.added_lines)))
+    return commits
+
+
+def backtest(ctx: Context, *, rev: str = "HEAD", max_commits: int = 300, window: int = 200,
+             deep: bool = True, progress=None) -> BacktestReport:
+    started = time.monotonic()
+    rep = BacktestReport(repo=str(ctx.repo.root))
+    commits = _stream_commits(ctx, rev, max_commits + window)
+    # The oldest `max_commits` of the range, so each has up to `window` later commits in
+    # which a regret can show up. The newest ones have shorter windows (right-censored).
+    evaluate = commits[:max_commits]
+    index_of = {c.sha: i for i, c in enumerate(commits)}
+    for n, c in enumerate(evaluate):
+        if progress and n % 25 == 0:
+            progress(n, len(evaluate))
+        rep.commits_evaluated += 1
+        if not c.parent or not c.files or len(c.files) > MAX_FILES_PER_COMMIT:
+            continue
+        row = ctx.store.commit(c.sha)
+        if row and (row.format_only or row.bulk):
+            continue
+        blocks = _removal_blocks(c)
+        if not blocks:
+            continue
+        rep.commits_with_removals += 1
+        i = index_of[c.sha]
+        later = commits[i + 1 : i + 1 + window]
+        reverted_by = set(ctx.store.reverts_of(c.sha))
+        labels = {}
+        for path, b, sh in blocks:
+            kind, lead = None, None
+            for lc in later:
+                if lc.sha in reverted_by:
+                    kind, lead = "reverted", (lc.time - c.time) / 86400
+                    break
+                # Same file only: boilerplate re-added elsewhere is not a restoration.
+                if containment(sh, lc.added.get(path, frozenset())) >= RESTORE_CONTAINMENT:
+                    kind, lead = "restored", (lc.time - c.time) / 86400
+                    break
+            labels[(path, b.start)] = (kind, lead)
+
+        ctx.as_of = ctx.store.commit(c.parent).committer_time if ctx.store.commit(c.parent) \
+            else c.time - 1
+        t0 = time.monotonic()
+        try:
+            res = check(ctx, diff_text=c.diff, base=c.parent, deep=deep, include_clear=True)
+        except (DiffError, GitError):
+            rep.errors += 1
+            continue
+        finally:
+            ctx.as_of = None
+        rep.latencies.append(time.monotonic() - t0)
+        by_block = {(f.path, f.start): f for f in res.findings}
+        warned_here = False
+        for (path, start), (kind, lead) in labels.items():
+            f = by_block.get((path, start))
+            verdict = f.verdict if f and f.verdict in (KEEP, VERIFY) else "CLEAR"
+            o = BlockOutcome(c.sha, path, start, kind is not None, kind, lead, verdict,
+                             f.risk.value if f else 0, f.risk.confidence if f else None,
+                             f.decision.reason_text if f else None,
+                             f.kind if f else None, f.decision.grade if f else None,
+                             f.decision.type if f else None,
+                             f.decision.strength.value if f else 0,
+                             len(f.lost_signals) if f else 0,
+                             tuple(x.id for x in f.risk.factors) if f else ())
+            rep.outcomes.append(o)
+            rep.blocks += 1
+            rep.regretted += o.regretted
+            if verdict in (KEEP, VERIFY):
+                warned_here = True
+                rep.warned += 1
+                rep.tp_warned += o.regretted
+                if o.regretted and lead is not None:
+                    rep.leads.append(lead)
+            if verdict == KEEP:
+                rep.keep += 1
+                rep.tp_keep += o.regretted
+        rep.commits_warned += warned_here
+    rep.seconds = time.monotonic() - started
+    return rep
+
+
+def _removal_blocks(c: _Commit):
+    out = []
+    same_commit_added = c.all_added
+    for f in c.files:
+        # Source code only: tests, docs and data are not fences.
+        if (f.old_path is None or f.binary or plugin_for(f.old_path) is None
+                or TEST_PATH.search(f.old_path)):
+            continue
+        for b in f.blocks:
+            toks = tokens_for(f.old_path, "\n".join(t for _, t in b.removed))
+            if len(toks) < MIN_TOKENS:
+                continue
+            sh = shingles(toks)
+            if containment(sh, same_commit_added) >= RESTORE_CONTAINMENT:
+                continue  # moved or rewritten in place, not removed
+            out.append((f.old_path, b, sh))
+    return out

fence/git/blame.py

@@ -0,0 +1,219 @@
+"""`git blame` against an explicit base revision, parsed from --line-porcelain.
+
+Two things here are security decisions, not style:
+- `.git-blame-ignore-revs` is read from the BASE revision. In CI the working tree is
+  the PR head, so reading it from disk would let a PR hide the commit that explains
+  the code it deletes.
+- `--ignore-revs-file=` (empty) is always passed first. It clears any list git would
+  otherwise load from the repo's `blame.ignoreRevsFile` config, which points at the
+  working tree.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+from fence.git.runner import GitError, Repo, UnsafeInput
+
+IGNORE_REVS_PATH = ".git-blame-ignore-revs"
+MAX_IGNORE_REVS = 500
+MAX_RANGES = 200
+MAX_LINES = 5000
+
+_SHA = re.compile(r"[0-9a-f]{40}|[0-9a-f]{64}")
+_HEADER = re.compile(r"^([0-9a-f]{40}|[0-9a-f]{64}) (\d+) (\d+)(?: \d+)?$")
+
+
+@dataclass(frozen=True)
+class BlameLine:
+    sha: str
+    orig_line: int  # line number in the commit that introduced it
+    final_line: int  # line number at the base revision
+    author: str
+    author_mail: str
+    author_time: int  # unix seconds
+    author_tz: str
+    summary: str
+    filename: str  # path in `sha`; differs from the input path when -C found a copy
+    previous_sha: str | None  # None on a root commit
+    previous_path: str | None
+    boundary: bool
+    content: str
+    # git 2.43 does not mark lines it could not move past an ignored commit, even
+    # with blame.markUnblamableLines set, so we derive it: the line still points
+    # at a commit we asked git to ignore.
+    ignored: bool = False
+
+
+_IGNORE_CACHE: dict[tuple[str, str], list[str]] = {}
+
+
+def read_ignore_revs(repo: Repo, base_sha: str) -> list[str]:
+    """Full SHAs listed in .git-blame-ignore-revs at `base_sha`. Missing file -> [].
+
+    Memoized per (repo, commit): a commit's tree never changes. Returns a copy.
+    """
+    key = (str(repo.root), base_sha)
+    if key not in _IGNORE_CACHE:
+        if len(_IGNORE_CACHE) > 10_000:
+            _IGNORE_CACHE.clear()
+        _IGNORE_CACHE[key] = _read_ignore_revs(repo, base_sha)
+    return list(_IGNORE_CACHE[key])
+
+
+def _read_ignore_revs(repo: Repo, base_sha: str) -> list[str]:
+    listed = repo.run(["ls-tree", "--name-only", base_sha, "--", IGNORE_REVS_PATH]).strip()
+    if not listed:
+        return []
+    text = repo.read_blob(base_sha, IGNORE_REVS_PATH, max_bytes=100_000)
+    revs: list[str] = []
+    for raw in text.split("\n"):
+        line = raw.split("#", 1)[0].strip().lower()
+        # Abbreviated SHAs and ref names are skipped: they could resolve differently
+        # over time, and the file is repository text, so it is untrusted.
+        if _SHA.fullmatch(line) and line not in revs:
+            revs.append(line)
+        if len(revs) >= MAX_IGNORE_REVS:
+            break
+    return revs
+
+
+def blame(
+    repo: Repo,
+    base: str,
+    path: str,
+    ranges: Sequence[tuple[int, int]],
+    *,
+    extra_ignore: Sequence[str] = (),
+) -> list[BlameLine]:
+    """Blame the 1-based inclusive `ranges` of `path` at `base`. One git call per file.
+
+    `extra_ignore` adds full SHAs to skip, such as formatting-only commits found by
+    the indexer. They come from Fence's own analysis, not from repository text.
+    """
+    base_sha = repo.verify_ref(base)
+    rel = repo.safe_relpath(path)
+    _check_ranges(ranges)
+    ignore = read_ignore_revs(repo, base_sha)
+    for sha in extra_ignore:
+        if _SHA.fullmatch(sha) and sha not in ignore and len(ignore) < MAX_IGNORE_REVS * 2:
+            ignore.append(sha)
+
+    args = ["blame", "-w", "-C", "--line-porcelain", "--ignore-revs-file="]
+    for sha in ignore:
+        args += ["--ignore-rev", sha]
+    for start, end in ranges:
+        args += ["-L", f"{start},{end}"]
+    args += [base_sha, "--", rel]
+
+    lines = parse_line_porcelain(repo.run(args))
+    expected = _distinct_line_count(ranges)
+    if len(lines) != expected:
+        raise GitError(f"blame returned {len(lines)} lines, expected {expected}")
+    ignored = set(ignore)
+    return [_with_ignored(bl, bl.sha in ignored) for bl in lines]
+
+
+def parse_line_porcelain(output: str) -> list[BlameLine]:
+    """Parse `git blame --line-porcelain`. Raises GitError on malformed or truncated output."""
+    result: list[BlameLine] = []
+    fields: dict[str, str] | None = None
+    header: tuple[str, int, int] | None = None
+    # Split on "\n" only: str.splitlines() would also split on \x0b, \x1c, 
 etc.
+    # that can legitimately appear inside a line of code.
+    rows = output.split("\n")
+    if rows and rows[-1] == "":
+        rows.pop()
+    for row in rows:
+        if header is None:
+            m = _HEADER.match(row)
+            if not m:
+                raise GitError(f"unexpected blame header: {row[:80]!r}")
+            header = (m.group(1), int(m.group(2)), int(m.group(3)))
+            fields = {}
+            continue
+        if fields is None:  # unreachable: always set together with header
+            raise GitError("blame parser lost its record state")
+        if row.startswith("\t"):
+            result.append(_build(header, fields, row[1:]))
+            header, fields = None, None
+            continue
+        key, _, value = row.partition(" ")
+        if key == "boundary":
+            fields["boundary"] = "1"
+        elif key in _KEPT_KEYS:
+            fields[key] = value
+        # committer-*, unknown keys: skipped on purpose.
+    if header is not None:
+        raise GitError("truncated blame output: header without content line")
+    return result
+
+
+_KEPT_KEYS = {
+    "author",
+    "author-mail",
+    "author-time",
+    "author-tz",
+    "summary",
+    "filename",
+    "previous",
+}
+
+
+def _build(header: tuple[str, int, int], f: dict[str, str], content: str) -> BlameLine:
+    sha, orig, final = header
+    for required in ("author", "author-time", "filename"):
+        if required not in f:
+            raise GitError(f"blame record for {sha[:12]} is missing {required!r}")
+    prev_sha = prev_path = None
+    if "previous" in f:
+        prev_sha, _, prev_path = f["previous"].partition(" ")
+        if not _SHA.fullmatch(prev_sha) or not prev_path:
+            raise GitError(f"malformed 'previous' in blame record for {sha[:12]}")
+    try:
+        author_time = int(f["author-time"])
+    except ValueError:
+        raise GitError(f"malformed author-time for {sha[:12]}") from None
+    return BlameLine(
+        sha=sha,
+        orig_line=orig,
+        final_line=final,
+        author=f["author"],
+        author_mail=f.get("author-mail", "").strip("<>"),
+        author_time=author_time,
+        author_tz=f.get("author-tz", ""),
+        summary=f.get("summary", ""),
+        filename=f["filename"],
+        previous_sha=prev_sha,
+        previous_path=prev_path,
+        boundary="boundary" in f,
+        content=content,
+    )
+
+
+def _with_ignored(bl: BlameLine, ignored: bool) -> BlameLine:
+    if not ignored:
+        return bl
+    return BlameLine(**{**bl.__dict__, "ignored": True})
+
+
+def _check_ranges(ranges: Sequence[tuple[int, int]]) -> None:
+    if not ranges:
+        raise UnsafeInput("no line ranges given")
+    if len(ranges) > MAX_RANGES:
+        raise UnsafeInput(f"too many ranges ({len(ranges)} > {MAX_RANGES})")
+    for start, end in ranges:
+        if not (isinstance(start, int) and isinstance(end, int)) or start < 1 or end < start:
+            raise UnsafeInput(f"bad line range: {start},{end}")
+    if _distinct_line_count(ranges) > MAX_LINES:
+        raise UnsafeInput(f"too many lines to blame (> {MAX_LINES})")
+
+
+def _distinct_line_count(ranges: Sequence[tuple[int, int]]) -> int:
+    # git merges overlapping -L ranges, so count distinct lines, not the sum.
+    seen: set[int] = set()
+    for start, end in ranges:
+        seen.update(range(start, end + 1))
+    return len(seen)

fence/git/history.py

@@ -0,0 +1,171 @@
+"""Stream `git log --raw --numstat -z` and parse it into commits with file changes.
+
+Commit messages are untrusted and may contain any byte except NUL, including our
+field separators. So every marker carries a random per-run nonce: a message
+cannot forge a record boundary it cannot predict.
+"""
+
+from __future__ import annotations
+
+import re
+import secrets
+from collections.abc import Iterable, Iterator
+from dataclasses import dataclass, field
+
+from fence.git.runner import GitError, Repo
+
+_SHA = re.compile(r"[0-9a-f]{40}|[0-9a-f]{64}")
+
+
+@dataclass
+class RawChange:
+    path: str
+    old_path: str | None
+    status: str  # A C D M R T U X (git --raw)
+    similarity: int | None
+    added: int | None = None  # None for binary files
+    deleted: int | None = None
+
+
+@dataclass
+class LogEntry:
+    sha: str
+    parents: tuple[str, ...]
+    author_name: str
+    author_email: str
+    author_time: int
+    committer_time: int
+    subject: str
+    body: str
+    changes: list[RawChange] = field(default_factory=list)
+
+
+class LogFormat:
+    def __init__(self) -> None:
+        nonce = secrets.token_hex(8)
+        self.start = f"\x1e{nonce}S"
+        self.sep = f"\x1f{nonce}F"
+        self.end = f"\x1d{nonce}E"
+        fields = ["%H", "%P", "%an", "%ae", "%at", "%ct", "%s", "%b"]
+        f_sep = f"%x1f{nonce}F"
+        self.arg = f"--format=%x1e{nonce}S" + f_sep.join(fields) + f"%x1d{nonce}E"
+
+
+def log_args(fmt: LogFormat, *, rename_threshold: int = 50) -> list[str]:
+    return [
+        "log",
+        "--no-color",
+        "--no-ext-diff",
+        "--no-textconv",
+        "--diff-merges=off",  # merges carry no diff; their PR mapping comes from the subject
+        "--raw",
+        "--numstat",
+        "-z",
+        "--no-abbrev",
+        f"-M{rename_threshold}%",
+        fmt.arg,
+    ]
+
+
+def iter_log(
+    repo: Repo, revs: Iterable[str], *, fmt: LogFormat | None = None, extra: Iterable[str] = (),
+    rename_threshold: int = 50, timeout: float = 600.0,
+) -> Iterator[LogEntry]:
+    """Yield commits reachable from `revs` (already-validated SHAs or `--not`), oldest first."""
+    fmt = fmt or LogFormat()
+    args = [*log_args(fmt, rename_threshold=rename_threshold), "--reverse", *extra, *revs, "--"]
+    yield from parse_stream(repo.stream(args, timeout=timeout), fmt)
+
+
+def parse_stream(chunks: Iterable[str], fmt: LogFormat) -> Iterator[LogEntry]:
+    buf = ""
+    for chunk in chunks:
+        buf += chunk
+        # Everything before the last start marker is complete.
+        last = buf.rfind(fmt.start)
+        if last <= 0:
+            continue
+        complete, buf = buf[:last], buf[last:]
+        yield from _parse_records(complete, fmt)
+    if buf:
+        yield from _parse_records(buf, fmt)
+
+
+def _parse_records(text: str, fmt: LogFormat) -> Iterator[LogEntry]:
+    for record in text.split(fmt.start):
+        if record.strip("\x00\n") == "":
+            continue
+        yield parse_record(record, fmt)
+
+
+def parse_record(record: str, fmt: LogFormat) -> LogEntry:
+    header, end, rest = record.partition(fmt.end)
+    if not end:
+        raise GitError("truncated git log record")
+    parts = header.split(fmt.sep)
+    if len(parts) != 8 or not _SHA.fullmatch(parts[0]):
+        raise GitError(f"unexpected git log header: {header[:80]!r}")
+    sha, parents, an, ae, at, ct, subject, body = parts
+    if not (at.isdigit() and ct.isdigit()):
+        raise GitError(f"bad timestamps in {sha[:12]}")
+    entry = LogEntry(
+        sha=sha,
+        parents=tuple(p for p in parents.split(" ") if p),
+        author_name=an,
+        author_email=ae,
+        author_time=int(at),
+        committer_time=int(ct),
+        subject=subject,
+        body=body.strip("\n"),
+    )
+    entry.changes = _parse_changes(rest)
+    return entry
+
+
+def _parse_changes(rest: str) -> list[RawChange]:
+    tokens = rest.split("\x00")
+    raw: dict[str, RawChange] = {}
+    order: list[str] = []
+    i = 0
+    n = len(tokens)
+    while i < n:
+        tok = tokens[i]
+        head = tok.lstrip("\n")
+        if head == "":
+            i += 1
+            continue
+        if head.startswith(":"):
+            fields = head[1:].split(" ")
+            if len(fields) < 5 or i + 1 >= n:
+                raise GitError(f"malformed raw diff entry: {head[:60]!r}")
+            status = fields[4]
+            letter = status[:1]
+            score = int(status[1:]) if status[1:].isdigit() else None
+            if letter in ("R", "C"):
+                old, new = tokens[i + 1], tokens[i + 2]
+                i += 3
+            else:
+                old, new = None, tokens[i + 1]
+                i += 2
+            raw[new] = RawChange(new, old, letter, score)
+            order.append(new)
+            continue
+        cols = head.split("\t")
+        if len(cols) != 3:
+            raise GitError(f"malformed numstat entry: {head[:60]!r}")
+        added, deleted, path = cols
+        if path == "":  # rename: old and new follow as separate tokens
+            if i + 2 >= n:
+                raise GitError("truncated numstat rename entry")
+            path = tokens[i + 2]
+            i += 3
+        else:
+            i += 1
+        change = raw.get(path)
+        if change is None:
+            change = RawChange(path, None, "M", None)
+            raw[path] = change
+            order.append(path)
+        change.added = int(added) if added.isdigit() else None
+        change.deleted = int(deleted) if deleted.isdigit() else None
+    return [raw[p] for p in order]