blot-git 0.3.0__tar.gz → 0.3.1__tar.gz

{blot_git-0.3.0 → blot_git-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: blot-git
-Version: 0.3.0
+Version: 0.3.1
 Summary: A pre-commit check that has read your project's history: warns when a change removes a fix, re-adds a fixed bug, or forgets a file that always changes too.
 Project-URL: Homepage, https://github.com/Sym-jay/Blot
 Project-URL: Issues, https://github.com/Sym-jay/Blot/issues

{blot_git-0.3.0 → blot_git-0.3.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "blot-git"
-version = "0.3.0"
+version = "0.3.1"
 description = "A pre-commit check that has read your project's history: warns when a change removes a fix, re-adds a fixed bug, or forgets a file that always changes too."
 readme = "docs/PYPI.md"  # text-only; the GitHub README's relative Sora images break on PyPI
 license = { text = "MIT" }

{blot_git-0.3.0 → blot_git-0.3.1}/src/blot/__init__.py

@@ -1,3 +1,3 @@
 """blot: history-aware pre-commit checks."""
 
-__version__ = "0.3.0"
+__version__ = "0.3.1"

{blot_git-0.3.0 → blot_git-0.3.1}/src/blot/index.py

@@ -10,9 +10,11 @@ Targets: under 60 s for 50k commits on a laptop; incremental refresh under 1 s.
 
 from __future__ import annotations
 
+import os
 import re
 import subprocess
 import time
+from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass
 
 from blot.classify import CommitInfo, fix_severity, is_refactor, linked_issues
@@ -85,12 +87,43 @@ def _head(repo: str, rev: str = "HEAD") -> str | None:
         return None
 
 
+_FORMAT = f"--format={_RECORD}%H{_FIELD}%aI{_FIELD}%s{_FIELD}%b{_FIELD}"
+# Most of indexing is git computing --numstat line counts, one commit after another. Big histories
+# are split into slices that separate git processes read in parallel (Django: 34k commits, ~170 s
+# serially). The result is the same set of commits; order doesn't matter to the index.
+_PARALLEL_FROM = 2000
+
+
+def _workers() -> int:
+    return max(1, min(8, (os.cpu_count() or 2) - 1))
+
+
+def _git_stdin(repo: str, args: list[str], revs: list[str]) -> str:
+    done = subprocess.run(["git", "-C", repo, *args], input="\n".join(revs) + "\n", capture_output=True,
+                          text=True, encoding="utf-8", errors="replace", check=True)
+    return done.stdout
+
+
+def _read_history(repo: str, rev: str) -> list[tuple[CommitInfo, str, list[str]]]:
+    args = ["log", "--no-walk=unsorted", "--stdin", "-M", "--numstat", _FORMAT]
+    shas = run_git(repo, "rev-list", "--no-merges", rev).split()
+    if len(shas) < _PARALLEL_FROM:
+        return _parse_log(_git_stdin(repo, args, shas)) if shas else []
+    # interleaved slices have similar sizes; never pass an empty one (git log would fall back to HEAD)
+    slices = [part for i in range(_workers() * 2) if (part := shas[i::_workers() * 2])]
+    with ThreadPoolExecutor(_workers()) as pool:
+        outs = pool.map(lambda part: _git_stdin(repo, args, part), slices)
+        return [commit for out in outs for commit in _parse_log(out)]
+
+
 def _removed_lines(repo: str, shas: list[str]) -> list[tuple[str, str, str]]:
     """(line_hash, path, sha) for every meaningful line the given commits deleted."""
+    batches = [shas[i : i + _SHOW_BATCH] for i in range(0, len(shas), _SHOW_BATCH)]
+    show = ["show", f"--format={_RECORD}%H", "-U0", "--no-color", "--no-ext-diff", "-M"]
+    with ThreadPoolExecutor(_workers()) as pool:
+        outs = list(pool.map(lambda batch: run_git(repo, *show, *batch), batches))
     rows: list[tuple[str, str, str]] = []
-    for i in range(0, len(shas), _SHOW_BATCH):
-        batch = shas[i : i + _SHOW_BATCH]
-        out = run_git(repo, "show", f"--format={_RECORD}%H", "-U0", "--no-color", "--no-ext-diff", "-M", *batch)
+    for out in outs:
         for chunk in out.split(_RECORD)[1:]:
             sha, _, diff_text = chunk.partition("\n")
             for change in parse_unified_diff(diff_text):
@@ -128,9 +161,7 @@ def build_index(
         return IndexResult(0, 0, time.perf_counter() - start, rebuild)
 
     rev = head if rebuild else f"{last}..{head}"
-    fmt = f"--format={_RECORD}%H{_FIELD}%aI{_FIELD}%s{_FIELD}%b{_FIELD}"
-    log = run_git(repo, "log", "--no-merges", "-M", "--numstat", fmt, rev)
-    parsed = _parse_log(log)
+    parsed = _read_history(repo, rev)
 
     fix_shas = []
     for info, date, paths in parsed:

{blot_git-0.3.0 → blot_git-0.3.1}/tests/test_index.py

@@ -66,6 +66,20 @@ def test_index_db_lives_in_git_dir(make_repo):
     assert os.path.exists(repo / ".git" / "blot.db")
 
 
+def test_parallel_history_read_matches_serial(make_repo, tmp_path, monkeypatch):
+    import blot.index as ix
+
+    repo = make_repo(history())
+    build_index(str(repo), db_path=str(tmp_path / "s.db"))
+    monkeypatch.setattr(ix, "_PARALLEL_FROM", 1)  # force slicing even for 3 commits
+    monkeypatch.setattr(ix, "_workers", lambda: 3)  # 6 slices for 3 commits: some would be empty
+    build_index(str(repo), db_path=str(tmp_path / "p.db"))
+    a, b = Store.open(str(repo), str(tmp_path / "s.db")), Store.open(str(repo), str(tmp_path / "p.db"))
+    for table in ("commits", "fix_removed_lines", "file_commits", "cochange"):
+        rows = lambda s: sorted(s.conn.execute(f"SELECT * FROM {table}").fetchall())  # noqa: E731
+        assert rows(a) == rows(b), table
+
+
 def test_empty_repo(tmp_path):
     subprocess.run(["git", "init", "-q", str(tmp_path)], check=True)
     assert build_index(str(tmp_path)).commits == 0