gitmole 0.6.9__tar.gz → 0.6.11__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gitmole-0.6.9 → gitmole-0.6.11}/PKG-INFO +2 -2
- {gitmole-0.6.9 → gitmole-0.6.11}/README.md +1 -1
- {gitmole-0.6.9 → gitmole-0.6.11}/gitmole/__init__.py +1 -1
- {gitmole-0.6.9 → gitmole-0.6.11}/gitmole/filetypes.py +38 -6
- {gitmole-0.6.9 → gitmole-0.6.11}/gitmole/findings.py +6 -2
- {gitmole-0.6.9 → gitmole-0.6.11}/gitmole/identity.py +12 -4
- {gitmole-0.6.9 → gitmole-0.6.11}/gitmole/leaks.py +48 -4
- {gitmole-0.6.9 → gitmole-0.6.11}/gitmole/load.py +6 -2
- {gitmole-0.6.9 → gitmole-0.6.11}/gitmole/maat.py +14 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/gitmole/render.py +14 -2
- {gitmole-0.6.9 → gitmole-0.6.11}/gitmole/run.py +15 -3
- {gitmole-0.6.9 → gitmole-0.6.11}/gitmole/watch.py +2 -1
- {gitmole-0.6.9 → gitmole-0.6.11}/gitmole.egg-info/PKG-INFO +2 -2
- {gitmole-0.6.9 → gitmole-0.6.11}/tests/test_filetypes.py +16 -5
- {gitmole-0.6.9 → gitmole-0.6.11}/tests/test_findings.py +21 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/tests/test_identity.py +8 -1
- {gitmole-0.6.9 → gitmole-0.6.11}/tests/test_leaks.py +42 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/tests/test_load.py +20 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/tests/test_maat.py +26 -2
- {gitmole-0.6.9 → gitmole-0.6.11}/tests/test_render.py +35 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/tests/test_run.py +27 -2
- {gitmole-0.6.9 → gitmole-0.6.11}/tests/test_watch.py +7 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/LICENSE +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/gitmole/__main__.py +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/gitmole/backtest.py +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/gitmole/banner.py +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/gitmole/blame.py +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/gitmole/clean.py +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/gitmole/cli.py +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/gitmole/coupling.py +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/gitmole/functions.py +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/gitmole/hotspots.py +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/gitmole/knowledge.py +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/gitmole/loss.py +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/gitmole/textfmt.py +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/gitmole/trend.py +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/gitmole.egg-info/SOURCES.txt +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/gitmole.egg-info/dependency_links.txt +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/gitmole.egg-info/entry_points.txt +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/gitmole.egg-info/requires.txt +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/gitmole.egg-info/top_level.txt +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/pyproject.toml +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/setup.cfg +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/tests/test_backtest.py +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/tests/test_banner.py +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/tests/test_blame.py +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/tests/test_clean.py +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/tests/test_cli.py +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/tests/test_coupling.py +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/tests/test_functions.py +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/tests/test_golden.py +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/tests/test_hotspots.py +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/tests/test_knowledge.py +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/tests/test_loss.py +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/tests/test_packaging.py +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/tests/test_textfmt.py +0 -0
- {gitmole-0.6.9 → gitmole-0.6.11}/tests/test_trend.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gitmole
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.11
|
|
4
4
|
Summary: Offline git repository analysis with a terminal report: hotspots, coupling, ownership, code age, secrets, repo health.
|
|
5
5
|
License: MIT
|
|
6
6
|
Project-URL: Homepage, https://github.com/antvinni/gitmole
|
|
@@ -55,7 +55,7 @@ Linux package names, the release binaries, `--plots` and the pip caveats:
|
|
|
55
55
|
```bash
|
|
56
56
|
gitmole . # the clone you are in
|
|
57
57
|
gitmole /path/to/clone # any local clone
|
|
58
|
-
gitmole owner/repo # clones
|
|
58
|
+
gitmole owner/repo # clones into a temp dir first, with gh or plain git
|
|
59
59
|
gitmole 'owner/*' # every non-archived repo of a user or org, one summary table
|
|
60
60
|
|
|
61
61
|
gitmole . --markdown report.md # the same report as a Markdown document
|
|
@@ -35,7 +35,7 @@ Linux package names, the release binaries, `--plots` and the pip caveats:
|
|
|
35
35
|
```bash
|
|
36
36
|
gitmole . # the clone you are in
|
|
37
37
|
gitmole /path/to/clone # any local clone
|
|
38
|
-
gitmole owner/repo # clones
|
|
38
|
+
gitmole owner/repo # clones into a temp dir first, with gh or plain git
|
|
39
39
|
gitmole 'owner/*' # every non-archived repo of a user or org, one summary table
|
|
40
40
|
|
|
41
41
|
gitmole . --markdown report.md # the same report as a Markdown document
|
|
@@ -50,20 +50,22 @@ def parse(spec):
|
|
|
50
50
|
return {t.strip().lstrip(".").lower() for t in spec.split(",") if t.strip()}
|
|
51
51
|
|
|
52
52
|
|
|
53
|
-
_TEST_PATH = re.compile(r"(^|/)(tests?|spec|specs|__tests__|testing
|
|
53
|
+
_TEST_PATH = re.compile(r"(^|/)(tests?|spec|specs|__tests__|testing|snapshots?|__snapshots__|[\w-]+[_-]tests?|tests?[_-][\w-]+)(/|$)"
|
|
54
|
+
r"|(^|/)(test_[^/]*|[^/]*_test\.[^/]+|[^/]*\.spec\.[^/]+|[^/]*\.test\.[^/]+|[^/]*\.snap)$", re.I)
|
|
54
55
|
|
|
55
56
|
|
|
56
57
|
def is_test_path(path: str) -> bool:
|
|
57
|
-
"""A test file or anything under a tests directory
|
|
58
|
+
"""A test file or anything under a tests directory (tests/, pending_tests/, e2e-tests/, test_utils/,
|
|
59
|
+
snapshots/ and .snap files): changes with every fix, so not a signal on its own."""
|
|
58
60
|
return bool(_TEST_PATH.search(path))
|
|
59
61
|
|
|
60
62
|
|
|
61
|
-
_DOC_PATH = re.compile(r"(^|/)docs?(/|$)|\.(md|markdown|rst|txt|adoc)$", re.I)
|
|
63
|
+
_DOC_PATH = re.compile(r"(^|/)docs?([-_][\w-]+)?(/|$)|\.(md|markdown|rst|txt|adoc)$", re.I)
|
|
62
64
|
|
|
63
65
|
|
|
64
66
|
def is_doc_path(path: str) -> bool:
|
|
65
|
-
"""Documentation: prose formats anywhere, or anything under docs
|
|
66
|
-
is far more often a template than a leak."""
|
|
67
|
+
"""Documentation: prose formats anywhere, or anything under docs/, doc/, docs_src/, docs-site/. A
|
|
68
|
+
key in a planning document or a tutorial is far more often a template than a leak."""
|
|
67
69
|
return bool(_DOC_PATH.search(path))
|
|
68
70
|
|
|
69
71
|
|
|
@@ -76,7 +78,7 @@ def is_sample_path(path: str) -> bool:
|
|
|
76
78
|
return bool(_SAMPLE_PATH.search(path))
|
|
77
79
|
|
|
78
80
|
|
|
79
|
-
_VENDOR_PATH = re.compile(r"(^|/)(_?vendor|vendored|node_modules|third_?party|external)(/|$)|^[^/]+/packages/", re.I)
|
|
81
|
+
_VENDOR_PATH = re.compile(r"(^|/)(_?vendor|vendored|node_modules|third_?party|external|deps)(/|$)|^[^/]+/packages/", re.I)
|
|
80
82
|
|
|
81
83
|
|
|
82
84
|
def is_vendor_path(path: str) -> bool:
|
|
@@ -98,6 +100,36 @@ def is_release_path(path: str) -> bool:
|
|
|
98
100
|
return name in _RELEASE_NAMES or name.endswith(".gemspec") or name.startswith(("changelog", "changes.", "history.", "news."))
|
|
99
101
|
|
|
100
102
|
|
|
103
|
+
_SOURCE_EXT = {"c", "cc", "cpp", "cxx", "m", "mm"}
|
|
104
|
+
_HEADER_EXT = {"h", "hh", "hpp", "hxx"}
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def is_header_pair(a: str, b: str) -> bool:
|
|
108
|
+
"""A C-family source file and its own header, same directory and stem: they change together by
|
|
109
|
+
construction, so the pair says nothing about a hidden dependency."""
|
|
110
|
+
if a == b:
|
|
111
|
+
return False
|
|
112
|
+
(da, sa, ea), (db, sb, eb) = _split(a), _split(b)
|
|
113
|
+
return da == db and sa == sb and ({ea, eb} & _SOURCE_EXT) and ({ea, eb} & _HEADER_EXT) and ea != eb
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _split(path: str) -> tuple:
|
|
117
|
+
head, _, name = path.rpartition("/")
|
|
118
|
+
stem, _, ext = name.rpartition(".")
|
|
119
|
+
return head, stem, ext.lower()
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def plumbing_paths(report: dict) -> set:
|
|
123
|
+
"""Files the change log showed to be release plumbing by behaviour rather than by name: nearly
|
|
124
|
+
every commit touching them changed a line or two (a version constant in __init__.py)."""
|
|
125
|
+
return {r["entity"] for r in report.get("plumbing") or []}
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def is_release(path: str, plumbing=frozenset()) -> bool:
|
|
129
|
+
"""is_release_path, or a path the change log showed to be plumbing (see plumbing_paths)."""
|
|
130
|
+
return is_release_path(path) or path in plumbing
|
|
131
|
+
|
|
132
|
+
|
|
101
133
|
# What a generated file says about itself in its first lines: protoc, ajv, code generators of every kind.
|
|
102
134
|
_GENERATED = re.compile(r"auto[- ]?generated|generated (by|from|file|code|automatically|with)|do not (edit|modify)|@generated|code generated", re.I)
|
|
103
135
|
GENERATED_HEAD_LINES = 5
|
|
@@ -176,7 +176,8 @@ def sizer_concerns(report: dict) -> list:
|
|
|
176
176
|
def hotspot_dominance(report: dict, ratio: float = 2.0, minimum: int = 20) -> list:
|
|
177
177
|
"""One source file takes most of the churn. Test files are left out: they change with everything.
|
|
178
178
|
So is release plumbing: a version file or a manifest changes on every release by design."""
|
|
179
|
-
|
|
179
|
+
plumb = filetypes.plumbing_paths(report)
|
|
180
|
+
revs = sorted((r for r in report.get("revisions") or [] if not (filetypes.is_test_path(r["entity"]) or filetypes.is_release(r["entity"], plumb))),
|
|
180
181
|
key=lambda r: -r["n-revs"])
|
|
181
182
|
if len(revs) < 2 or revs[0]["n-revs"] < minimum or revs[0]["n-revs"] < ratio * revs[1]["n-revs"]:
|
|
182
183
|
return []
|
|
@@ -194,6 +195,8 @@ def tight_coupling(report: dict, min_degree: int = 80, min_revs: int = 5) -> lis
|
|
|
194
195
|
pairs = [p for p in report.get("coupling") or [] if p["degree"] >= min_degree and p["average-revs"] >= min_revs
|
|
195
196
|
and not (filetypes.is_test_path(p["entity"]) or filetypes.is_test_path(p["coupled"]))
|
|
196
197
|
and not (filetypes.is_release_path(p["entity"]) and filetypes.is_release_path(p["coupled"]))
|
|
198
|
+
and not filetypes.is_header_pair(p["entity"], p["coupled"])
|
|
199
|
+
and not (filetypes.is_vendor_path(p["entity"]) or filetypes.is_vendor_path(p["coupled"]))
|
|
197
200
|
and not (tree and (p["entity"] not in tree or p["coupled"] not in tree))]
|
|
198
201
|
if not pairs:
|
|
199
202
|
return []
|
|
@@ -266,8 +269,9 @@ def stale_files(report: dict, months: int = 12, share: float = 0.3) -> list:
|
|
|
266
269
|
def bug_magnets(report: dict, min_recent: int = 3, warn_at: int = 5) -> list:
|
|
267
270
|
"""Source files with a run of recent fix commits. Test files are left out: they change with every fix.
|
|
268
271
|
So is release plumbing: a manifest touched by every fix release is not where the bug was."""
|
|
272
|
+
plumb = filetypes.plumbing_paths(report)
|
|
269
273
|
hot = [f for f in report.get("fixes") or [] if f["recent-fixes"] >= min_recent
|
|
270
|
-
and not (filetypes.is_test_path(f["entity"]) or filetypes.
|
|
274
|
+
and not (filetypes.is_test_path(f["entity"]) or filetypes.is_release(f["entity"], plumb))]
|
|
271
275
|
if not hot:
|
|
272
276
|
return []
|
|
273
277
|
hot.sort(key=lambda f: (-f["recent-fixes"], -f["n-fixes"], f["entity"]))
|
|
@@ -5,15 +5,17 @@ import re
|
|
|
5
5
|
|
|
6
6
|
|
|
7
7
|
_BOT_WORDS = ("dependabot", "renovate", "github-actions", "github actions", "copilot", "cursor agent", "cursor-agent", "cursoragent")
|
|
8
|
+
_BOT_NAME = re.compile(r"\bbot\b|\bci\b|deploy|automation", re.I) # "Deploy from CI", "Release Bot", "Homebrew Automation"
|
|
8
9
|
|
|
9
10
|
|
|
10
11
|
def is_bot(name: str, email: str = "") -> bool:
|
|
11
|
-
"""A commit author that is a service, not a person: GitHub's *[bot] suffix,
|
|
12
|
-
|
|
12
|
+
"""A commit author that is a service, not a person: GitHub's *[bot] suffix, one of the common
|
|
13
|
+
automation names in the name or the mailbox, or a name that says bot, CI, deploy or automation."""
|
|
13
14
|
n, local = name.strip().lower(), email.strip().lower().split("@")[0]
|
|
14
15
|
if n.endswith("[bot]") or local.endswith("[bot]"):
|
|
15
16
|
return True
|
|
16
|
-
return any(w in n for w in _BOT_WORDS) or any(w in local for w in _BOT_WORDS) or email.strip().lower() == "actions@github.com"
|
|
17
|
+
return (any(w in n for w in _BOT_WORDS) or any(w in local for w in _BOT_WORDS) or email.strip().lower() == "actions@github.com"
|
|
18
|
+
or bool(_BOT_NAME.search(name)))
|
|
17
19
|
|
|
18
20
|
|
|
19
21
|
def _tokens(name: str) -> set:
|
|
@@ -53,7 +55,13 @@ def same_person(a: dict, b: dict) -> bool:
|
|
|
53
55
|
for handle, full in ((na, tb), (nb, ta)):
|
|
54
56
|
if " " not in handle and handle in full and len(full) >= 2 and _distinctive(handle):
|
|
55
57
|
return True
|
|
56
|
-
|
|
58
|
+
# RobinMalfait and Robin Malfait: the full name run together, six letters or more so it is not anyone
|
|
59
|
+
sa, sb = _squash(a["name"]), _squash(b["name"])
|
|
60
|
+
return bool(sa) and sa == sb and len(sa) >= 6 and (len(ta) >= 2 or len(tb) >= 2)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _squash(name: str) -> str:
|
|
64
|
+
return re.sub(r"[^a-z0-9]", "", name.lower())
|
|
57
65
|
|
|
58
66
|
|
|
59
67
|
def merge(identities: list) -> list:
|
|
@@ -59,18 +59,59 @@ def digest(value: str, key: bytes) -> str:
|
|
|
59
59
|
return hmac.new(key, value.encode("utf-8", "surrogateescape"), hashlib.sha256).hexdigest()[:12]
|
|
60
60
|
|
|
61
61
|
|
|
62
|
-
|
|
62
|
+
# A header followed by no key material is a pattern a script greps for, not a key (ohmyzsh's ssh-agent
|
|
63
|
+
# plugin: the scanner captures the header and the shell code after it).
|
|
64
|
+
_HEADER = re.compile(r"^\^?-----BEGIN[ \\A-Z]*KEY-----")
|
|
65
|
+
_BASE64_RUN = re.compile(r"[A-Za-z0-9+/=]{40,}")
|
|
66
|
+
# A line that calls its own value an example is documentation, wherever it sits: "# Example password: ...".
|
|
67
|
+
_EXAMPLE_LINE = re.compile(r"\b(example|sample|dummy|fake|placeholder)|\be\.g\.", re.I)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _made_up(value: str) -> bool:
|
|
71
|
+
"""A run up the alphabet and the digits (qr6stu789vwxyz, abcd1234): typed, not generated. Eight
|
|
72
|
+
characters or more, letters non-decreasing, digits non-decreasing, both present or one long."""
|
|
73
|
+
alnum = re.sub(r"[^A-Za-z0-9]", "", value).lower()
|
|
74
|
+
letters = [c for c in alnum if c.isalpha()]
|
|
75
|
+
digits = [c for c in alnum if c.isdigit()]
|
|
76
|
+
if len(alnum) < 8 or len(alnum) != len(letters) + len(digits):
|
|
77
|
+
return False
|
|
78
|
+
return letters == sorted(letters) and digits == sorted(digits) and len(set(letters)) >= 4 or (not letters and digits == sorted(digits))
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def is_placeholder(value: str, line: str = "") -> bool:
|
|
82
|
+
"""Whether `value` has a shape that cannot be a live secret. `line` is the source line the value
|
|
83
|
+
sat on, read from the clone at scan time and never written."""
|
|
63
84
|
value = (value or "").strip()
|
|
64
85
|
if (_VERSION.match(value) or value.endswith("...") or value.endswith("…") or _MARKER.match(value) or value.lower() in _EXAMPLE_WORDS
|
|
65
|
-
or _ENV_REF.match(value)):
|
|
86
|
+
or _ENV_REF.match(value) or _made_up(value)):
|
|
87
|
+
return True
|
|
88
|
+
if line and _EXAMPLE_LINE.search(line):
|
|
66
89
|
return True
|
|
67
90
|
m = _KEY_BLOCK.search(value)
|
|
68
91
|
if m:
|
|
69
92
|
body = re.sub(r"\s|\.|…|\\n", "", m.group(1)) # literal \n sequences appear in JSON samples
|
|
70
93
|
return len(body) < _KEY_MATERIAL
|
|
94
|
+
h = _HEADER.match(value)
|
|
95
|
+
if h and not _BASE64_RUN.search(value[h.end():]):
|
|
96
|
+
return True
|
|
71
97
|
return False
|
|
72
98
|
|
|
73
99
|
|
|
100
|
+
def line_of(repo: str, commit: str, path: str, number: int) -> str:
|
|
101
|
+
"""Line `number` of `path` as it was at `commit`, from the clone; "" when it cannot be read. Read
|
|
102
|
+
for the placeholder rule and dropped, like every other raw field."""
|
|
103
|
+
if not (commit and path and number):
|
|
104
|
+
return ""
|
|
105
|
+
proc = subprocess.run(["git", "-C", repo, "show", f"{commit}:{path}"], capture_output=True)
|
|
106
|
+
if proc.returncode != 0:
|
|
107
|
+
return ""
|
|
108
|
+
lines = proc.stdout.decode("utf-8", "replace").split("\n")
|
|
109
|
+
return lines[number - 1] if 0 < number <= len(lines) else ""
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
LINE_LOOKUPS = 400 # one git call per finding; past this many the rest go without their line
|
|
113
|
+
|
|
114
|
+
|
|
74
115
|
def sanitise(rows: list) -> list:
|
|
75
116
|
key = new_key() # one key for the whole report, so repeats of a value still group
|
|
76
117
|
out = []
|
|
@@ -78,7 +119,7 @@ def sanitise(rows: list) -> list:
|
|
|
78
119
|
value = r.get("Secret") or ""
|
|
79
120
|
clean = {k: v for k, v in r.items() if k not in RAW_FIELDS}
|
|
80
121
|
clean["SecretHash"] = digest(value, key)
|
|
81
|
-
clean["Placeholder"] = is_placeholder(value)
|
|
122
|
+
clean["Placeholder"] = is_placeholder(value, r.get("Line") or "") # the line is read here and dropped with the other raw fields
|
|
82
123
|
out.append(clean)
|
|
83
124
|
return out
|
|
84
125
|
|
|
@@ -130,7 +171,10 @@ def main(argv=None) -> int:
|
|
|
130
171
|
print(f"leaks.py: betterleaks exited {proc.returncode}; no report written", file=sys.stderr)
|
|
131
172
|
return proc.returncode
|
|
132
173
|
text = proc.stdout.decode("utf-8", "surrogateescape").strip()
|
|
133
|
-
|
|
174
|
+
raw = (json.loads(text) if text else None) or [] # a clean repository is reported as null
|
|
175
|
+
for r in raw[:LINE_LOOKUPS]: # betterleaks does not report the line; the clone in the current directory has it
|
|
176
|
+
r["Line"] = line_of(os.getcwd(), r.get("Commit") or "", r.get("File") or "", int(r.get("StartLine") or 0))
|
|
177
|
+
rows = sanitise(raw)
|
|
134
178
|
fd, tmp = tempfile.mkstemp(dir=os.path.dirname(os.path.abspath(target)), prefix=".secrets-", suffix=".json")
|
|
135
179
|
try:
|
|
136
180
|
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
|
@@ -65,7 +65,7 @@ def parse_scc(text: str, types=None) -> dict:
|
|
|
65
65
|
}
|
|
66
66
|
|
|
67
67
|
|
|
68
|
-
NUMERIC_COLUMNS = {"n-revs", "degree", "average-revs", "n-authors", "age-months", "added", "deleted", "n-fixes", "recent-fixes"}
|
|
68
|
+
NUMERIC_COLUMNS = {"n-revs", "degree", "average-revs", "n-authors", "age-months", "added", "deleted", "n-fixes", "recent-fixes", "tiny-revs"}
|
|
69
69
|
|
|
70
70
|
|
|
71
71
|
def parse_maat_csv(text: str) -> list:
|
|
@@ -236,7 +236,10 @@ def load_report(out_dir: str, nested: bool = True) -> dict:
|
|
|
236
236
|
surviving = OrderedDict()
|
|
237
237
|
for name, lines in (parse_theseus(authors) if authors else {}).items():
|
|
238
238
|
key = canonical.get(name, name)
|
|
239
|
+
if identity.is_bot(key): # a deploy job that committed a built site owns nothing anyone needs to know
|
|
240
|
+
continue
|
|
239
241
|
surviving[key] = surviving.get(key, 0) + lines
|
|
242
|
+
ownership = [r for r in parse_maat_csv(_read(out_dir, "maat-entity-ownership.csv")) if not identity.is_bot(r.get("author") or "")]
|
|
240
243
|
return {
|
|
241
244
|
"out_dir": out_dir,
|
|
242
245
|
"meta": meta,
|
|
@@ -244,10 +247,11 @@ def load_report(out_dir: str, nested: bool = True) -> dict:
|
|
|
244
247
|
# was measured unfiltered, so it is re-rendered unfiltered rather than with a guessed list
|
|
245
248
|
"size": parse_scc(_read(out_dir, "size.json"), filetypes.parse(meta["file_types"]) if "file_types" in meta else None),
|
|
246
249
|
"revisions": parse_maat_csv(_read(out_dir, "maat-revisions.csv")),
|
|
250
|
+
"plumbing": parse_maat_csv(_read(out_dir, "maat-plumbing.csv")),
|
|
247
251
|
"coupling": parse_maat_csv(_read(out_dir, "maat-coupling.csv")),
|
|
248
252
|
"authors": parse_maat_csv(_read(out_dir, "maat-authors.csv")),
|
|
249
253
|
"age": parse_maat_csv(_read(out_dir, "maat-age.csv")),
|
|
250
|
-
"ownership":
|
|
254
|
+
"ownership": ownership,
|
|
251
255
|
"fixes": parse_maat_csv(_read(out_dir, "maat-fixes.csv")),
|
|
252
256
|
"sizer": parse_git_sizer(_read(out_dir, "repo-health.txt")),
|
|
253
257
|
"cohorts": parse_theseus(cohorts) if cohorts else {},
|
|
@@ -221,8 +221,22 @@ def activity(commits: list) -> dict:
|
|
|
221
221
|
"reverted": dict(sorted(reverted.items(), key=lambda kv: (-kv[1], kv[0])))}
|
|
222
222
|
|
|
223
223
|
|
|
224
|
+
def plumbing(commits: list, min_revs: int = 20, share: float = 0.8, max_lines: int = 2) -> list:
|
|
225
|
+
"""Files whose commits nearly always change a line or two: a version constant in __init__.py, a
|
|
226
|
+
date in a header. Their churn is the release cadence, not where the next bug lands. Needs enough
|
|
227
|
+
commits to judge by."""
|
|
228
|
+
revs, tiny = Counter(), Counter()
|
|
229
|
+
for c in commits:
|
|
230
|
+
for path, added, deleted in c["files"]:
|
|
231
|
+
revs[path] += 1
|
|
232
|
+
if added + deleted <= max_lines:
|
|
233
|
+
tiny[path] += 1
|
|
234
|
+
return [{"entity": e, "n-revs": n, "tiny-revs": tiny[e]} for e, n in sorted(revs.items()) if n >= min_revs and tiny[e] / n >= share]
|
|
235
|
+
|
|
236
|
+
|
|
224
237
|
ANALYSES = {
|
|
225
238
|
"revisions": (revisions, ["entity", "n-revs"]),
|
|
239
|
+
"plumbing": (plumbing, ["entity", "n-revs", "tiny-revs"]),
|
|
226
240
|
"coupling": (coupling, ["entity", "coupled", "degree", "average-revs"]),
|
|
227
241
|
"authors": (authors, ["entity", "n-authors", "n-revs"]),
|
|
228
242
|
"age": (age, ["entity", "age-months"]),
|
|
@@ -129,6 +129,15 @@ def _hide_release(pairs: list, full) -> tuple:
|
|
|
129
129
|
return kept, (f"{hidden} release pair{'s' if hidden != 1 else ''} hidden{HIDDEN_SUFFIX}" if hidden else None)
|
|
130
130
|
|
|
131
131
|
|
|
132
|
+
def _hide_header_pairs(pairs: list, full) -> tuple:
|
|
133
|
+
"""A C-family source file and its own header change together by construction."""
|
|
134
|
+
if full is True:
|
|
135
|
+
return pairs, None
|
|
136
|
+
kept = [p for p in pairs if not filetypes.is_header_pair(p["entity"], p["coupled"])]
|
|
137
|
+
hidden = len(pairs) - len(kept)
|
|
138
|
+
return kept, (f"{hidden} header pair{'s' if hidden != 1 else ''} hidden{HIDDEN_SUFFIX}" if hidden else None)
|
|
139
|
+
|
|
140
|
+
|
|
132
141
|
def _join_hidden(*notes) -> str:
|
|
133
142
|
"""Several hidden-row notes as one caption phrase: 'A hidden; B hidden; --full shows them'."""
|
|
134
143
|
parts = [n[:-len(HIDDEN_SUFFIX)] if n.endswith(HIDDEN_SUFFIX) else n for n in notes if n]
|
|
@@ -395,7 +404,8 @@ def hotspots_section(report: dict, full: bool = True, width=None) -> dict:
|
|
|
395
404
|
scored, hidden_note = _hide_tests(scored, lambda h: h["entity"], full)
|
|
396
405
|
scored, deleted_note = _hide_deleted(scored, report, full)
|
|
397
406
|
scored, generated_note = _hide_generated(scored, lambda h: h["entity"], report, full)
|
|
398
|
-
|
|
407
|
+
plumb = filetypes.plumbing_paths(report)
|
|
408
|
+
scored, release_note = _hide_rows(scored, lambda h: h["entity"], full, lambda p: filetypes.is_release(p, plumb), "release file")
|
|
399
409
|
hidden_note = _join_hidden(hidden_note, deleted_note, generated_note, release_note)
|
|
400
410
|
title = "Hotspots (score = revisions × lines of code)" if full is True else "Hotspots"
|
|
401
411
|
limit = _limit("Hotspots", full)
|
|
@@ -429,7 +439,9 @@ def coupling_section(report: dict, full: bool = True, width=None) -> dict:
|
|
|
429
439
|
pairs, hidden_note = _hide_tests(pairs, lambda p: (p["entity"], p["coupled"]), full, noun="test pair")
|
|
430
440
|
pairs, gone_note = _hide_gone(pairs, report, full)
|
|
431
441
|
pairs, release_note = _hide_release(pairs, full)
|
|
432
|
-
|
|
442
|
+
pairs, header_note = _hide_header_pairs(pairs, full)
|
|
443
|
+
pairs, vendor_note = _hide_vendor(pairs, lambda p: (p["entity"], p["coupled"]), full, noun="vendored pair", plural="vendored pairs")
|
|
444
|
+
gone_note = _join_hidden(gone_note, release_note, header_note, vendor_note)
|
|
433
445
|
groups, cluster_note = [], None
|
|
434
446
|
if full is not True:
|
|
435
447
|
# a directory whose files all change together is one row; --full lists every pair
|
|
@@ -96,10 +96,22 @@ def list_repos(owner: str, lister=_gh) -> list:
|
|
|
96
96
|
return sorted(set(out.split()))
|
|
97
97
|
|
|
98
98
|
|
|
99
|
-
def
|
|
100
|
-
"""
|
|
99
|
+
def clone_url(target: str) -> str:
|
|
100
|
+
"""The URL plain git can clone: a URL as given, owner/repo on github.com."""
|
|
101
|
+
return target if _URL.match(target) else f"https://github.com/{target}.git"
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def clone(target: str, dest_parent: str, runner=_gh, git_runner=_gh) -> str:
|
|
105
|
+
"""Clone a remote target with gh (so private repos use the existing auth), or with plain git when
|
|
106
|
+
gh is missing or fails: a public repository needs no token. Raises GhError naming both failures."""
|
|
101
107
|
dest = os.path.join(dest_parent, repo_name(target))
|
|
102
|
-
|
|
108
|
+
try:
|
|
109
|
+
_wrap(runner, ["gh", "repo", "clone", target, dest, "--", "--quiet"])
|
|
110
|
+
except GhError as gh_error:
|
|
111
|
+
try:
|
|
112
|
+
_wrap(git_runner, [*filetypes.GIT, "clone", "--quiet", clone_url(target), dest])
|
|
113
|
+
except GhError as git_error:
|
|
114
|
+
raise GhError(f"{gh_error}; git clone also failed: {git_error}") from None
|
|
103
115
|
return dest
|
|
104
116
|
|
|
105
117
|
|
|
@@ -64,9 +64,10 @@ def risks(report: dict, min_revs: int = 2) -> list:
|
|
|
64
64
|
fixes = {f["entity"]: f for f in report.get("fixes") or []}
|
|
65
65
|
n_authors = {a["entity"]: a["n-authors"] for a in report.get("authors") or []}
|
|
66
66
|
|
|
67
|
+
plumb = filetypes.plumbing_paths(report)
|
|
67
68
|
rows = []
|
|
68
69
|
for h in hotspots.ranked(report):
|
|
69
|
-
if h["code"] is None or h["revs"] < min_revs or filetypes.is_test_path(h["entity"]) or filetypes.
|
|
70
|
+
if h["code"] is None or h["revs"] < min_revs or filetypes.is_test_path(h["entity"]) or filetypes.is_release(h["entity"], plumb):
|
|
70
71
|
continue # a version file or a manifest changes on every release, not where the next bug lands
|
|
71
72
|
fx = fixes.get(h["entity"], {})
|
|
72
73
|
own = owners.get(h["entity"]) or Counter()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gitmole
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.11
|
|
4
4
|
Summary: Offline git repository analysis with a terminal report: hotspots, coupling, ownership, code age, secrets, repo health.
|
|
5
5
|
License: MIT
|
|
6
6
|
Project-URL: Homepage, https://github.com/antvinni/gitmole
|
|
@@ -55,7 +55,7 @@ Linux package names, the release binaries, `--plots` and the pip caveats:
|
|
|
55
55
|
```bash
|
|
56
56
|
gitmole . # the clone you are in
|
|
57
57
|
gitmole /path/to/clone # any local clone
|
|
58
|
-
gitmole owner/repo # clones
|
|
58
|
+
gitmole owner/repo # clones into a temp dir first, with gh or plain git
|
|
59
59
|
gitmole 'owner/*' # every non-archived repo of a user or org, one summary table
|
|
60
60
|
|
|
61
61
|
gitmole . --markdown report.md # the same report as a Markdown document
|
|
@@ -86,15 +86,18 @@ class Discover(unittest.TestCase):
|
|
|
86
86
|
|
|
87
87
|
class TestPaths(unittest.TestCase):
|
|
88
88
|
def test_test_files_and_directories(self):
|
|
89
|
-
for path in ("tests/test_a.py", "a/spec/b.rb", "src/__tests__/x.js", "x/y_test.go", "app.spec.ts", "app.test.tsx", "test_x.py"
|
|
89
|
+
for path in ("tests/test_a.py", "a/spec/b.rb", "src/__tests__/x.js", "x/y_test.go", "app.spec.ts", "app.test.tsx", "test_x.py",
|
|
90
|
+
"pending_tests/main.py", "e2e-tests/login.ts", "src/test_utils/helpers.py", "crates/x/snapshots/rule__S105.py.snap",
|
|
91
|
+
"src/__snapshots__/a.js.snap", "lib/render.snap"):
|
|
90
92
|
self.assertTrue(filetypes.is_test_path(path), path)
|
|
91
|
-
for path in ("src/contest.py", "gitmole/render.py", "attest/x.py", "latest.md"):
|
|
93
|
+
for path in ("src/contest.py", "gitmole/render.py", "attest/x.py", "latest.md", "src/testimony.py", "snapshot.py"):
|
|
92
94
|
self.assertFalse(filetypes.is_test_path(path), path)
|
|
93
95
|
|
|
94
96
|
def test_documentation_files_and_directories(self):
|
|
95
|
-
for path in ("README.md", "docs/GA4-API-INTEGRATION.md", "doc/guide.rst", "NOTES.txt", "a/b/CHANGELOG.markdown", "docs/conf.py", "x.adoc"
|
|
97
|
+
for path in ("README.md", "docs/GA4-API-INTEGRATION.md", "doc/guide.rst", "NOTES.txt", "a/b/CHANGELOG.markdown", "docs/conf.py", "x.adoc",
|
|
98
|
+
"docs_src/security/tutorial004.py", "docs-site/app.js", "doc_examples/x.py"):
|
|
96
99
|
self.assertTrue(filetypes.is_doc_path(path), path)
|
|
97
|
-
for path in ("app/settings.py", "static/index.html", "docsite/app.js", "mdx/a.py", "config.yaml"):
|
|
100
|
+
for path in ("app/settings.py", "static/index.html", "docsite/app.js", "mdx/a.py", "config.yaml", "doctor/a.py"):
|
|
98
101
|
self.assertFalse(filetypes.is_doc_path(path), path)
|
|
99
102
|
|
|
100
103
|
def test_example_fixture_and_rule_directories(self):
|
|
@@ -104,6 +107,13 @@ class TestPaths(unittest.TestCase):
|
|
|
104
107
|
for path in ("app/settings.py", "examplesite/app.py", "src/rulesets/a.go", "sampler/x.py", "config/betterleaks.toml"):
|
|
105
108
|
self.assertFalse(filetypes.is_sample_path(path), path)
|
|
106
109
|
|
|
110
|
+
def test_a_source_file_and_its_own_header_are_a_header_pair(self):
|
|
111
|
+
for a, b in (("src/vector.c", "src/vector.h"), ("src/vector.h", "src/vector.c"), ("lib/x.cpp", "lib/x.hpp"), ("lib/x.cc", "lib/x.hh"),
|
|
112
|
+
("ui/view.m", "ui/view.h"), ("ui/view.mm", "ui/view.h")):
|
|
113
|
+
self.assertTrue(filetypes.is_header_pair(a, b), (a, b))
|
|
114
|
+
for a, b in (("src/vector.c", "src/list.h"), ("src/vector.c", "include/vector.h"), ("src/a.py", "src/a.pyi"), ("src/vector.c", "src/vector.c")):
|
|
115
|
+
self.assertFalse(filetypes.is_header_pair(a, b), (a, b))
|
|
116
|
+
|
|
107
117
|
def test_release_plumbing_files(self):
|
|
108
118
|
for path in ("lib/sinatra/version.rb", "VERSION", "src/pkg/__version__.py", "package.json", "package-lock.json", "Gemfile.lock",
|
|
109
119
|
"Cargo.toml", "pyproject.toml", "go.sum", "CHANGELOG.md", "CHANGES.rst", "sinatra.gemspec", "uv.lock"):
|
|
@@ -133,7 +143,8 @@ class TestPaths(unittest.TestCase):
|
|
|
133
143
|
|
|
134
144
|
def test_vendored_trees(self):
|
|
135
145
|
for path in ("vendor/github.com/x/y.go", "web/node_modules/a/index.js", "third_party/z/a.c", "thirdparty/a.c", "_vendor/a.py",
|
|
136
|
-
"external/lib/a.cpp", "requests/packages/urllib3/a.py", "pip/_vendor/six.py", "botocore/vendored/requests/a.py"
|
|
146
|
+
"external/lib/a.cpp", "requests/packages/urllib3/a.py", "pip/_vendor/six.py", "botocore/vendored/requests/a.py",
|
|
147
|
+
"deps/lua/src/strbuf.c", "deps/jemalloc/Makefile"):
|
|
137
148
|
self.assertTrue(filetypes.is_vendor_path(path), path)
|
|
138
149
|
for path in ("vendors.py", "src/vendoring/a.py", "node/a.js", "externals.txt", "app/main.go",
|
|
139
150
|
"packages/runtime-core/src/renderer.ts", "packages-private/x.ts"): # a monorepo's own packages/ at the root
|
|
@@ -261,6 +261,21 @@ class TightCoupling(unittest.TestCase):
|
|
|
261
261
|
f = findings.tight_coupling(report(coupling=pairs))
|
|
262
262
|
self.assertIn("2 pairs", f[0]["detail"], "without a tree listing every pair counts")
|
|
263
263
|
|
|
264
|
+
def test_vendored_pairs_are_somebody_elses_coupling(self):
|
|
265
|
+
pairs = [{"entity": "deps/hiredis/adapters/ae.h", "coupled": "deps/hiredis/adapters/libev.h", "degree": 84, "average-revs": 10},
|
|
266
|
+
{"entity": "src/ae.c", "coupled": "deps/hiredis/net.h", "degree": 85, "average-revs": 10},
|
|
267
|
+
{"entity": "src/ae.c", "coupled": "src/networking.c", "degree": 85, "average-revs": 10}]
|
|
268
|
+
f = findings.tight_coupling(report(coupling=pairs))
|
|
269
|
+
self.assertIn("1 pair changes together", f[0]["detail"], "a pair with a vendored file on either side is not this repository's dependency")
|
|
270
|
+
|
|
271
|
+
def test_a_source_file_and_its_header_are_expected_to_change_together(self):
|
|
272
|
+
pairs = [{"entity": "src/vector.c", "coupled": "src/vector.h", "degree": 100, "average-revs": 20},
|
|
273
|
+
{"entity": "deps/lua/src/strbuf.c", "coupled": "deps/lua/src/strbuf.h", "degree": 94, "average-revs": 8},
|
|
274
|
+
{"entity": "src/ae.c", "coupled": "src/networking.c", "degree": 85, "average-revs": 10}]
|
|
275
|
+
f = findings.tight_coupling(report(coupling=pairs))
|
|
276
|
+
self.assertIn("1 pair changes together", f[0]["detail"])
|
|
277
|
+
self.assertIn("src/ae.c", f[0]["detail"])
|
|
278
|
+
|
|
264
279
|
def test_release_plumbing_pairs_are_not_a_dependency(self):
|
|
265
280
|
pairs = [{"entity": "lib/sinatra/version.rb", "coupled": "rack-protection/lib/rack/protection/version.rb", "degree": 100, "average-revs": 60},
|
|
266
281
|
{"entity": "package.json", "coupled": "package-lock.json", "degree": 95, "average-revs": 40},
|
|
@@ -297,6 +312,12 @@ class ReleasePlumbing(unittest.TestCase):
|
|
|
297
312
|
self.assertIn("requests/models.py changed 60 times", f[0]["detail"])
|
|
298
313
|
self.assertNotIn("setup.py", f[0]["detail"])
|
|
299
314
|
|
|
315
|
+
def test_a_file_the_change_log_shows_as_plumbing_does_not_dominate_the_churn(self):
|
|
316
|
+
revs = [{"entity": "fastapi/__init__.py", "n-revs": 331}, {"entity": "fastapi/routing.py", "n-revs": 187}, {"entity": "fastapi/utils.py", "n-revs": 70}]
|
|
317
|
+
r = report(revisions=revs, plumbing=[{"entity": "fastapi/__init__.py", "n-revs": 331, "tiny-revs": 300}])
|
|
318
|
+
f = findings.hotspot_dominance(r)
|
|
319
|
+
self.assertIn("fastapi/routing.py changed 187 times", f[0]["detail"])
|
|
320
|
+
|
|
300
321
|
def test_a_manifest_is_not_a_bug_magnet(self):
|
|
301
322
|
fixes = [{"entity": "package.json", "n-fixes": 20, "last-fix": "2026-09-01", "recent-fixes": 6},
|
|
302
323
|
{"entity": "lib/reply.js", "n-fixes": 10, "last-fix": "2026-09-01", "recent-fixes": 4}]
|
|
@@ -34,6 +34,12 @@ class Merge(unittest.TestCase):
|
|
|
34
34
|
self.assertEqual(merged, {"Junegunn Choi": 3047, "Sam Altman": 5, "sam": 2, "Kevin Brown": 76, "Kevin": 21},
|
|
35
35
|
"a short or common first name is not distinctive enough")
|
|
36
36
|
|
|
37
|
+
def test_a_handle_that_is_the_full_name_run_together_is_the_same_person(self):
|
|
38
|
+
ids = [{"name": "Robin Malfait", "email": "malfait.robin@a.com", "commits": 1271}, {"name": "RobinMalfait", "email": "1834413+RobinMalfait@users.noreply.github.com", "commits": 4},
|
|
39
|
+
{"name": "Jo Li", "email": "jo@a.com", "commits": 3}, {"name": "joli", "email": "x@b.com", "commits": 1}]
|
|
40
|
+
merged = {m["name"]: m["commits"] for m in identity.merge(ids)}
|
|
41
|
+
self.assertEqual(merged, {"Robin Malfait": 1275, "Jo Li": 3, "joli": 1}, "a run-together name shorter than six letters could be anyone")
|
|
42
|
+
|
|
37
43
|
def test_a_bare_common_first_name_is_not_enough(self):
|
|
38
44
|
ids = [{"name": "Jean", "email": "jean@a.com", "commits": 24}, {"name": "Jean", "email": "jean@b.com", "commits": 18},
|
|
39
45
|
{"name": "Alex", "email": "alex@a.com", "commits": 3}, {"name": "alex", "email": "alex@b.com", "commits": 2}]
|
|
@@ -85,7 +91,8 @@ class IsBot(unittest.TestCase):
|
|
|
85
91
|
("github-actions[bot]", "41898282+github-actions[bot]@users.noreply.github.com"),
|
|
86
92
|
("dependabot[bot]", "support@github.com"), ("Dependabot", "dependabot@example.com"),
|
|
87
93
|
("Renovate Bot", "bot@renovateapp.com"), ("GitHub Actions", "actions@github.com"),
|
|
88
|
-
("Copilot", "198982749+Copilot@users.noreply.github.com"), ("Cursor Agent", "cursoragent@cursor.com")
|
|
94
|
+
("Copilot", "198982749+Copilot@users.noreply.github.com"), ("Cursor Agent", "cursoragent@cursor.com"),
|
|
95
|
+
("Deploy from CI", ""), ("Release Bot", "release@x.com"), ("CI", "ci@x.com"), ("Homebrew Automation", "a@x.com")]:
|
|
89
96
|
self.assertTrue(identity.is_bot(name, email), (name, email))
|
|
90
97
|
|
|
91
98
|
def test_people_are_not_bots(self):
|
|
@@ -74,6 +74,42 @@ class Placeholder(unittest.TestCase):
|
|
|
74
74
|
for value in ["p@ssw0rd!", "AKIA" + "X" * 16, "env-9f8a7b6c5d4e3f2a", "${not closed", "hello$world"]:
|
|
75
75
|
self.assertFalse(leaks.is_placeholder(value), value)
|
|
76
76
|
|
|
77
|
+
def test_a_key_header_with_no_key_material_after_it_is_a_marker_not_a_key(self):
|
|
78
|
+
# ohmyzsh's ssh-agent plugin greps files for the header; the scanner captures the header and the shell code after it
|
|
79
|
+
for value in ["-----BEGIN OPENSSH PRIVATE KEY-----", "^-----BEGIN\\ OPENSSH\\ PRIVATE\\ KEY-----", "-----BEGIN RSA PRIVATE KEY-----\\n",
|
|
80
|
+
'-----BEGIN OPENSSH PRIVATE KEY-----".\n if [[ -f "$file" && $(command head -n 1 "$file") =~ ^-----BEGIN ]]; then']:
|
|
81
|
+
self.assertTrue(leaks.is_placeholder(value), value)
|
|
82
|
+
real = "-----BEGIN OPENSSH PRIVATE KEY-----\n" + "b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW" * 3
|
|
83
|
+
self.assertFalse(leaks.is_placeholder(real), "a header followed by base64 is a key, END or no END")
|
|
84
|
+
|
|
85
|
+
def test_the_line_is_read_from_the_clone_at_the_commit(self):
|
|
86
|
+
with tempfile.TemporaryDirectory() as d:
|
|
87
|
+
subprocess.run(["git", "init", "-q", d], check=True)
|
|
88
|
+
with open(os.path.join(d, "gen.sh"), "w") as fh:
|
|
89
|
+
fh.write("#!/bin/sh\n# Example password: nz5ej2kypkvcw0rn5cvhs6qxtm\necho hi\n")
|
|
90
|
+
subprocess.run(["git", "-C", d, "add", "gen.sh"], check=True)
|
|
91
|
+
subprocess.run(["git", "-C", d, "-c", "user.name=T", "-c", "user.email=t@x.com", "commit", "-q", "-m", "gen"], check=True)
|
|
92
|
+
sha = subprocess.run(["git", "-C", d, "rev-parse", "HEAD"], capture_output=True, text=True, check=True).stdout.strip()
|
|
93
|
+
self.assertEqual(leaks.line_of(d, sha, "gen.sh", 2), "# Example password: nz5ej2kypkvcw0rn5cvhs6qxtm")
|
|
94
|
+
self.assertEqual(leaks.line_of(d, sha, "gen.sh", 9), "", "past the end is nothing, not an error")
|
|
95
|
+
self.assertEqual(leaks.line_of(d, sha, "missing.sh", 1), "")
|
|
96
|
+
self.assertEqual(leaks.line_of(d, "", "gen.sh", 1), "")
|
|
97
|
+
|
|
98
|
+
def test_a_line_that_calls_itself_an_example_is_a_placeholder(self):
|
|
99
|
+
# `# Example password: nz5ej2kypkvcw0rn5cvhs6qxtm` in a password generator's header comment
|
|
100
|
+
self.assertTrue(leaks.is_placeholder("nz5ej2kypkvcw0rn5cvhs6qxtm", line="# Example password: nz5ej2kypkvcw0rn5cvhs6qxtm"))
|
|
101
|
+
self.assertTrue(leaks.is_placeholder("nz5ej2kypkvcw0rn5cvhs6qxtm", line="token = 'nz5ej2kypkvcw0rn5cvhs6qxtm' # e.g. from the dashboard"))
|
|
102
|
+
self.assertTrue(leaks.is_placeholder("nz5ej2kypkvcw0rn5cvhs6qxtm", line="SAMPLE_KEY = 'nz5ej2kypkvcw0rn5cvhs6qxtm'"))
|
|
103
|
+
self.assertFalse(leaks.is_placeholder("nz5ej2kypkvcw0rn5cvhs6qxtm", line="api_key = 'nz5ej2kypkvcw0rn5cvhs6qxtm'"))
|
|
104
|
+
self.assertFalse(leaks.is_placeholder("nz5ej2kypkvcw0rn5cvhs6qxtm"), "without the line there is nothing to go on")
|
|
105
|
+
|
|
106
|
+
def test_a_run_up_the_alphabet_or_the_digits_is_made_up(self):
|
|
107
|
+
# ohmyzsh's spotify plugin: CLIENT_SECRET="qr6stu789vwxyz" in a usage message
|
|
108
|
+
for value in ["qr6stu789vwxyz", "abcdefghijklmnop", "abcd1234efgh5678", "ABCDEF123456", "0123456789abcdef"]:
|
|
109
|
+
self.assertTrue(leaks.is_placeholder(value), value)
|
|
110
|
+
for value in ["nz5ej2kypkvcw0rn5cvhs6qxtm", "abcdefg", "zyxwvutsrq", "ghp_" + "a1" * 18]:
|
|
111
|
+
self.assertFalse(leaks.is_placeholder(value), value)
|
|
112
|
+
|
|
77
113
|
def test_anything_else_is_taken_seriously(self):
|
|
78
114
|
# built at runtime: a literal in these shapes would trip secret scanners on this very file
|
|
79
115
|
key_id, long_key = "AKIA" + "X" * 16, "6L" + "x" * 38
|
|
@@ -97,6 +133,12 @@ class Sanitise(unittest.TestCase):
|
|
|
97
133
|
self.assertEqual(rows[0]["Fingerprint"], RAW[0]["Fingerprint"])
|
|
98
134
|
self.assertEqual(rows[0]["Author"], "Ann")
|
|
99
135
|
|
|
136
|
+
def test_the_line_is_read_for_the_placeholder_flag_before_it_is_dropped(self):
|
|
137
|
+
row = dict(RAW[0], Line="# Example token: " + RAW[0]["Secret"])
|
|
138
|
+
[clean] = leaks.sanitise([row])
|
|
139
|
+
self.assertTrue(clean["Placeholder"])
|
|
140
|
+
self.assertNotIn("Line", clean)
|
|
141
|
+
|
|
100
142
|
|
|
101
143
|
class Script(unittest.TestCase):
|
|
102
144
|
"""Runs the script against a stand-in betterleaks on PATH, so the wiring is tested without real keys."""
|
|
@@ -237,6 +237,7 @@ class LoadReport(unittest.TestCase):
|
|
|
237
237
|
"meta.json": json.dumps({"name": "demo", "commits": 3, "identities": []}),
|
|
238
238
|
"size.json": json.dumps([{"Name": "Python", "Count": 1, "Code": 10, "Comment": 0, "Blank": 0, "Complexity": 1}]),
|
|
239
239
|
"maat-revisions.csv": "entity,n-revs\na.py,3\n",
|
|
240
|
+
"maat-plumbing.csv": "entity,n-revs,tiny-revs\npkg/__init__.py,25,24\n",
|
|
240
241
|
"maat-coupling.csv": "entity,coupled,degree,average-revs\n",
|
|
241
242
|
"maat-age.csv": "entity,age-months\na.py,0\n",
|
|
242
243
|
"maat-authors.csv": "entity,n-authors,n-revs\na.py,1,3\n",
|
|
@@ -256,6 +257,7 @@ class LoadReport(unittest.TestCase):
|
|
|
256
257
|
self.assertEqual(r["meta"]["name"], "demo")
|
|
257
258
|
self.assertEqual(r["size"]["total_code"], 10)
|
|
258
259
|
self.assertEqual(r["revisions"][0]["entity"], "a.py")
|
|
260
|
+
self.assertEqual(r["plumbing"], [{"entity": "pkg/__init__.py", "n-revs": 25, "tiny-revs": 24}])
|
|
259
261
|
self.assertEqual(r["coupling"], [])
|
|
260
262
|
self.assertEqual(r["fixes"][0]["recent-fixes"], 1)
|
|
261
263
|
self.assertEqual(r["cohorts"], {"Code added in 2026": 10})
|
|
@@ -268,6 +270,24 @@ class LoadReport(unittest.TestCase):
|
|
|
268
270
|
self.assertEqual(r["duplicates"]["rate"], 5.0)
|
|
269
271
|
self.assertEqual(r["out_dir"], out)
|
|
270
272
|
|
|
273
|
+
def test_bot_authors_are_dropped_from_ownership_and_surviving_code(self):
|
|
274
|
+
# mdBook: a gh-pages deploy job committed the built site under the root, so "Deploy from CI" owned the root files
|
|
275
|
+
import os, tempfile
|
|
276
|
+
with tempfile.TemporaryDirectory() as out:
|
|
277
|
+
os.makedirs(os.path.join(out, "theseus"))
|
|
278
|
+
files = {
|
|
279
|
+
"meta.json": json.dumps({"name": "demo", "commits": 3, "identities": [{"name": "Ann", "email": "a@x", "commits": 2}],
|
|
280
|
+
"bots": [{"name": "Deploy from CI", "commits": 40}]}),
|
|
281
|
+
"maat-entity-ownership.csv": "entity,author,added,deleted\nindex.html,Deploy from CI,6000,0\nsrc/a.rs,Ann,300,0\nsrc/b.rs,dependabot[bot],20,0\n",
|
|
282
|
+
"theseus/authors.json": json.dumps({"labels": ["Ann", "Deploy from CI"], "ts": ["t"], "y": [[300], [6000]]}),
|
|
283
|
+
}
|
|
284
|
+
for name, text in files.items():
|
|
285
|
+
with open(os.path.join(out, name), "w") as fh:
|
|
286
|
+
fh.write(text)
|
|
287
|
+
r = load.load_report(out)
|
|
288
|
+
self.assertEqual([o["author"] for o in r["ownership"]], ["Ann"])
|
|
289
|
+
self.assertEqual(r["theseus_authors"], {"Ann": 300})
|
|
290
|
+
|
|
271
291
|
def test_size_is_filtered_by_the_file_types_recorded_in_meta(self):
|
|
272
292
|
import os, tempfile
|
|
273
293
|
size = json.dumps([{"Name": "JSON", "Count": 1, "Code": 9000, "Comment": 0, "Blank": 0, "Complexity": 0,
|
|
@@ -99,6 +99,29 @@ class Revisions(unittest.TestCase):
|
|
|
99
99
|
self.assertEqual(dict((r["entity"], r["n-revs"]) for r in rows)["src/c.py"], 1)
|
|
100
100
|
|
|
101
101
|
|
|
102
|
+
class Plumbing(unittest.TestCase):
|
|
103
|
+
def _commits(self, path, tiny, big):
|
|
104
|
+
out = [{"hash": f"t{i}", "date": "2026-01-01", "time": "", "author": "A", "subject": "bump", "files": [(path, 1, 1)]} for i in range(tiny)]
|
|
105
|
+
out += [{"hash": f"b{i}", "date": "2026-01-01", "time": "", "author": "A", "subject": "work", "files": [(path, 40, 12)]} for i in range(big)]
|
|
106
|
+
return out
|
|
107
|
+
|
|
108
|
+
def test_a_file_whose_commits_nearly_always_change_a_line_or_two_is_plumbing(self):
|
|
109
|
+
commits = self._commits("fastapi/__init__.py", tiny=300, big=31) + self._commits("fastapi/routing.py", tiny=10, big=90) + self._commits("VERSION", tiny=5, big=0)
|
|
110
|
+
rows = maat.plumbing(commits)
|
|
111
|
+
self.assertEqual(rows, [{"entity": "fastapi/__init__.py", "n-revs": 331, "tiny-revs": 300}],
|
|
112
|
+
"routing.py has real edits; VERSION has too few commits to judge")
|
|
113
|
+
|
|
114
|
+
def test_written_alongside_the_other_analyses(self):
|
|
115
|
+
with tempfile.TemporaryDirectory() as d:
|
|
116
|
+
log = os.path.join(d, "log.txt")
|
|
117
|
+
with open(log, "w", encoding="utf-8") as fh:
|
|
118
|
+
fh.write("".join(f"--h{i}--2026-01-{1 + i % 28:02d}T10:00:00+00:00--Ann--bump\n1\t1\tpkg/__init__.py\n" for i in range(25)))
|
|
119
|
+
maat.write_all(log, d, types=None)
|
|
120
|
+
with open(os.path.join(d, "maat-plumbing.csv"), encoding="utf-8") as fh:
|
|
121
|
+
text = fh.read()
|
|
122
|
+
self.assertEqual(text.splitlines(), ["entity,n-revs,tiny-revs", "pkg/__init__.py,25,25"])
|
|
123
|
+
|
|
124
|
+
|
|
102
125
|
class Coupling(unittest.TestCase):
|
|
103
126
|
def test_degree_is_shared_over_average_revisions(self):
|
|
104
127
|
rows = maat.coupling(maat.parse_log(LOG))
|
|
@@ -311,14 +334,15 @@ class SinceWindow(unittest.TestCase):
|
|
|
311
334
|
|
|
312
335
|
|
|
313
336
|
class WriteAll(unittest.TestCase):
|
|
314
|
-
def
|
|
337
|
+
def test_writes_the_csv_files_in_code_maat_layout_plus_plumbing(self):
|
|
315
338
|
with tempfile.TemporaryDirectory() as d:
|
|
316
339
|
log = os.path.join(d, "log.txt")
|
|
317
340
|
with open(log, "w") as fh:
|
|
318
341
|
fh.write(LOG)
|
|
319
342
|
maat.write_all(log, d)
|
|
320
343
|
names = sorted(n for n in os.listdir(d) if n.startswith("maat-"))
|
|
321
|
-
self.assertEqual(names, ["maat-age.csv", "maat-authors.csv", "maat-coupling.csv", "maat-entity-ownership.csv", "maat-fixes.csv",
|
|
344
|
+
self.assertEqual(names, ["maat-age.csv", "maat-authors.csv", "maat-coupling.csv", "maat-entity-ownership.csv", "maat-fixes.csv",
|
|
345
|
+
"maat-plumbing.csv", "maat-revisions.csv"])
|
|
322
346
|
self.assertTrue(os.path.isfile(os.path.join(d, "activity.json")))
|
|
323
347
|
with open(os.path.join(d, "maat-revisions.csv")) as fh:
|
|
324
348
|
self.assertEqual(fh.readline().strip(), "entity,n-revs")
|
|
@@ -345,6 +345,41 @@ class Report(unittest.TestCase):
|
|
|
345
345
|
full = rendered(r, [], width=200, full=True)
|
|
346
346
|
self.assertIn("setup.py", full[full.index("◆ Hotspots"):])
|
|
347
347
|
|
|
348
|
+
def test_default_hotspots_hide_files_the_change_log_shows_as_plumbing(self):
|
|
349
|
+
r = sample_report()
|
|
350
|
+
r["size"]["files"]["pkg/__init__.py"] = {"code": 40, "complexity": 0}
|
|
351
|
+
r["revisions"].append({"entity": "pkg/__init__.py", "n-revs": 331})
|
|
352
|
+
r["plumbing"] = [{"entity": "pkg/__init__.py", "n-revs": 331, "tiny-revs": 300}]
|
|
353
|
+
hot = rendered(r, [], width=200)
|
|
354
|
+
hot = hot[hot.index("◆ Hotspots"):hot.index("Change coupling")]
|
|
355
|
+
self.assertNotIn("pkg/__init__.py", hot)
|
|
356
|
+
self.assertIn("1 release file hidden; --full shows them", hot)
|
|
357
|
+
|
|
358
|
+
def test_default_coupling_hides_vendored_pairs_and_says_so(self):
|
|
359
|
+
r = sample_report()
|
|
360
|
+
for f in ("deps/lua/a.c", "deps/lua/b.c", "src/x.c", "src/y.c"):
|
|
361
|
+
r["size"]["files"][f] = {"code": 30, "complexity": 1}
|
|
362
|
+
r["coupling"] = [{"entity": "deps/lua/a.c", "coupled": "deps/lua/b.c", "degree": 90, "average-revs": 20},
|
|
363
|
+
{"entity": "deps/lua/a.c", "coupled": "src/x.c", "degree": 70, "average-revs": 9},
|
|
364
|
+
{"entity": "src/x.c", "coupled": "src/y.c", "degree": 60, "average-revs": 9}]
|
|
365
|
+
coupling = _section_text(rendered(r, [], width=200), "Change coupling")
|
|
366
|
+
self.assertIn("src/y.c", coupling)
|
|
367
|
+
self.assertNotIn("deps/", coupling)
|
|
368
|
+
self.assertIn("2 vendored pairs hidden; --full shows them", coupling)
|
|
369
|
+
|
|
370
|
+
def test_default_coupling_hides_header_pairs_and_says_so(self):
|
|
371
|
+
r = sample_report()
|
|
372
|
+
for f in ("src/vector.c", "src/vector.h", "src/list.c"):
|
|
373
|
+
r["size"]["files"][f] = {"code": 30, "complexity": 1}
|
|
374
|
+
r["coupling"] = [{"entity": "src/vector.c", "coupled": "src/vector.h", "degree": 100, "average-revs": 20},
|
|
375
|
+
{"entity": "src/list.c", "coupled": "src/vector.h", "degree": 60, "average-revs": 9}]
|
|
376
|
+
coupling = _section_text(rendered(r, [], width=200), "Change coupling")
|
|
377
|
+
self.assertIn("src/list.c", coupling)
|
|
378
|
+
self.assertNotIn("100%", coupling)
|
|
379
|
+
self.assertIn("1 header pair hidden; --full shows them", coupling)
|
|
380
|
+
full = _section_text(rendered(r, [], width=200, full=True), "Change coupling")
|
|
381
|
+
self.assertIn("100%", full)
|
|
382
|
+
|
|
348
383
|
def test_default_coupling_hides_release_plumbing_pairs_and_says_so(self):
|
|
349
384
|
r = sample_report()
|
|
350
385
|
for f in ("lib/version.rb", "contrib/version.rb", "Gemfile", "Gemfile.lock"):
|
|
@@ -77,11 +77,36 @@ class GhErrors(unittest.TestCase):
|
|
|
77
77
|
run.list_repos("acme", lister=lister)
|
|
78
78
|
self.assertIn("failed to verify certificate", str(ctx.exception))
|
|
79
79
|
|
|
80
|
-
def
|
|
80
|
+
def test_clone_falls_back_to_plain_git_when_gh_is_missing_or_fails(self):
|
|
81
|
+
# a public repository needs no gh and no token: git alone can clone it
|
|
82
|
+
calls = []
|
|
83
|
+
def no_gh(argv):
|
|
84
|
+
raise FileNotFoundError("gh")
|
|
85
|
+
def git(argv):
|
|
86
|
+
calls.append(argv)
|
|
87
|
+
return ""
|
|
88
|
+
with tempfile.TemporaryDirectory() as d:
|
|
89
|
+
dest = run.clone("acme/widgets", d, runner=no_gh, git_runner=git)
|
|
90
|
+
self.assertEqual(dest, os.path.join(d, "widgets"))
|
|
91
|
+
self.assertEqual(calls, [["git", "-c", "core.quotePath=false", "clone", "--quiet", "https://github.com/acme/widgets.git", dest]])
|
|
92
|
+
calls.clear()
|
|
93
|
+
run.clone("https://github.com/acme/widgets.git", d, runner=lambda argv: (_ for _ in ()).throw(subprocess.CalledProcessError(1, argv, stderr="tls")), git_runner=git)
|
|
94
|
+
self.assertEqual(calls[0][5], "https://github.com/acme/widgets.git", "a URL is cloned as given")
|
|
95
|
+
|
|
96
|
+
def test_clone_reports_both_failures_when_git_fails_too(self):
|
|
81
97
|
with tempfile.TemporaryDirectory() as d:
|
|
82
98
|
with self.assertRaises(run.GhError) as ctx:
|
|
83
|
-
run.clone("acme/definitely-missing-repo-xyz", d,
|
|
99
|
+
run.clone("acme/definitely-missing-repo-xyz", d,
|
|
100
|
+
runner=lambda argv: (_ for _ in ()).throw(subprocess.CalledProcessError(1, argv, stderr="repository not found")),
|
|
101
|
+
git_runner=lambda argv: (_ for _ in ()).throw(subprocess.CalledProcessError(128, argv, stderr="fatal: not found")))
|
|
84
102
|
self.assertIn("repository not found", str(ctx.exception))
|
|
103
|
+
self.assertIn("git clone also failed: fatal: not found", str(ctx.exception))
|
|
104
|
+
|
|
105
|
+
def test_clone_does_not_touch_git_when_gh_succeeds(self):
|
|
106
|
+
calls = []
|
|
107
|
+
with tempfile.TemporaryDirectory() as d:
|
|
108
|
+
run.clone("acme/widgets", d, runner=lambda argv: "", git_runner=lambda argv: calls.append(argv))
|
|
109
|
+
self.assertEqual(calls, [])
|
|
85
110
|
|
|
86
111
|
|
|
87
112
|
class ParseSince(unittest.TestCase):
|
|
@@ -61,6 +61,13 @@ class Risks(unittest.TestCase):
|
|
|
61
61
|
files = sorted(x["file"] for x in watch.risks(r))
|
|
62
62
|
self.assertEqual(files, ["Makefile", "core/parser.py"], "a version file or a manifest changes on every release, not where the next bug lands")
|
|
63
63
|
|
|
64
|
+
def test_a_file_the_change_log_shows_as_plumbing_is_not_on_the_list(self):
|
|
65
|
+
r = report()
|
|
66
|
+
r["size"]["files"]["pkg/__init__.py"] = {"code": 40, "complexity": 0}
|
|
67
|
+
r["revisions"] = [{"entity": "pkg/__init__.py", "n-revs": 331}, {"entity": "core/parser.py", "n-revs": 40}]
|
|
68
|
+
r["plumbing"] = [{"entity": "pkg/__init__.py", "n-revs": 331, "tiny-revs": 300}]
|
|
69
|
+
self.assertEqual([x["file"] for x in watch.risks(r)], ["core/parser.py"])
|
|
70
|
+
|
|
64
71
|
def test_test_companions_and_weak_pairs_are_not_reasons(self):
|
|
65
72
|
top = watch.risks(report())[0]
|
|
66
73
|
coupling = [r for r in top["reasons"] if r.startswith("changes with")][0]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|