gitmole 0.3.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gitmole-0.3.0 → gitmole-0.4.0}/PKG-INFO +21 -20
- {gitmole-0.3.0 → gitmole-0.4.0}/README.md +20 -19
- {gitmole-0.3.0 → gitmole-0.4.0}/gitmole/__init__.py +1 -1
- {gitmole-0.3.0 → gitmole-0.4.0}/gitmole/cli.py +1 -1
- {gitmole-0.3.0 → gitmole-0.4.0}/gitmole/findings.py +1 -1
- {gitmole-0.3.0 → gitmole-0.4.0}/gitmole/leaks.py +9 -8
- {gitmole-0.3.0 → gitmole-0.4.0}/gitmole/load.py +1 -1
- {gitmole-0.3.0 → gitmole-0.4.0}/gitmole/run.py +2 -2
- {gitmole-0.3.0 → gitmole-0.4.0}/gitmole.egg-info/PKG-INFO +21 -20
- {gitmole-0.3.0 → gitmole-0.4.0}/tests/test_findings.py +2 -2
- {gitmole-0.3.0 → gitmole-0.4.0}/tests/test_leaks.py +12 -10
- {gitmole-0.3.0 → gitmole-0.4.0}/tests/test_run.py +6 -6
- {gitmole-0.3.0 → gitmole-0.4.0}/LICENSE +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/gitmole/__main__.py +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/gitmole/backtest.py +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/gitmole/banner.py +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/gitmole/blame.py +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/gitmole/filetypes.py +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/gitmole/functions.py +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/gitmole/hotspots.py +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/gitmole/identity.py +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/gitmole/knowledge.py +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/gitmole/loss.py +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/gitmole/maat.py +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/gitmole/render.py +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/gitmole/textfmt.py +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/gitmole/trend.py +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/gitmole/watch.py +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/gitmole.egg-info/SOURCES.txt +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/gitmole.egg-info/dependency_links.txt +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/gitmole.egg-info/entry_points.txt +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/gitmole.egg-info/requires.txt +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/gitmole.egg-info/top_level.txt +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/pyproject.toml +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/setup.cfg +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/tests/test_backtest.py +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/tests/test_banner.py +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/tests/test_blame.py +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/tests/test_cli.py +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/tests/test_filetypes.py +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/tests/test_functions.py +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/tests/test_golden.py +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/tests/test_hotspots.py +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/tests/test_identity.py +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/tests/test_knowledge.py +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/tests/test_load.py +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/tests/test_loss.py +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/tests/test_maat.py +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/tests/test_packaging.py +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/tests/test_render.py +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/tests/test_textfmt.py +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/tests/test_trend.py +0 -0
- {gitmole-0.3.0 → gitmole-0.4.0}/tests/test_watch.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gitmole
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Offline git repository analysis with a terminal report: hotspots, coupling, ownership, code age, secrets, repo health.
|
|
5
5
|
License: MIT
|
|
6
6
|
Project-URL: Homepage, https://github.com/antvinni/gitmole
|
|
@@ -22,11 +22,11 @@ Dynamic: license-file
|
|
|
22
22
|
|
|
23
23
|
# gitmole
|
|
24
24
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
25
|
+
A toolkit for digging into any cloned git repository: who works on it,
|
|
26
|
+
where the risk is, how old the code is, whether the repo itself is healthy,
|
|
27
|
+
and whether anything sensitive was ever committed.
|
|
28
|
+
|
|
29
|
+
Any stack. Free. Offline. No token. No AI. Light.
|
|
30
30
|
|
|
31
31
|
## Principles
|
|
32
32
|
|
|
@@ -64,7 +64,7 @@ you about a clone.
|
|
|
64
64
|
| How old is the surviving code, per year and author | gitmole's own blame pass (one `git blame` per file at HEAD) | built in |
|
|
65
65
|
| Code-age and survival plots over time | [git-of-theseus](https://github.com/erikbern/git-of-theseus) | pip, opt-in with `--plots` |
|
|
66
66
|
| Per-function complexity, length, parameters; duplicated blocks with `--duplicates` | [lizard](https://github.com/terryyin/lizard) | pip, installed with gitmole; tracked code files only |
|
|
67
|
-
| Have secrets ever been committed | [
|
|
67
|
+
| Have secrets ever been committed | [betterleaks](https://github.com/betterleaks/betterleaks) | brew |
|
|
68
68
|
|
|
69
69
|
Why these and not others: [docs/tools.md](https://github.com/antvinni/gitmole/blob/main/docs/tools.md).
|
|
70
70
|
|
|
@@ -73,7 +73,7 @@ Why these and not others: [docs/tools.md](https://github.com/antvinni/gitmole/bl
|
|
|
73
73
|
gitmole needs git, Python 3.9 or newer, and three tools on your PATH:
|
|
74
74
|
[scc](https://github.com/boyter/scc) for size,
|
|
75
75
|
[git-sizer](https://github.com/github/git-sizer) for repository health and
|
|
76
|
-
[
|
|
76
|
+
[betterleaks](https://github.com/betterleaks/betterleaks) for secrets. gitmole itself
|
|
77
77
|
is a Python package; install it with [pipx](https://pipx.pypa.io) so it gets
|
|
78
78
|
its own environment and a `gitmole` command.
|
|
79
79
|
|
|
@@ -109,23 +109,23 @@ a static binary, so dropping it into `~/.local/bin` is enough.
|
|
|
109
109
|
sudo apt install git git-sizer pipx
|
|
110
110
|
pipx ensurepath # once; then open a new shell
|
|
111
111
|
|
|
112
|
-
# scc and
|
|
113
|
-
# https://github.com/boyter/scc/releases
|
|
114
|
-
# https://github.com/
|
|
112
|
+
# scc and betterleaks: one static binary each, from their release pages
|
|
113
|
+
# https://github.com/boyter/scc/releases (the Linux x86_64 or arm64 archive)
|
|
114
|
+
# https://github.com/betterleaks/betterleaks/releases (the linux x64 or arm64 archive)
|
|
115
115
|
# unpack and move the binary into ~/.local/bin, then:
|
|
116
|
-
chmod +x ~/.local/bin/scc ~/.local/bin/
|
|
116
|
+
chmod +x ~/.local/bin/scc ~/.local/bin/betterleaks
|
|
117
117
|
|
|
118
118
|
pipx install gitmole
|
|
119
119
|
```
|
|
120
120
|
|
|
121
121
|
On a distribution without a `pipx` package, `python3 -m pip install --user
|
|
122
|
-
pipx` installs it. Some distributions package scc or
|
|
122
|
+
pipx` installs it. Some distributions package scc or betterleaks as well; if
|
|
123
123
|
yours does, prefer that to a downloaded binary.
|
|
124
124
|
|
|
125
125
|
### Check
|
|
126
126
|
|
|
127
127
|
```bash
|
|
128
|
-
scc --version && git-sizer --version &&
|
|
128
|
+
scc --version && git-sizer --version && betterleaks version && gitmole --version
|
|
129
129
|
gitmole . # a report of the clone you are in
|
|
130
130
|
```
|
|
131
131
|
|
|
@@ -326,8 +326,9 @@ The full report is in [docs/example.md](https://github.com/antvinni/gitmole/blob
|
|
|
326
326
|
web pages, is a warning. Version strings and tokens shortened with "..."
|
|
327
327
|
cannot be live secrets, so they are left out and counted on the footer
|
|
328
328
|
line. Nothing is skipped by prefix. To silence a false positive for
|
|
329
|
-
good, copy its fingerprint from `secrets.json` into a `.
|
|
330
|
-
at the repository root;
|
|
329
|
+
good, copy its fingerprint from `secrets.json` into a `.betterleaksignore`
|
|
330
|
+
at the repository root; betterleaks reads it on the next run, and an
|
|
331
|
+
existing `.gitleaksignore` works too.
|
|
331
332
|
|
|
332
333
|
A commit counts as a fix when its subject starts with `fix:`, `hotfix:` or
|
|
333
334
|
`bugfix:` in the conventional style, or mentions fix, bug, hotfix,
|
|
@@ -387,7 +388,7 @@ The files each run writes, and how to read them: [docs/output.md](https://github
|
|
|
387
388
|
## Development
|
|
388
389
|
|
|
389
390
|
Developer setup: Homebrew for the three tools,
|
|
390
|
-
`brew install scc git-sizer
|
|
391
|
+
`brew install scc git-sizer betterleaks`. Then either a virtual environment
|
|
391
392
|
with `pip install -e .`, or the checkout style:
|
|
392
393
|
`python3 -m pip install --user rich lizard` and
|
|
393
394
|
`ln -sfn "$PWD/bin/gitmole" "$(brew --prefix)/bin/gitmole"`, which makes
|
|
@@ -438,8 +439,8 @@ and `render.py` draws the report. `bin/gitmole` is a thin launcher.
|
|
|
438
439
|
- Everything here is offline except the optional clone step, which uses
|
|
439
440
|
your existing gh auth. None of the tools send data anywhere.
|
|
440
441
|
- Remote targets are cloned into a fresh temp directory. Local clones are
|
|
441
|
-
only read, but the log export and the
|
|
442
|
-
- Secret values never reach the output directory.
|
|
442
|
+
only read, but the log export and the betterleaks scan touch all branches.
|
|
443
|
+
- Secret values never reach the output directory. betterleaks writes its report
|
|
443
444
|
to gitmole in memory, and gitmole stores a short keyed hash of each value
|
|
444
445
|
in place of the value, the matched text and the commit message. The key is
|
|
445
446
|
random, made for that one report and never saved, so a stored hash cannot
|
|
@@ -459,7 +460,7 @@ separate processes. Their licences:
|
|
|
459
460
|
|---|---|
|
|
460
461
|
| scc | MIT |
|
|
461
462
|
| git-sizer | MIT |
|
|
462
|
-
|
|
|
463
|
+
| betterleaks | MIT |
|
|
463
464
|
| rich | MIT |
|
|
464
465
|
| lizard | MIT |
|
|
465
466
|
| git-of-theseus | Apache-2.0 |
|
|
@@ -2,11 +2,11 @@
|
|
|
2
2
|
|
|
3
3
|
# gitmole
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
5
|
+
A toolkit for digging into any cloned git repository: who works on it,
|
|
6
|
+
where the risk is, how old the code is, whether the repo itself is healthy,
|
|
7
|
+
and whether anything sensitive was ever committed.
|
|
8
|
+
|
|
9
|
+
Any stack. Free. Offline. No token. No AI. Light.
|
|
10
10
|
|
|
11
11
|
## Principles
|
|
12
12
|
|
|
@@ -44,7 +44,7 @@ you about a clone.
|
|
|
44
44
|
| How old is the surviving code, per year and author | gitmole's own blame pass (one `git blame` per file at HEAD) | built in |
|
|
45
45
|
| Code-age and survival plots over time | [git-of-theseus](https://github.com/erikbern/git-of-theseus) | pip, opt-in with `--plots` |
|
|
46
46
|
| Per-function complexity, length, parameters; duplicated blocks with `--duplicates` | [lizard](https://github.com/terryyin/lizard) | pip, installed with gitmole; tracked code files only |
|
|
47
|
-
| Have secrets ever been committed | [
|
|
47
|
+
| Have secrets ever been committed | [betterleaks](https://github.com/betterleaks/betterleaks) | brew |
|
|
48
48
|
|
|
49
49
|
Why these and not others: [docs/tools.md](https://github.com/antvinni/gitmole/blob/main/docs/tools.md).
|
|
50
50
|
|
|
@@ -53,7 +53,7 @@ Why these and not others: [docs/tools.md](https://github.com/antvinni/gitmole/bl
|
|
|
53
53
|
gitmole needs git, Python 3.9 or newer, and three tools on your PATH:
|
|
54
54
|
[scc](https://github.com/boyter/scc) for size,
|
|
55
55
|
[git-sizer](https://github.com/github/git-sizer) for repository health and
|
|
56
|
-
[
|
|
56
|
+
[betterleaks](https://github.com/betterleaks/betterleaks) for secrets. gitmole itself
|
|
57
57
|
is a Python package; install it with [pipx](https://pipx.pypa.io) so it gets
|
|
58
58
|
its own environment and a `gitmole` command.
|
|
59
59
|
|
|
@@ -89,23 +89,23 @@ a static binary, so dropping it into `~/.local/bin` is enough.
|
|
|
89
89
|
sudo apt install git git-sizer pipx
|
|
90
90
|
pipx ensurepath # once; then open a new shell
|
|
91
91
|
|
|
92
|
-
# scc and
|
|
93
|
-
# https://github.com/boyter/scc/releases
|
|
94
|
-
# https://github.com/
|
|
92
|
+
# scc and betterleaks: one static binary each, from their release pages
|
|
93
|
+
# https://github.com/boyter/scc/releases (the Linux x86_64 or arm64 archive)
|
|
94
|
+
# https://github.com/betterleaks/betterleaks/releases (the linux x64 or arm64 archive)
|
|
95
95
|
# unpack and move the binary into ~/.local/bin, then:
|
|
96
|
-
chmod +x ~/.local/bin/scc ~/.local/bin/
|
|
96
|
+
chmod +x ~/.local/bin/scc ~/.local/bin/betterleaks
|
|
97
97
|
|
|
98
98
|
pipx install gitmole
|
|
99
99
|
```
|
|
100
100
|
|
|
101
101
|
On a distribution without a `pipx` package, `python3 -m pip install --user
|
|
102
|
-
pipx` installs it. Some distributions package scc or
|
|
102
|
+
pipx` installs it. Some distributions package scc or betterleaks as well; if
|
|
103
103
|
yours does, prefer that to a downloaded binary.
|
|
104
104
|
|
|
105
105
|
### Check
|
|
106
106
|
|
|
107
107
|
```bash
|
|
108
|
-
scc --version && git-sizer --version &&
|
|
108
|
+
scc --version && git-sizer --version && betterleaks version && gitmole --version
|
|
109
109
|
gitmole . # a report of the clone you are in
|
|
110
110
|
```
|
|
111
111
|
|
|
@@ -306,8 +306,9 @@ The full report is in [docs/example.md](https://github.com/antvinni/gitmole/blob
|
|
|
306
306
|
web pages, is a warning. Version strings and tokens shortened with "..."
|
|
307
307
|
cannot be live secrets, so they are left out and counted on the footer
|
|
308
308
|
line. Nothing is skipped by prefix. To silence a false positive for
|
|
309
|
-
good, copy its fingerprint from `secrets.json` into a `.
|
|
310
|
-
at the repository root;
|
|
309
|
+
good, copy its fingerprint from `secrets.json` into a `.betterleaksignore`
|
|
310
|
+
at the repository root; betterleaks reads it on the next run, and an
|
|
311
|
+
existing `.gitleaksignore` works too.
|
|
311
312
|
|
|
312
313
|
A commit counts as a fix when its subject starts with `fix:`, `hotfix:` or
|
|
313
314
|
`bugfix:` in the conventional style, or mentions fix, bug, hotfix,
|
|
@@ -367,7 +368,7 @@ The files each run writes, and how to read them: [docs/output.md](https://github
|
|
|
367
368
|
## Development
|
|
368
369
|
|
|
369
370
|
Developer setup: Homebrew for the three tools,
|
|
370
|
-
`brew install scc git-sizer
|
|
371
|
+
`brew install scc git-sizer betterleaks`. Then either a virtual environment
|
|
371
372
|
with `pip install -e .`, or the checkout style:
|
|
372
373
|
`python3 -m pip install --user rich lizard` and
|
|
373
374
|
`ln -sfn "$PWD/bin/gitmole" "$(brew --prefix)/bin/gitmole"`, which makes
|
|
@@ -418,8 +419,8 @@ and `render.py` draws the report. `bin/gitmole` is a thin launcher.
|
|
|
418
419
|
- Everything here is offline except the optional clone step, which uses
|
|
419
420
|
your existing gh auth. None of the tools send data anywhere.
|
|
420
421
|
- Remote targets are cloned into a fresh temp directory. Local clones are
|
|
421
|
-
only read, but the log export and the
|
|
422
|
-
- Secret values never reach the output directory.
|
|
422
|
+
only read, but the log export and the betterleaks scan touch all branches.
|
|
423
|
+
- Secret values never reach the output directory. betterleaks writes its report
|
|
423
424
|
to gitmole in memory, and gitmole stores a short keyed hash of each value
|
|
424
425
|
in place of the value, the matched text and the commit message. The key is
|
|
425
426
|
random, made for that one report and never saved, so a stored hash cannot
|
|
@@ -439,7 +440,7 @@ separate processes. Their licences:
|
|
|
439
440
|
|---|---|
|
|
440
441
|
| scc | MIT |
|
|
441
442
|
| git-sizer | MIT |
|
|
442
|
-
|
|
|
443
|
+
| betterleaks | MIT |
|
|
443
444
|
| rich | MIT |
|
|
444
445
|
| lizard | MIT |
|
|
445
446
|
| git-of-theseus | Apache-2.0 |
|
|
@@ -95,7 +95,7 @@ def main(argv=None, console: Console = None, tool_check=run.missing_tools, plann
|
|
|
95
95
|
missing = tool_check(plots=args.plots)
|
|
96
96
|
if missing:
|
|
97
97
|
err.print("[red]missing tools:[/red] " + ", ".join(missing))
|
|
98
|
-
err.print("brew install scc git-sizer
|
|
98
|
+
err.print("brew install scc git-sizer betterleaks; see README.md for other ways")
|
|
99
99
|
return 2
|
|
100
100
|
args.lizard = lizard_check() # decided once, for every repository this run analyses
|
|
101
101
|
|
|
@@ -45,7 +45,7 @@ def secrets_found(report: dict) -> list:
|
|
|
45
45
|
groups = leaks.group(report.get("secrets") or [])
|
|
46
46
|
source = [g for g in groups if not g["test"]]
|
|
47
47
|
tests = [g for g in groups if g["test"]]
|
|
48
|
-
ignore = "Add the fingerprint of any false positive from secrets.json to .
|
|
48
|
+
ignore = "Add the fingerprint of any false positive from secrets.json to .betterleaksignore in the repository."
|
|
49
49
|
out = []
|
|
50
50
|
if source:
|
|
51
51
|
out.append(_f("critical", f"{len(source)} secret(s) in history", _secret_statement(source),
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
|
-
"""
|
|
2
|
+
"""betterleaks, with the secret values kept out of the output directory.
|
|
3
3
|
|
|
4
|
-
gitmole runs this as the
|
|
5
|
-
|
|
4
|
+
gitmole runs this as the betterleaks step: `python3 leaks.py OUT_JSON`, from inside the repository.
|
|
5
|
+
betterleaks writes its JSON report to our stdout, so the raw report is never a file; each value is
|
|
6
6
|
replaced by a short keyed hash (enough to tell one value repeated in many places from many values)
|
|
7
7
|
and a flag for shapes that cannot be a live secret, and only that is written. The key is random,
|
|
8
8
|
made for one report and never stored, so a hash in secrets.json cannot be checked against a list of
|
|
@@ -27,8 +27,9 @@ try:
|
|
|
27
27
|
except ImportError: # run as a script: the package directory is sys.path[0]
|
|
28
28
|
import filetypes
|
|
29
29
|
|
|
30
|
-
ARGV = ["
|
|
31
|
-
|
|
30
|
+
ARGV = ["betterleaks", "git", "--no-banner", "--report-format", "json", "--report-path", "-", "--exit-code", "0"]
|
|
31
|
+
# the value, the text around it, and the commit message, which can quote it; Attributes repeats the message
|
|
32
|
+
RAW_FIELDS = ("Secret", "Match", "Line", "Message", "Attributes")
|
|
32
33
|
|
|
33
34
|
# A version string (5.0.0-1667386184.dfbbb54) and a token shortened with an ellipsis are the only shapes
|
|
34
35
|
# skipped. Nothing is skipped by prefix: a public and a private key of the same service often share one.
|
|
@@ -102,13 +103,13 @@ def main(argv=None) -> int:
|
|
|
102
103
|
print("usage: leaks.py OUT_JSON", file=sys.stderr)
|
|
103
104
|
return 2
|
|
104
105
|
target = args[0]
|
|
105
|
-
# stderr is inherited, so
|
|
106
|
+
# stderr is inherited, so betterleaks' own log lands in run.log as before
|
|
106
107
|
proc = subprocess.run(ARGV, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE)
|
|
107
108
|
if proc.returncode != 0:
|
|
108
|
-
print(f"leaks.py:
|
|
109
|
+
print(f"leaks.py: betterleaks exited {proc.returncode}; no report written", file=sys.stderr)
|
|
109
110
|
return proc.returncode
|
|
110
111
|
text = proc.stdout.decode("utf-8", "surrogateescape").strip()
|
|
111
|
-
rows = sanitise(json.loads(text) if text else [])
|
|
112
|
+
rows = sanitise((json.loads(text) if text else None) or []) # a clean repository is reported as null
|
|
112
113
|
fd, tmp = tempfile.mkstemp(dir=os.path.dirname(os.path.abspath(target)), prefix=".secrets-", suffix=".json")
|
|
113
114
|
try:
|
|
114
115
|
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
|
@@ -180,7 +180,7 @@ def parse_duplicates(text: str) -> dict:
|
|
|
180
180
|
|
|
181
181
|
|
|
182
182
|
def parse_secrets(text: str) -> list:
|
|
183
|
-
"""
|
|
183
|
+
"""betterleaks rows as rule, file, short commit, line, fingerprint, the hashed value and the placeholder
|
|
184
184
|
flag. A report written before values were hashed still has them: hash them here, keep nothing raw."""
|
|
185
185
|
rows = json.loads(text) if text.strip() else []
|
|
186
186
|
key = leaks.new_key() # for an older report with raw values: one key per read, as the wrapper does per run
|
|
@@ -109,7 +109,7 @@ def env_path() -> str:
|
|
|
109
109
|
return os.pathsep.join(parts + [os.environ.get("PATH", "")])
|
|
110
110
|
|
|
111
111
|
|
|
112
|
-
REQUIRED_TOOLS = ["scc", "git-sizer", "
|
|
112
|
+
REQUIRED_TOOLS = ["scc", "git-sizer", "betterleaks"]
|
|
113
113
|
PLOT_TOOLS = ["git-of-theseus-analyze"]
|
|
114
114
|
|
|
115
115
|
|
|
@@ -168,7 +168,7 @@ def plan(repo_dir: str, out_dir: str, branch: str = "HEAD", age: bool = True, pl
|
|
|
168
168
|
steps = [
|
|
169
169
|
{"name": "scc", "argv": ["scc", "--by-file", "--format", "json"], "stdout": o("size.json"), "deps": []},
|
|
170
170
|
{"name": "git-sizer", "argv": ["git-sizer", "--verbose"], "stdout": o("repo-health.txt"), "deps": []},
|
|
171
|
-
{"name": "
|
|
171
|
+
{"name": "betterleaks", "argv": [sys.executable, LEAKS_SCRIPT, o("secrets.json")], "stdout": None, "deps": []}, # hashes the values before anything is written
|
|
172
172
|
{"name": "git-log", "argv": [*filetypes.GIT, "log", "--all", "--use-mailmap", "--numstat", "--date=iso-strict", "--pretty=format:--%h--%ad--%aN--%s", "--no-renames"], "stdout": log, "deps": []},
|
|
173
173
|
{"name": "change analysis", "argv": [sys.executable, MAAT_SCRIPT, log, out_dir, *type_args, *(["--now", now] if now else []), *(["--since", since] if since else []), "--aliases", o("meta.json")], "stdout": None, "deps": ["git-log"]},
|
|
174
174
|
]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gitmole
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Offline git repository analysis with a terminal report: hotspots, coupling, ownership, code age, secrets, repo health.
|
|
5
5
|
License: MIT
|
|
6
6
|
Project-URL: Homepage, https://github.com/antvinni/gitmole
|
|
@@ -22,11 +22,11 @@ Dynamic: license-file
|
|
|
22
22
|
|
|
23
23
|
# gitmole
|
|
24
24
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
25
|
+
A toolkit for digging into any cloned git repository: who works on it,
|
|
26
|
+
where the risk is, how old the code is, whether the repo itself is healthy,
|
|
27
|
+
and whether anything sensitive was ever committed.
|
|
28
|
+
|
|
29
|
+
Any stack. Free. Offline. No token. No AI. Light.
|
|
30
30
|
|
|
31
31
|
## Principles
|
|
32
32
|
|
|
@@ -64,7 +64,7 @@ you about a clone.
|
|
|
64
64
|
| How old is the surviving code, per year and author | gitmole's own blame pass (one `git blame` per file at HEAD) | built in |
|
|
65
65
|
| Code-age and survival plots over time | [git-of-theseus](https://github.com/erikbern/git-of-theseus) | pip, opt-in with `--plots` |
|
|
66
66
|
| Per-function complexity, length, parameters; duplicated blocks with `--duplicates` | [lizard](https://github.com/terryyin/lizard) | pip, installed with gitmole; tracked code files only |
|
|
67
|
-
| Have secrets ever been committed | [
|
|
67
|
+
| Have secrets ever been committed | [betterleaks](https://github.com/betterleaks/betterleaks) | brew |
|
|
68
68
|
|
|
69
69
|
Why these and not others: [docs/tools.md](https://github.com/antvinni/gitmole/blob/main/docs/tools.md).
|
|
70
70
|
|
|
@@ -73,7 +73,7 @@ Why these and not others: [docs/tools.md](https://github.com/antvinni/gitmole/bl
|
|
|
73
73
|
gitmole needs git, Python 3.9 or newer, and three tools on your PATH:
|
|
74
74
|
[scc](https://github.com/boyter/scc) for size,
|
|
75
75
|
[git-sizer](https://github.com/github/git-sizer) for repository health and
|
|
76
|
-
[
|
|
76
|
+
[betterleaks](https://github.com/betterleaks/betterleaks) for secrets. gitmole itself
|
|
77
77
|
is a Python package; install it with [pipx](https://pipx.pypa.io) so it gets
|
|
78
78
|
its own environment and a `gitmole` command.
|
|
79
79
|
|
|
@@ -109,23 +109,23 @@ a static binary, so dropping it into `~/.local/bin` is enough.
|
|
|
109
109
|
sudo apt install git git-sizer pipx
|
|
110
110
|
pipx ensurepath # once; then open a new shell
|
|
111
111
|
|
|
112
|
-
# scc and
|
|
113
|
-
# https://github.com/boyter/scc/releases
|
|
114
|
-
# https://github.com/
|
|
112
|
+
# scc and betterleaks: one static binary each, from their release pages
|
|
113
|
+
# https://github.com/boyter/scc/releases (the Linux x86_64 or arm64 archive)
|
|
114
|
+
# https://github.com/betterleaks/betterleaks/releases (the linux x64 or arm64 archive)
|
|
115
115
|
# unpack and move the binary into ~/.local/bin, then:
|
|
116
|
-
chmod +x ~/.local/bin/scc ~/.local/bin/
|
|
116
|
+
chmod +x ~/.local/bin/scc ~/.local/bin/betterleaks
|
|
117
117
|
|
|
118
118
|
pipx install gitmole
|
|
119
119
|
```
|
|
120
120
|
|
|
121
121
|
On a distribution without a `pipx` package, `python3 -m pip install --user
|
|
122
|
-
pipx` installs it. Some distributions package scc or
|
|
122
|
+
pipx` installs it. Some distributions package scc or betterleaks as well; if
|
|
123
123
|
yours does, prefer that to a downloaded binary.
|
|
124
124
|
|
|
125
125
|
### Check
|
|
126
126
|
|
|
127
127
|
```bash
|
|
128
|
-
scc --version && git-sizer --version &&
|
|
128
|
+
scc --version && git-sizer --version && betterleaks version && gitmole --version
|
|
129
129
|
gitmole . # a report of the clone you are in
|
|
130
130
|
```
|
|
131
131
|
|
|
@@ -326,8 +326,9 @@ The full report is in [docs/example.md](https://github.com/antvinni/gitmole/blob
|
|
|
326
326
|
web pages, is a warning. Version strings and tokens shortened with "..."
|
|
327
327
|
cannot be live secrets, so they are left out and counted on the footer
|
|
328
328
|
line. Nothing is skipped by prefix. To silence a false positive for
|
|
329
|
-
good, copy its fingerprint from `secrets.json` into a `.
|
|
330
|
-
at the repository root;
|
|
329
|
+
good, copy its fingerprint from `secrets.json` into a `.betterleaksignore`
|
|
330
|
+
at the repository root; betterleaks reads it on the next run, and an
|
|
331
|
+
existing `.gitleaksignore` works too.
|
|
331
332
|
|
|
332
333
|
A commit counts as a fix when its subject starts with `fix:`, `hotfix:` or
|
|
333
334
|
`bugfix:` in the conventional style, or mentions fix, bug, hotfix,
|
|
@@ -387,7 +388,7 @@ The files each run writes, and how to read them: [docs/output.md](https://github
|
|
|
387
388
|
## Development
|
|
388
389
|
|
|
389
390
|
Developer setup: Homebrew for the three tools,
|
|
390
|
-
`brew install scc git-sizer
|
|
391
|
+
`brew install scc git-sizer betterleaks`. Then either a virtual environment
|
|
391
392
|
with `pip install -e .`, or the checkout style:
|
|
392
393
|
`python3 -m pip install --user rich lizard` and
|
|
393
394
|
`ln -sfn "$PWD/bin/gitmole" "$(brew --prefix)/bin/gitmole"`, which makes
|
|
@@ -438,8 +439,8 @@ and `render.py` draws the report. `bin/gitmole` is a thin launcher.
|
|
|
438
439
|
- Everything here is offline except the optional clone step, which uses
|
|
439
440
|
your existing gh auth. None of the tools send data anywhere.
|
|
440
441
|
- Remote targets are cloned into a fresh temp directory. Local clones are
|
|
441
|
-
only read, but the log export and the
|
|
442
|
-
- Secret values never reach the output directory.
|
|
442
|
+
only read, but the log export and the betterleaks scan touch all branches.
|
|
443
|
+
- Secret values never reach the output directory. betterleaks writes its report
|
|
443
444
|
to gitmole in memory, and gitmole stores a short keyed hash of each value
|
|
444
445
|
in place of the value, the matched text and the commit message. The key is
|
|
445
446
|
random, made for that one report and never saved, so a stored hash cannot
|
|
@@ -459,7 +460,7 @@ separate processes. Their licences:
|
|
|
459
460
|
|---|---|
|
|
460
461
|
| scc | MIT |
|
|
461
462
|
| git-sizer | MIT |
|
|
462
|
-
|
|
|
463
|
+
| betterleaks | MIT |
|
|
463
464
|
| rich | MIT |
|
|
464
465
|
| lizard | MIT |
|
|
465
466
|
| git-of-theseus | Apache-2.0 |
|
|
@@ -46,11 +46,11 @@ class SecretsFound(unittest.TestCase):
|
|
|
46
46
|
self.assertEqual(crit["title"], "1 secret(s) in history")
|
|
47
47
|
self.assertIn("1 distinct value in 2 places: generic-api-key in app/settings.py (c1, c2)", crit["detail"])
|
|
48
48
|
self.assertIn("Rotate", crit["advice"])
|
|
49
|
-
self.assertIn(".
|
|
49
|
+
self.assertIn(".betterleaksignore", crit["advice"])
|
|
50
50
|
self.assertEqual(warn["title"], "2 secret(s) only in test files")
|
|
51
51
|
self.assertIn("2 distinct values in 3 places", warn["detail"])
|
|
52
52
|
self.assertIn("tests/data/a.html and 1 other file", warn["detail"])
|
|
53
|
-
self.assertIn(".
|
|
53
|
+
self.assertIn(".betterleaksignore", warn["advice"])
|
|
54
54
|
|
|
55
55
|
def test_test_only_secrets_do_not_fail_a_critical_gate(self):
|
|
56
56
|
r = report(secrets=[self.row("h3", "tests/t.py")])
|
|
@@ -11,14 +11,15 @@ from gitmole import leaks
|
|
|
11
11
|
|
|
12
12
|
SCRIPT = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "gitmole", "leaks.py")
|
|
13
13
|
|
|
14
|
-
# Synthetic values only, built at runtime: a secret-shaped literal would make
|
|
14
|
+
# Synthetic values only, built at runtime: a secret-shaped literal would make betterleaks, GitHub push
|
|
15
15
|
# protection and gitmole itself flag this file.
|
|
16
16
|
FAKE = "0123456789abcdef" * 2
|
|
17
17
|
VERSION = "5.0.0-" + "1667386184.dfbbb54"
|
|
18
18
|
RAW = [
|
|
19
19
|
{"RuleID": "generic-api-key", "File": "app/settings.py", "Commit": "c1c1c1c1c1", "StartLine": 9, "Fingerprint": "c1c1c1c1c1:app/settings.py:generic-api-key:9",
|
|
20
20
|
"Secret": FAKE, "Match": f'SECRET = "{FAKE}"', "Line": f'SECRET = "{FAKE}"',
|
|
21
|
-
"Author": "Ann", "Message": f"rotate {FAKE} out of settings"
|
|
21
|
+
"Author": "Ann", "Message": f"rotate {FAKE} out of settings", # a message can quote the value
|
|
22
|
+
"Attributes": {"confidence": "high", "git.message": f"rotate {FAKE} out of settings", "path": "app/settings.py"}}, # betterleaks repeats it here
|
|
22
23
|
{"RuleID": "generic-api-key", "File": "web/package.json", "Commit": "d2d2d2d2d2", "StartLine": 21, "Fingerprint": "d2d2d2d2d2:web/package.json:generic-api-key:21",
|
|
23
24
|
"Secret": VERSION, "Match": f'auth-next": "{VERSION}"'},
|
|
24
25
|
]
|
|
@@ -58,7 +59,7 @@ class Sanitise(unittest.TestCase):
|
|
|
58
59
|
self.assertNotIn("0123456789abcdef", text)
|
|
59
60
|
self.assertNotIn("dfbbb54", text)
|
|
60
61
|
for row in rows:
|
|
61
|
-
self.assertFalse({"Secret", "Match", "Line", "Message"} & set(row), row)
|
|
62
|
+
self.assertFalse({"Secret", "Match", "Line", "Message", "Attributes"} & set(row), row)
|
|
62
63
|
self.assertEqual(len(rows[0]["SecretHash"]), 12)
|
|
63
64
|
again = leaks.sanitise(RAW + RAW)
|
|
64
65
|
self.assertEqual(again[0]["SecretHash"], again[2]["SecretHash"], "one key per report: repeats still group")
|
|
@@ -69,7 +70,7 @@ class Sanitise(unittest.TestCase):
|
|
|
69
70
|
|
|
70
71
|
|
|
71
72
|
class Script(unittest.TestCase):
|
|
72
|
-
"""Runs the script against a stand-in
|
|
73
|
+
"""Runs the script against a stand-in betterleaks on PATH, so the wiring is tested without real keys."""
|
|
73
74
|
|
|
74
75
|
def _run(self, stdout: str, rc: int = 0):
|
|
75
76
|
with tempfile.TemporaryDirectory() as d:
|
|
@@ -78,7 +79,7 @@ class Script(unittest.TestCase):
|
|
|
78
79
|
os.makedirs(p)
|
|
79
80
|
with open(os.path.join(d, "stdout.json"), "w") as fh:
|
|
80
81
|
fh.write(stdout)
|
|
81
|
-
fake = os.path.join(bindir, "
|
|
82
|
+
fake = os.path.join(bindir, "betterleaks")
|
|
82
83
|
with open(fake, "w") as fh:
|
|
83
84
|
fh.write(f"#!/bin/sh\necho \"$@\" > {d}/argv\npwd > {d}/cwd\ncat {d}/stdout.json\necho 'INF scanned' >&2\nexit {rc}\n")
|
|
84
85
|
os.chmod(fake, os.stat(fake).st_mode | stat.S_IEXEC)
|
|
@@ -102,16 +103,17 @@ class Script(unittest.TestCase):
|
|
|
102
103
|
self.assertEqual(argv[:1], ["git"])
|
|
103
104
|
self.assertEqual(argv[argv.index("--report-path") + 1], "-", "the raw report must never be a file")
|
|
104
105
|
self.assertEqual(argv[argv.index("--report-format") + 1], "json")
|
|
105
|
-
self.assertEqual(os.path.realpath(cwd), os.path.realpath(repo), "
|
|
106
|
+
self.assertEqual(os.path.realpath(cwd), os.path.realpath(repo), "betterleaks scans the repository it is started in")
|
|
106
107
|
self.assertNotIn("0123456789abcdef", written)
|
|
107
108
|
self.assertEqual(len(json.loads(written)), 2)
|
|
108
109
|
self.assertEqual(leftovers, ["secrets.json"], "no temporary file is left behind")
|
|
109
|
-
self.assertIn("INF scanned", p.stderr, "
|
|
110
|
+
self.assertIn("INF scanned", p.stderr, "betterleaks' own log still reaches run.log")
|
|
110
111
|
|
|
111
112
|
def test_no_findings_writes_an_empty_list(self):
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
113
|
+
for empty in ["[]", "null"]: # betterleaks prints null for a clean repository
|
|
114
|
+
p, _, _, written, _, _ = self._run(empty)
|
|
115
|
+
self.assertEqual(p.returncode, 0, p.stderr)
|
|
116
|
+
self.assertEqual(json.loads(written), [], empty)
|
|
115
117
|
|
|
116
118
|
def test_a_failed_scan_writes_nothing_and_fails_the_step(self):
|
|
117
119
|
p, _, _, written, leftovers, _ = self._run(json.dumps(RAW), rc=2)
|
|
@@ -170,7 +170,7 @@ class Plan(unittest.TestCase):
|
|
|
170
170
|
def test_lists_every_tool_and_theseus_plots_depend_on_analyze(self):
|
|
171
171
|
steps = run.plan("/r", "/o")
|
|
172
172
|
names = [s["name"] for s in steps]
|
|
173
|
-
for expected in ["scc", "git-sizer", "
|
|
173
|
+
for expected in ["scc", "git-sizer", "betterleaks", "git-log", "change analysis", "code age"]:
|
|
174
174
|
self.assertIn(expected, names)
|
|
175
175
|
for gone in ["onefetch", "git-quick-stats"]:
|
|
176
176
|
self.assertNotIn(gone, names)
|
|
@@ -218,13 +218,13 @@ class Plan(unittest.TestCase):
|
|
|
218
218
|
self.assertEqual(procs_for(procs=1), "1", "a smaller count is kept")
|
|
219
219
|
self.assertEqual(procs_for(), str(min(2, blame.default_procs())), "the default is clamped too")
|
|
220
220
|
|
|
221
|
-
def
|
|
221
|
+
def test_betterleaks_runs_through_the_bundled_wrapper_so_raw_secrets_never_reach_disk(self):
|
|
222
222
|
by = {s["name"]: s for s in run.plan("/r", "/o")}
|
|
223
|
-
argv = by["
|
|
223
|
+
argv = by["betterleaks"]["argv"]
|
|
224
224
|
self.assertEqual(argv[0], sys.executable)
|
|
225
225
|
self.assertTrue(argv[1].endswith("gitmole/leaks.py"), argv)
|
|
226
226
|
self.assertEqual(argv[2:], ["/o/secrets.json"])
|
|
227
|
-
self.assertIsNone(by["
|
|
227
|
+
self.assertIsNone(by["betterleaks"]["stdout"])
|
|
228
228
|
|
|
229
229
|
def test_lizard_is_detected_as_a_python_module_not_a_command(self):
|
|
230
230
|
self.assertNotIn("lizard", run.REQUIRED_TOOLS)
|
|
@@ -273,7 +273,7 @@ class Plan(unittest.TestCase):
|
|
|
273
273
|
|
|
274
274
|
def test_only_three_tools_required_by_default_and_theseus_with_plots(self):
|
|
275
275
|
"""Checked against a directory of stub executables, not this machine's PATH."""
|
|
276
|
-
self.assertEqual(run.REQUIRED_TOOLS, ["scc", "git-sizer", "
|
|
276
|
+
self.assertEqual(run.REQUIRED_TOOLS, ["scc", "git-sizer", "betterleaks"])
|
|
277
277
|
with tempfile.TemporaryDirectory() as d:
|
|
278
278
|
for name in run.REQUIRED_TOOLS:
|
|
279
279
|
stub = os.path.join(d, name)
|
|
@@ -285,7 +285,7 @@ class Plan(unittest.TestCase):
|
|
|
285
285
|
open(stub, "w").close()
|
|
286
286
|
os.chmod(stub, 0o755)
|
|
287
287
|
self.assertEqual(run.missing_tools(plots=True, path=d), [])
|
|
288
|
-
self.assertEqual(run.missing_tools(plots=True, path="/nonexistent"), ["scc", "git-sizer", "
|
|
288
|
+
self.assertEqual(run.missing_tools(plots=True, path="/nonexistent"), ["scc", "git-sizer", "betterleaks", "git-of-theseus-analyze"])
|
|
289
289
|
|
|
290
290
|
def test_theseus_tracks_the_given_branch(self):
|
|
291
291
|
by = {s["name"]: s for s in run.plan("/r", "/o", branch="trunk", plots=True)}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|