gitmole 0.6.15__tar.gz → 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gitmole-0.6.15 → gitmole-0.7.0}/PKG-INFO +24 -20
- {gitmole-0.6.15 → gitmole-0.7.0}/README.md +23 -19
- {gitmole-0.6.15 → gitmole-0.7.0}/gitmole/__init__.py +1 -1
- {gitmole-0.6.15 → gitmole-0.7.0}/gitmole/cli.py +21 -12
- gitmole-0.7.0/gitmole/deps.py +186 -0
- gitmole-0.7.0/gitmole/duplicates.py +153 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/gitmole/findings.py +62 -7
- {gitmole-0.6.15 → gitmole-0.7.0}/gitmole/functions.py +11 -39
- {gitmole-0.6.15 → gitmole-0.7.0}/gitmole/load.py +30 -2
- {gitmole-0.6.15 → gitmole-0.7.0}/gitmole/render.py +50 -8
- {gitmole-0.6.15 → gitmole-0.7.0}/gitmole/run.py +32 -17
- {gitmole-0.6.15 → gitmole-0.7.0}/gitmole.egg-info/PKG-INFO +24 -20
- {gitmole-0.6.15 → gitmole-0.7.0}/gitmole.egg-info/SOURCES.txt +4 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/tests/test_cli.py +45 -15
- gitmole-0.7.0/tests/test_deps.py +160 -0
- gitmole-0.7.0/tests/test_duplicates.py +168 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/tests/test_findings.py +68 -2
- {gitmole-0.6.15 → gitmole-0.7.0}/tests/test_functions.py +9 -33
- {gitmole-0.6.15 → gitmole-0.7.0}/tests/test_load.py +24 -2
- {gitmole-0.6.15 → gitmole-0.7.0}/tests/test_render.py +32 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/tests/test_run.py +32 -20
- {gitmole-0.6.15 → gitmole-0.7.0}/LICENSE +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/gitmole/__main__.py +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/gitmole/backtest.py +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/gitmole/banner.py +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/gitmole/blame.py +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/gitmole/clean.py +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/gitmole/coupling.py +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/gitmole/filetypes.py +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/gitmole/hotspots.py +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/gitmole/identity.py +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/gitmole/knowledge.py +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/gitmole/leaks.py +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/gitmole/loss.py +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/gitmole/maat.py +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/gitmole/textfmt.py +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/gitmole/trend.py +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/gitmole/watch.py +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/gitmole.egg-info/dependency_links.txt +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/gitmole.egg-info/entry_points.txt +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/gitmole.egg-info/requires.txt +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/gitmole.egg-info/top_level.txt +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/pyproject.toml +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/setup.cfg +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/tests/test_backtest.py +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/tests/test_banner.py +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/tests/test_blame.py +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/tests/test_clean.py +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/tests/test_coupling.py +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/tests/test_filetypes.py +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/tests/test_golden.py +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/tests/test_hotspots.py +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/tests/test_identity.py +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/tests/test_knowledge.py +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/tests/test_leaks.py +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/tests/test_loss.py +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/tests/test_maat.py +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/tests/test_packaging.py +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/tests/test_textfmt.py +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/tests/test_trend.py +0 -0
- {gitmole-0.6.15 → gitmole-0.7.0}/tests/test_watch.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gitmole
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Summary: Offline git repository analysis with a terminal report: hotspots, coupling, ownership, code age, secrets, repo health.
|
|
5
5
|
License: MIT
|
|
6
6
|
Project-URL: Homepage, https://github.com/antvinni/gitmole
|
|
@@ -30,7 +30,7 @@ Any stack. Free. Offline. No token. No AI. Light.
|
|
|
30
30
|
|
|
31
31
|
- **Free.** MIT licence, no paid tier, no account. The tools it runs are open source too.
|
|
32
32
|
- **Any stack.** It reads what every repository has: the git log, git blame and the files themselves.
|
|
33
|
-
- **Offline.** Everything runs against a clone on your machine. Nothing is uploaded, nothing phones home.
|
|
33
|
+
- **Offline.** Everything runs against a clone on your machine. Nothing is uploaded, nothing phones home; the vulnerability database is a copy you download once.
|
|
34
34
|
- **No token.** A local clone needs no credentials. The optional `owner/repo` shortcut uses the `gh` login you already have, and you ask for it.
|
|
35
35
|
- **No AI.** Every finding is a plain rule over counts you can recompute by hand. The same clone gives the same report every time.
|
|
36
36
|
- **Light.** A 4,400-commit repository takes under thirty seconds. A few thousand lines of Python plus two libraries.
|
|
@@ -38,12 +38,12 @@ Any stack. Free. Offline. No token. No AI. Light.
|
|
|
38
38
|
## Install
|
|
39
39
|
|
|
40
40
|
```bash
|
|
41
|
-
# macOS, or Linux with Homebrew: gitmole and the
|
|
41
|
+
# macOS, or Linux with Homebrew: gitmole and the five tools it runs
|
|
42
42
|
brew tap antvinni/gitmole https://github.com/antvinni/gitmole
|
|
43
43
|
brew trust antvinni/gitmole
|
|
44
44
|
brew install gitmole
|
|
45
45
|
|
|
46
|
-
# anywhere else: scc, git-sizer and
|
|
46
|
+
# anywhere else: scc, git-sizer, betterleaks, jscpd and osv-scanner on your PATH, then
|
|
47
47
|
pipx install gitmole
|
|
48
48
|
```
|
|
49
49
|
|
|
@@ -76,31 +76,31 @@ Running `gitmole .` inside this repository:
|
|
|
76
76
|
|
|
77
77
|
```text
|
|
78
78
|
╭─ gitmole ────────────────────────────────────────────────────────────────────────────────────────╮
|
|
79
|
-
│
|
|
80
|
-
│
|
|
79
|
+
│ 185 commits · 2026-09-15 → 2026-09-17 · 1 identity · branch main │
|
|
80
|
+
│ 9,475 lines in 52 files · Python, Ruby │
|
|
81
81
|
│ most commits on Wed at 20:00 · 4% of commits are fixes · 100% of surviving code from 2026 │
|
|
82
82
|
│ 3 warnings, 1 note │
|
|
83
83
|
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
84
84
|
╭─ Findings (4) ───────────────────────────────────────────────────────────────────────────────────╮
|
|
85
85
|
│ ▲ Bus factor of one │
|
|
86
86
|
│ vinni wrote 100% of the code that survives today │
|
|
87
|
-
│ ↳ Pair someone with vinni on gitmole/
|
|
87
|
+
│ ↳ Pair someone with vinni on gitmole/ first; it is 100% theirs. │
|
|
88
88
|
│ ▲ Hotspots getting more complex │
|
|
89
|
-
│ 4 of the 10 top source hotspots grew by 25% or more in a year: gitmole/render.py (+
|
|
90
|
-
│ gitmole/
|
|
91
|
-
│ ↳ Split gitmole/render.py before the next change; its complexity grew
|
|
89
|
+
│ 4 of the 10 top source hotspots grew by 25% or more in a year: gitmole/render.py (+194%), │
|
|
90
|
+
│ gitmole/findings.py (+360%), gitmole/cli.py (+64%), gitmole/run.py (+34%) │
|
|
91
|
+
│ ↳ Split gitmole/render.py before the next change; its complexity grew 194% in a year. │
|
|
92
92
|
│ ▲ Knowledge islands │
|
|
93
|
-
│
|
|
94
|
-
│ 100%)
|
|
95
|
-
│ ↳ Pair someone with vinni on gitmole/ first; it is the largest at
|
|
93
|
+
│ 1 area(s) with at least 200 lines were written almost entirely by one person: gitmole/ (vinni │
|
|
94
|
+
│ 100%). That is 98% of all lines added │
|
|
95
|
+
│ ↳ Pair someone with vinni on gitmole/ first; it is the largest at 7,607 lines. │
|
|
96
96
|
│ ● Bug magnets │
|
|
97
|
-
│
|
|
98
|
-
│ gitmole/
|
|
99
|
-
│ (3 recent, 3 total)
|
|
100
|
-
│ ↳ Review gitmole/
|
|
97
|
+
│ 5 file(s) were fixed 3+ times in the last six months: gitmole/load.py (4 recent, 4 total); │
|
|
98
|
+
│ gitmole/cli.py (3 recent, 3 total); gitmole/findings.py (3 recent, 3 total); gitmole/render.py │
|
|
99
|
+
│ (3 recent, 3 total); gitmole/run.py (3 recent, 3 total) │
|
|
100
|
+
│ ↳ Review gitmole/load.py and gitmole/cli.py before the next release; expect the next bug │
|
|
101
101
|
│ there. │
|
|
102
102
|
│ ✔ No secrets in history │
|
|
103
|
-
│ betterleaks scanned every commit on every branch
|
|
103
|
+
│ betterleaks scanned every commit on every branch; 28 placeholder-shaped hits left out │
|
|
104
104
|
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
105
105
|
```
|
|
106
106
|
|
|
@@ -126,8 +126,10 @@ you about a clone.
|
|
|
126
126
|
| Where is the risk: hotspots, coupling, ownership | gitmole's own change analysis over `git log --numstat` | built in |
|
|
127
127
|
| How old is the surviving code, per year and author | gitmole's own blame pass (one `git blame` per file at HEAD) | built in |
|
|
128
128
|
| Code-age and survival plots over time | [git-of-theseus](https://github.com/erikbern/git-of-theseus) | pip, opt-in with `--plots` |
|
|
129
|
-
| Per-function complexity, length, parameters
|
|
129
|
+
| Per-function complexity, length, parameters | [lizard](https://github.com/terryyin/lizard) | pip, installed with gitmole; tracked code files only |
|
|
130
|
+
| Which blocks of code appear more than once | [jscpd](https://github.com/kucherenko/jscpd) | brew |
|
|
130
131
|
| Have secrets ever been committed | [betterleaks](https://github.com/betterleaks/betterleaks) | brew |
|
|
132
|
+
| Do the dependencies have known vulnerabilities | [osv-scanner](https://github.com/google/osv-scanner), offline against a local copy of the OSV database | brew, plus a one-time database download |
|
|
131
133
|
|
|
132
134
|
Why these and not others: [docs/tools.md](https://github.com/antvinni/gitmole/blob/main/docs/tools.md).
|
|
133
135
|
|
|
@@ -144,7 +146,9 @@ Why these and not others: [docs/tools.md](https://github.com/antvinni/gitmole/bl
|
|
|
144
146
|
## Safety
|
|
145
147
|
|
|
146
148
|
- Everything is offline except the optional clone step, which uses your
|
|
147
|
-
existing gh auth. None of the tools send data anywhere
|
|
149
|
+
existing gh auth. None of the tools send data anywhere; osv-scanner runs
|
|
150
|
+
against a local copy of its database that you download once, and gitmole
|
|
151
|
+
never downloads it for you.
|
|
148
152
|
- Remote targets are cloned into a fresh temp directory that is removed when
|
|
149
153
|
the run ends. Local clones are only read, but the log export and the
|
|
150
154
|
secrets scan touch all branches. `gitmole --clean` lists every directory
|
|
@@ -10,7 +10,7 @@ Any stack. Free. Offline. No token. No AI. Light.
|
|
|
10
10
|
|
|
11
11
|
- **Free.** MIT licence, no paid tier, no account. The tools it runs are open source too.
|
|
12
12
|
- **Any stack.** It reads what every repository has: the git log, git blame and the files themselves.
|
|
13
|
-
- **Offline.** Everything runs against a clone on your machine. Nothing is uploaded, nothing phones home.
|
|
13
|
+
- **Offline.** Everything runs against a clone on your machine. Nothing is uploaded, nothing phones home; the vulnerability database is a copy you download once.
|
|
14
14
|
- **No token.** A local clone needs no credentials. The optional `owner/repo` shortcut uses the `gh` login you already have, and you ask for it.
|
|
15
15
|
- **No AI.** Every finding is a plain rule over counts you can recompute by hand. The same clone gives the same report every time.
|
|
16
16
|
- **Light.** A 4,400-commit repository takes under thirty seconds. A few thousand lines of Python plus two libraries.
|
|
@@ -18,12 +18,12 @@ Any stack. Free. Offline. No token. No AI. Light.
|
|
|
18
18
|
## Install
|
|
19
19
|
|
|
20
20
|
```bash
|
|
21
|
-
# macOS, or Linux with Homebrew: gitmole and the
|
|
21
|
+
# macOS, or Linux with Homebrew: gitmole and the five tools it runs
|
|
22
22
|
brew tap antvinni/gitmole https://github.com/antvinni/gitmole
|
|
23
23
|
brew trust antvinni/gitmole
|
|
24
24
|
brew install gitmole
|
|
25
25
|
|
|
26
|
-
# anywhere else: scc, git-sizer and
|
|
26
|
+
# anywhere else: scc, git-sizer, betterleaks, jscpd and osv-scanner on your PATH, then
|
|
27
27
|
pipx install gitmole
|
|
28
28
|
```
|
|
29
29
|
|
|
@@ -56,31 +56,31 @@ Running `gitmole .` inside this repository:
|
|
|
56
56
|
|
|
57
57
|
```text
|
|
58
58
|
╭─ gitmole ────────────────────────────────────────────────────────────────────────────────────────╮
|
|
59
|
-
│
|
|
60
|
-
│
|
|
59
|
+
│ 185 commits · 2026-09-15 → 2026-09-17 · 1 identity · branch main │
|
|
60
|
+
│ 9,475 lines in 52 files · Python, Ruby │
|
|
61
61
|
│ most commits on Wed at 20:00 · 4% of commits are fixes · 100% of surviving code from 2026 │
|
|
62
62
|
│ 3 warnings, 1 note │
|
|
63
63
|
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
64
64
|
╭─ Findings (4) ───────────────────────────────────────────────────────────────────────────────────╮
|
|
65
65
|
│ ▲ Bus factor of one │
|
|
66
66
|
│ vinni wrote 100% of the code that survives today │
|
|
67
|
-
│ ↳ Pair someone with vinni on gitmole/
|
|
67
|
+
│ ↳ Pair someone with vinni on gitmole/ first; it is 100% theirs. │
|
|
68
68
|
│ ▲ Hotspots getting more complex │
|
|
69
|
-
│ 4 of the 10 top source hotspots grew by 25% or more in a year: gitmole/render.py (+
|
|
70
|
-
│ gitmole/
|
|
71
|
-
│ ↳ Split gitmole/render.py before the next change; its complexity grew
|
|
69
|
+
│ 4 of the 10 top source hotspots grew by 25% or more in a year: gitmole/render.py (+194%), │
|
|
70
|
+
│ gitmole/findings.py (+360%), gitmole/cli.py (+64%), gitmole/run.py (+34%) │
|
|
71
|
+
│ ↳ Split gitmole/render.py before the next change; its complexity grew 194% in a year. │
|
|
72
72
|
│ ▲ Knowledge islands │
|
|
73
|
-
│
|
|
74
|
-
│ 100%)
|
|
75
|
-
│ ↳ Pair someone with vinni on gitmole/ first; it is the largest at
|
|
73
|
+
│ 1 area(s) with at least 200 lines were written almost entirely by one person: gitmole/ (vinni │
|
|
74
|
+
│ 100%). That is 98% of all lines added │
|
|
75
|
+
│ ↳ Pair someone with vinni on gitmole/ first; it is the largest at 7,607 lines. │
|
|
76
76
|
│ ● Bug magnets │
|
|
77
|
-
│
|
|
78
|
-
│ gitmole/
|
|
79
|
-
│ (3 recent, 3 total)
|
|
80
|
-
│ ↳ Review gitmole/
|
|
77
|
+
│ 5 file(s) were fixed 3+ times in the last six months: gitmole/load.py (4 recent, 4 total); │
|
|
78
|
+
│ gitmole/cli.py (3 recent, 3 total); gitmole/findings.py (3 recent, 3 total); gitmole/render.py │
|
|
79
|
+
│ (3 recent, 3 total); gitmole/run.py (3 recent, 3 total) │
|
|
80
|
+
│ ↳ Review gitmole/load.py and gitmole/cli.py before the next release; expect the next bug │
|
|
81
81
|
│ there. │
|
|
82
82
|
│ ✔ No secrets in history │
|
|
83
|
-
│ betterleaks scanned every commit on every branch
|
|
83
|
+
│ betterleaks scanned every commit on every branch; 28 placeholder-shaped hits left out │
|
|
84
84
|
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
|
|
85
85
|
```
|
|
86
86
|
|
|
@@ -106,8 +106,10 @@ you about a clone.
|
|
|
106
106
|
| Where is the risk: hotspots, coupling, ownership | gitmole's own change analysis over `git log --numstat` | built in |
|
|
107
107
|
| How old is the surviving code, per year and author | gitmole's own blame pass (one `git blame` per file at HEAD) | built in |
|
|
108
108
|
| Code-age and survival plots over time | [git-of-theseus](https://github.com/erikbern/git-of-theseus) | pip, opt-in with `--plots` |
|
|
109
|
-
| Per-function complexity, length, parameters
|
|
109
|
+
| Per-function complexity, length, parameters | [lizard](https://github.com/terryyin/lizard) | pip, installed with gitmole; tracked code files only |
|
|
110
|
+
| Which blocks of code appear more than once | [jscpd](https://github.com/kucherenko/jscpd) | brew |
|
|
110
111
|
| Have secrets ever been committed | [betterleaks](https://github.com/betterleaks/betterleaks) | brew |
|
|
112
|
+
| Do the dependencies have known vulnerabilities | [osv-scanner](https://github.com/google/osv-scanner), offline against a local copy of the OSV database | brew, plus a one-time database download |
|
|
111
113
|
|
|
112
114
|
Why these and not others: [docs/tools.md](https://github.com/antvinni/gitmole/blob/main/docs/tools.md).
|
|
113
115
|
|
|
@@ -124,7 +126,9 @@ Why these and not others: [docs/tools.md](https://github.com/antvinni/gitmole/bl
|
|
|
124
126
|
## Safety
|
|
125
127
|
|
|
126
128
|
- Everything is offline except the optional clone step, which uses your
|
|
127
|
-
existing gh auth. None of the tools send data anywhere
|
|
129
|
+
existing gh auth. None of the tools send data anywhere; osv-scanner runs
|
|
130
|
+
against a local copy of its database that you download once, and gitmole
|
|
131
|
+
never downloads it for you.
|
|
128
132
|
- Remote targets are cloned into a fresh temp directory that is removed when
|
|
129
133
|
the run ends. Local clones are only read, but the log export and the
|
|
130
134
|
secrets scan touch all branches. `gitmole --clean` lists every directory
|
|
@@ -37,7 +37,7 @@ def parse_args(argv):
|
|
|
37
37
|
p.add_argument("--list-file-types", action="store_true", help="list the file types in the repository, with counts and whether they count as code, then exit")
|
|
38
38
|
p.add_argument("--clean", action="store_true", help="list the directories gitmole created (temp clones, analysis-* under the target) and delete them after a y/N question, then exit")
|
|
39
39
|
p.add_argument("--yes", action="store_true", help="with --clean: delete without asking")
|
|
40
|
-
p.add_argument("--duplicates", action="store_true", help=
|
|
40
|
+
p.add_argument("--duplicates", action="store_true", help=argparse.SUPPRESS) # duplicates always run now; kept so older scripts still parse
|
|
41
41
|
p.add_argument("--full", action="store_true", help="every column and every row in the terminal report, and the test files the default tables hide (the default is the tighter, readable one)")
|
|
42
42
|
p.add_argument("--json", metavar="PATH", help="write the report and findings as JSON to PATH, or - for stdout")
|
|
43
43
|
p.add_argument("--markdown", metavar="PATH", help="write the report as Markdown to PATH, or - for stdout")
|
|
@@ -100,7 +100,7 @@ def main(argv=None, console: Console = None, tool_check=run.missing_tools, plann
|
|
|
100
100
|
missing = tool_check(plots=args.plots)
|
|
101
101
|
if missing:
|
|
102
102
|
err.print("[red]missing tools:[/red] " + ", ".join(missing))
|
|
103
|
-
err.print("brew install
|
|
103
|
+
err.print(f"brew install {' '.join(run.REQUIRED_TOOLS)}; see README.md for other ways")
|
|
104
104
|
return 2
|
|
105
105
|
args.lizard = lizard_check() # decided once, for every repository this run analyses
|
|
106
106
|
|
|
@@ -269,12 +269,17 @@ def _list_file_types(repo_dir: str, args, console: Console) -> int:
|
|
|
269
269
|
return 0
|
|
270
270
|
|
|
271
271
|
|
|
272
|
-
def _budgets(args, estimate, ui) -> tuple[bool, bool, float]:
|
|
273
|
-
"""Decide whether code age and
|
|
274
|
-
one cut. The projected blame time comes back with them: meta.json records it."""
|
|
272
|
+
def _budgets(args, estimate, ui) -> tuple[bool, bool, float, bool]:
|
|
273
|
+
"""Decide whether code age, plots and the duplicates step fit their time/size budgets, printing a
|
|
274
|
+
skip notice for each one cut. The projected blame time comes back with them: meta.json records it."""
|
|
275
275
|
projected = float(estimate.get("seconds", 0.0))
|
|
276
276
|
age_ok = args.deep or projected <= args.time_budget
|
|
277
277
|
plots_ok = args.plots and (args.deep or estimate["blames"] <= args.budget)
|
|
278
|
+
text_mb = estimate.get("text_bytes", 0) / 1e6
|
|
279
|
+
duplicates_ok = args.deep or text_mb <= run.DUPLICATES_BUDGET_MB
|
|
280
|
+
if not duplicates_ok:
|
|
281
|
+
ui.print(f"[yellow]duplicates skipped:[/yellow] {text_mb:,.0f} MB of tracked text is over the {run.DUPLICATES_BUDGET_MB} MB budget; "
|
|
282
|
+
f"jscpd would need about {text_mb / 25:,.0f} GB of memory. Rerun with --deep to force it, or --ignore-data to shrink it.")
|
|
278
283
|
if not age_ok:
|
|
279
284
|
ui.print(f"[yellow]code age skipped:[/yellow] a blame pass over {estimate.get('code_files', estimate['files']):,} files is projected "
|
|
280
285
|
f"to take about {projected:,.0f}s, over the {args.time_budget:,.0f}s time budget. "
|
|
@@ -283,10 +288,10 @@ def _budgets(args, estimate, ui) -> tuple[bool, bool, float]:
|
|
|
283
288
|
ui.print(f"[yellow]plots skipped:[/yellow] about {estimate['blames']:,} git blames "
|
|
284
289
|
f"({estimate['files']:,} files × {estimate['samples']} samples) exceeds the budget of {args.budget:,}. "
|
|
285
290
|
f"Rerun with --deep to force them, or --ignore-data to shrink them.")
|
|
286
|
-
return age_ok, plots_ok, projected
|
|
291
|
+
return age_ok, plots_ok, projected, duplicates_ok
|
|
287
292
|
|
|
288
293
|
|
|
289
|
-
def _meta_for_run(repo_dir: str, args, estimate, age_ok: bool, plots_ok: bool, projected: float) -> tuple[dict, str | None]:
|
|
294
|
+
def _meta_for_run(repo_dir: str, args, estimate, age_ok: bool, plots_ok: bool, projected: float, duplicates_ok: bool = True) -> tuple[dict, str | None]:
|
|
290
295
|
"""Collect this run's meta.json: repo facts plus a planned status record for every optional step.
|
|
291
296
|
Raises NoCommits when --since leaves no commits to analyse."""
|
|
292
297
|
types_spec = _types_spec(args.file_types)
|
|
@@ -308,6 +313,8 @@ def _meta_for_run(repo_dir: str, args, estimate, age_ok: bool, plots_ok: bool, p
|
|
|
308
313
|
meta["plots"] = {"status": "run" if plots_ok else "skipped", "blames": estimate["blames"], "samples": estimate["samples"], "budget": args.budget}
|
|
309
314
|
lizard_ok = args.lizard
|
|
310
315
|
meta["functions"] = {"status": "planned" if lizard_ok else "skipped"} # "run" only once the step has finished
|
|
316
|
+
meta["duplicates"] = {"status": "planned" if duplicates_ok else "skipped", "text_mb": round(estimate.get("text_bytes", 0) / 1e6, 1),
|
|
317
|
+
"budget_mb": run.DUPLICATES_BUDGET_MB}
|
|
311
318
|
meta["trend"] = {"status": "planned"}
|
|
312
319
|
from . import maat as _maat
|
|
313
320
|
cut = _maat.months_before(meta["last_date"], 6) if meta["last_date"] else None
|
|
@@ -320,7 +327,7 @@ def _meta_for_run(repo_dir: str, args, estimate, age_ok: bool, plots_ok: bool, p
|
|
|
320
327
|
return meta, cut
|
|
321
328
|
|
|
322
329
|
|
|
323
|
-
def _record_statuses(meta, results, age_ok: bool, plots_ok: bool, lizard_ok: bool, cut) -> None:
|
|
330
|
+
def _record_statuses(meta, results, age_ok: bool, plots_ok: bool, lizard_ok: bool, cut, duplicates_ok: bool = True) -> None:
|
|
324
331
|
"""Turn each planned step's exit code into its final status: run, timeout or failed."""
|
|
325
332
|
def status(step, default="run"):
|
|
326
333
|
rc = results.get(step, 0)
|
|
@@ -331,6 +338,8 @@ def _record_statuses(meta, results, age_ok: bool, plots_ok: bool, lizard_ok: boo
|
|
|
331
338
|
meta["plots"]["status"] = status("git-of-theseus")
|
|
332
339
|
if lizard_ok:
|
|
333
340
|
meta["functions"]["status"] = status("functions")
|
|
341
|
+
if duplicates_ok and "duplicates" in results:
|
|
342
|
+
meta["duplicates"]["status"] = status("duplicates")
|
|
334
343
|
if "trend" in results:
|
|
335
344
|
meta["trend"]["status"] = status("trend")
|
|
336
345
|
if cut and "backtest" in results:
|
|
@@ -345,14 +354,14 @@ def _analyse(repo_dir: str, out_dir: str, args, ui: Console, planner, estimator)
|
|
|
345
354
|
|
|
346
355
|
ignore = list(run.DATA_IGNORES if args.ignore_data else []) + list(args.ignore)
|
|
347
356
|
estimate = estimator(repo_dir, run.MONTH, ignore=ignore, types=filetypes.parse(args.file_types))
|
|
348
|
-
age_ok, plots_ok, projected = _budgets(args, estimate, ui)
|
|
357
|
+
age_ok, plots_ok, projected, duplicates_ok = _budgets(args, estimate, ui)
|
|
349
358
|
|
|
350
|
-
meta, cut = _meta_for_run(repo_dir, args, estimate, age_ok, plots_ok, projected)
|
|
359
|
+
meta, cut = _meta_for_run(repo_dir, args, estimate, age_ok, plots_ok, projected, duplicates_ok)
|
|
351
360
|
types_spec = meta["file_types"]
|
|
352
361
|
lizard_ok = args.lizard
|
|
353
362
|
run.clear_outputs(out_dir)
|
|
354
363
|
steps = planner(repo_dir, out_dir, branch=meta["branch"], age=age_ok, plots=plots_ok, ignore=ignore, types=types_spec, now=args.now,
|
|
355
|
-
since=args.since_date, lizard=lizard_ok, duplicates=
|
|
364
|
+
since=args.since_date, lizard=lizard_ok, duplicates=duplicates_ok, backtest=cut)
|
|
356
365
|
run.save_meta(meta, out_dir)
|
|
357
366
|
results = _execute(steps, log_path, repo_dir, args.workers, ui, timeout=args.timeout)
|
|
358
367
|
if _control.cancelled.is_set():
|
|
@@ -360,7 +369,7 @@ def _analyse(repo_dir: str, out_dir: str, args, ui: Console, planner, estimator)
|
|
|
360
369
|
ui.print(f"[red]interrupted:[/red] killed {len(killed)} step(s)")
|
|
361
370
|
raise Interrupted()
|
|
362
371
|
|
|
363
|
-
_record_statuses(meta, results, age_ok, plots_ok, lizard_ok, cut)
|
|
372
|
+
_record_statuses(meta, results, age_ok, plots_ok, lizard_ok, cut, duplicates_ok)
|
|
364
373
|
run.save_meta(meta, out_dir)
|
|
365
374
|
|
|
366
375
|
failed = [n for n, rc in results.items() if rc != 0]
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Known vulnerabilities in the dependencies, from osv-scanner over the lock files, offline.
|
|
3
|
+
|
|
4
|
+
gitmole runs this as the osv-scanner step: `python3 deps.py OUT_JSON`, from inside the repository.
|
|
5
|
+
osv-scanner reads every lock file it knows (package-lock.json, yarn.lock, uv.lock, poetry.lock, go.sum,
|
|
6
|
+
Cargo.lock, Gemfile.lock and the rest) and matches the packages against a copy of the OSV database on
|
|
7
|
+
this machine; with --offline nothing leaves the machine, and the copy is downloaded once by the user, never
|
|
8
|
+
by gitmole. Its report repeats every advisory in full; this wrapper keeps one row per vulnerable package
|
|
9
|
+
(ids, the CVE aliases, the worst score, the version that fixes it) and the lock files with their package
|
|
10
|
+
counts, and writes only that. Standalone, like leaks.py and duplicates.py.
|
|
11
|
+
|
|
12
|
+
The same module reads the status back for the report footer.
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import datetime as dt
|
|
17
|
+
import json
|
|
18
|
+
import os
|
|
19
|
+
import re
|
|
20
|
+
import subprocess
|
|
21
|
+
import sys
|
|
22
|
+
import tempfile
|
|
23
|
+
|
|
24
|
+
ARGV = ["osv-scanner", "scan", "source", "-r", "--offline", "--format", "json", "--all-packages", "."]
|
|
25
|
+
NO_SOURCES = 128 # osv-scanner: no lock file found
|
|
26
|
+
NO_DATABASE = "no offline version of the OSV database" # its message when the local copy is missing
|
|
27
|
+
DOWNLOAD = "osv-scanner scan source -r --offline-vulnerabilities --download-offline-databases ."
|
|
28
|
+
CACHE_DIRS = ("osv-scalibr", "osv-scanner") # where the local copy lives under the cache directory, by version
|
|
29
|
+
|
|
30
|
+
_NUMBER = re.compile(r"\d+")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def cache_dir() -> str:
|
|
34
|
+
"""The directory osv-scanner keeps its local database in: its own variable, else the platform cache."""
|
|
35
|
+
explicit = os.environ.get("OSV_SCANNER_LOCAL_DB_CACHE_DIRECTORY")
|
|
36
|
+
if explicit:
|
|
37
|
+
return explicit
|
|
38
|
+
if sys.platform == "darwin":
|
|
39
|
+
return os.path.expanduser("~/Library/Caches")
|
|
40
|
+
if sys.platform.startswith("win"):
|
|
41
|
+
return os.environ.get("LOCALAPPDATA") or os.path.expanduser("~")
|
|
42
|
+
return os.environ.get("XDG_CACHE_HOME") or os.path.expanduser("~/.cache")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def database_date(base: str = None) -> str | None:
|
|
46
|
+
"""The day the local database was last refreshed (the newest file under it), or None when none."""
|
|
47
|
+
base = cache_dir() if base is None else base
|
|
48
|
+
newest = None
|
|
49
|
+
for name in CACHE_DIRS:
|
|
50
|
+
root = os.path.join(base, name)
|
|
51
|
+
for dirpath, _, files in os.walk(root):
|
|
52
|
+
for f in files:
|
|
53
|
+
try:
|
|
54
|
+
mtime = os.path.getmtime(os.path.join(dirpath, f))
|
|
55
|
+
except OSError:
|
|
56
|
+
continue
|
|
57
|
+
newest = mtime if newest is None else max(newest, mtime)
|
|
58
|
+
return dt.datetime.fromtimestamp(newest).date().isoformat() if newest else None
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _key(version: str) -> tuple:
|
|
62
|
+
return tuple(int(n) for n in _NUMBER.findall(version or ""))
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def fixed_version(vulns: list, name: str, version: str) -> str | None:
|
|
66
|
+
"""The smallest fixed version above the installed one across the advisories' ranges, else the
|
|
67
|
+
highest named; None when no advisory names a fix."""
|
|
68
|
+
fixes = set()
|
|
69
|
+
for v in vulns:
|
|
70
|
+
for a in v.get("affected") or []:
|
|
71
|
+
if (a.get("package") or {}).get("name", "").lower() != (name or "").lower():
|
|
72
|
+
continue
|
|
73
|
+
for r in a.get("ranges") or []:
|
|
74
|
+
for e in r.get("events") or []:
|
|
75
|
+
if e.get("fixed"):
|
|
76
|
+
fixes.add(e["fixed"])
|
|
77
|
+
if not fixes:
|
|
78
|
+
return None
|
|
79
|
+
current = _key(version)
|
|
80
|
+
above = [f for f in fixes if _key(f) > current]
|
|
81
|
+
return min(above, key=_key) if above else max(fixes, key=_key)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _score(groups: list, vulns: list) -> float | None:
|
|
85
|
+
scores = []
|
|
86
|
+
for g in groups:
|
|
87
|
+
try:
|
|
88
|
+
scores.append(float(g.get("max_severity") or ""))
|
|
89
|
+
except ValueError:
|
|
90
|
+
pass
|
|
91
|
+
if scores:
|
|
92
|
+
return max(scores)
|
|
93
|
+
return None
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
_WORDS = {"CRITICAL": 9.5, "HIGH": 8.0, "MODERATE": 5.5, "MEDIUM": 5.5, "LOW": 2.0}
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _label(score, vulns: list) -> str:
|
|
100
|
+
"""critical / high / medium / low from the CVSS score, or from the advisory's own word when it has
|
|
101
|
+
no score, else unknown."""
|
|
102
|
+
if score is None:
|
|
103
|
+
words = [(v.get("database_specific") or {}).get("severity", "").upper() for v in vulns]
|
|
104
|
+
known = [w for w in words if w in _WORDS]
|
|
105
|
+
score = max(_WORDS[w] for w in known) if known else None
|
|
106
|
+
if score is None:
|
|
107
|
+
return "unknown"
|
|
108
|
+
return "critical" if score >= 9 else "high" if score >= 7 else "medium" if score >= 4 else "low"
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _relative(path: str, cwd: str) -> str:
|
|
112
|
+
if os.path.isabs(path):
|
|
113
|
+
try:
|
|
114
|
+
rel = os.path.relpath(path, cwd)
|
|
115
|
+
except ValueError:
|
|
116
|
+
return path
|
|
117
|
+
return path if rel.startswith("..") else rel
|
|
118
|
+
return path[2:] if path.startswith("./") else path
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def summarise(data: dict, cwd: str) -> dict:
|
|
122
|
+
sources, vulnerable, packages = [], [], 0
|
|
123
|
+
for r in data.get("results") or []:
|
|
124
|
+
path = _relative((r.get("source") or {}).get("path") or "", cwd)
|
|
125
|
+
pkgs = r.get("packages") or []
|
|
126
|
+
sources.append({"path": path, "packages": len(pkgs)})
|
|
127
|
+
packages += len(pkgs)
|
|
128
|
+
for p in pkgs:
|
|
129
|
+
vulns = p.get("vulnerabilities") or []
|
|
130
|
+
if not vulns:
|
|
131
|
+
continue
|
|
132
|
+
info = p.get("package") or {}
|
|
133
|
+
groups = p.get("groups") or []
|
|
134
|
+
score = _score(groups, vulns)
|
|
135
|
+
aliases = sorted({a for v in vulns for a in v.get("aliases") or [] if a.startswith("CVE-")})
|
|
136
|
+
vulnerable.append({"name": info.get("name", ""), "version": info.get("version", ""), "ecosystem": info.get("ecosystem", ""),
|
|
137
|
+
"source": path, "ids": [v.get("id", "") for v in vulns], "aliases": aliases,
|
|
138
|
+
"advisories": len(groups) or len(vulns), "score": score, "severity": _label(score, vulns),
|
|
139
|
+
"summary": next((v.get("summary") for v in vulns if v.get("summary")), ""),
|
|
140
|
+
"fixed": fixed_version(vulns, info.get("name", ""), info.get("version", ""))})
|
|
141
|
+
vulnerable.sort(key=lambda r: (-(r["score"] if r["score"] is not None else -1), r["name"], r["source"]))
|
|
142
|
+
return {"status": "scanned", "sources": sources, "packages": packages, "vulnerable": vulnerable}
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def write(result: dict, target: str) -> None:
|
|
146
|
+
fd, tmp = tempfile.mkstemp(dir=os.path.dirname(os.path.abspath(target)), prefix=".dependencies-", suffix=".json")
|
|
147
|
+
try:
|
|
148
|
+
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
|
149
|
+
json.dump(result, fh, indent=1)
|
|
150
|
+
os.replace(tmp, target)
|
|
151
|
+
finally:
|
|
152
|
+
if os.path.exists(tmp):
|
|
153
|
+
os.remove(tmp)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def main(argv=None) -> int:
|
|
157
|
+
args = sys.argv[1:] if argv is None else argv
|
|
158
|
+
if len(args) != 1:
|
|
159
|
+
print("usage: deps.py OUT_JSON", file=sys.stderr)
|
|
160
|
+
return 2
|
|
161
|
+
target = args[0]
|
|
162
|
+
proc = subprocess.run(ARGV, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
163
|
+
err = proc.stderr.decode("utf-8", "replace")
|
|
164
|
+
sys.stderr.write(err) # osv-scanner's own log lands in run.log
|
|
165
|
+
if proc.returncode == NO_SOURCES:
|
|
166
|
+
result = {"status": "no-sources"}
|
|
167
|
+
elif NO_DATABASE in err:
|
|
168
|
+
result = {"status": "no-database", "download": DOWNLOAD}
|
|
169
|
+
elif proc.returncode in (0, 1): # 1: packages with vulnerabilities were found
|
|
170
|
+
text = proc.stdout.decode("utf-8", "replace").strip()
|
|
171
|
+
try:
|
|
172
|
+
data = json.loads(text) if text else {}
|
|
173
|
+
except json.JSONDecodeError:
|
|
174
|
+
print("deps.py: osv-scanner printed no JSON report; nothing written", file=sys.stderr)
|
|
175
|
+
return 1
|
|
176
|
+
result = summarise(data, os.getcwd())
|
|
177
|
+
result["database_date"] = database_date()
|
|
178
|
+
else:
|
|
179
|
+
print(f"deps.py: osv-scanner exited {proc.returncode}; no report written", file=sys.stderr)
|
|
180
|
+
return proc.returncode
|
|
181
|
+
write(result, target)
|
|
182
|
+
return 0
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
if __name__ == "__main__":
|
|
186
|
+
sys.exit(main())
|