gitmole 0.17.0__tar.gz → 0.18.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gitmole-0.17.0 → gitmole-0.18.0}/PKG-INFO +13 -6
- {gitmole-0.17.0 → gitmole-0.18.0}/README.md +12 -5
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/__init__.py +1 -1
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/cli.py +1 -1
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/compare.py +10 -1
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/deps.py +24 -1
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/findings.py +14 -1
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/hook.py +2 -1
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/hygiene.py +2 -1
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/leaks.py +3 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/load.py +5 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/maat.py +3 -1
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/render.py +35 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/sarif.py +1 -1
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/signing.py +2 -1
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/watch.py +3 -1
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole.egg-info/PKG-INFO +13 -6
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_compare.py +2 -2
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_deps.py +14 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_findings.py +23 -2
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_golden.py +20 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_leaks.py +1 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_load.py +10 -3
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_render.py +33 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/LICENSE +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/__main__.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/backtest.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/banner.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/blame.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/classify.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/clean.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/coupling.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/duplicates.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/evaluate.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/filetypes.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/functions.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/hotspots.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/identity.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/knowledge.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/loss.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/provenance.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/run.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/structure.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/szz.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/textfmt.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole/trend.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole.egg-info/SOURCES.txt +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole.egg-info/dependency_links.txt +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole.egg-info/entry_points.txt +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole.egg-info/requires.txt +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/gitmole.egg-info/top_level.txt +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/pyproject.toml +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/setup.cfg +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_backtest.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_banner.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_blame.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_classify.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_clean.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_cli.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_coupling.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_duplicates.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_evaluate.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_filetypes.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_functions.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_hook.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_hotspots.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_hygiene.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_identity.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_knowledge.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_loss.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_maat.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_packaging.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_provenance.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_render_examples.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_run.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_sarif.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_signing.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_structure.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_szz.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_textfmt.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_trend.py +0 -0
- {gitmole-0.17.0 → gitmole-0.18.0}/tests/test_watch.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gitmole
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.18.0
|
|
4
4
|
Summary: Offline git repository analysis with a terminal report: hotspots, coupling, ownership, code age, secrets, repo health.
|
|
5
5
|
License: MIT
|
|
6
6
|
Project-URL: Homepage, https://github.com/antvinni/gitmole
|
|
@@ -44,7 +44,7 @@ Free. Any Stack. Local. Offline. Deterministic. Fast.
|
|
|
44
44
|
- **Free.** MIT licence, no paid tier, no account, no token. A local clone needs no credentials, and a public `owner/repo` is cloned with plain git. Your `gh` login is only used for private repositories and for `owner/*`, and only when you ask for them. The tools it runs are open source too.
|
|
45
45
|
- **Any stack.** It reads what every repository has: the git log, git blame and the files themselves.
|
|
46
46
|
- **Local & Offline.** Everything runs against a clone on your machine. Nothing is uploaded, nothing phones home; the vulnerability database is a copy you download once.
|
|
47
|
-
- **Deterministic.** No AI at runtime. Every finding is a plain rule over counts you can recompute by hand. The JSON export carries each finding's rule
|
|
47
|
+
- **Deterministic.** No AI at runtime. Every finding is a plain rule over counts you can recompute by hand. The JSON export carries each finding's rule, the numbers it fired on and, where a rule rests on a paper, the citation. The same commit gives the same bytes: gitmole's own CI runs it twice on every commit, compares the exports and attests the report.
|
|
48
48
|
|
|
49
49
|
## Install
|
|
50
50
|
|
|
@@ -73,12 +73,17 @@ gitmole . --markdown report.md # the same report as a Markdown document
|
|
|
73
73
|
gitmole . --json report.json # every table, the watch list and the findings
|
|
74
74
|
gitmole . --fail-on warning # exit 3 if any finding is a warning or worse
|
|
75
75
|
gitmole . --risk main --risk-threshold 10 # exit 3 if the files changed since main hold over 10% of the risk
|
|
76
|
+
gitmole . --sarif gitmole.sarif # the findings for GitHub code scanning or GitLab
|
|
77
|
+
gitmole . --compare last.json # what changed since an earlier --json export
|
|
78
|
+
gitmole analysis-repo --no-run --hook # a coding agent's edit hook: history's view of the files it just touched
|
|
76
79
|
gitmole . --since 2y --full # the current team, every row and column
|
|
77
80
|
gitmole --clean # list what gitmole left behind, delete on a yes
|
|
78
81
|
```
|
|
79
82
|
|
|
80
83
|
A CI job that runs `gitmole . --fail-on critical --markdown - >> "$GITHUB_STEP_SUMMARY"`
|
|
81
|
-
blocks on secrets in source files and still posts the report.
|
|
84
|
+
blocks on secrets in source files and still posts the report. The same
|
|
85
|
+
scoring wires into Claude Code, Cursor, Gemini CLI and pre-commit as a hook
|
|
86
|
+
that exits 2 over a threshold. Every option:
|
|
82
87
|
[docs/cli.md](https://github.com/antvinni/gitmole/blob/main/docs/cli.md).
|
|
83
88
|
|
|
84
89
|
## What you get
|
|
@@ -87,9 +92,9 @@ Reports on repositories you know, each at a pinned commit, published as gitmole
|
|
|
87
92
|
|
|
88
93
|
| Repository | Commit | Commits | Lines | gitmole run |
|
|
89
94
|
|---|---|---:|---:|---:|
|
|
90
|
-
| [curl](https://github.com/antvinni/gitmole/blob/main/docs/examples/curl.md) | [`540ee5b5`](https://github.com/curl/curl/commit/540ee5b560cc6e775e11317048a13cc7e355bf91) | 39,758 | 247,179 |
|
|
91
|
-
| [django](https://github.com/antvinni/gitmole/blob/main/docs/examples/django.md) | [`8cbdd4a8`](https://github.com/django/django/commit/8cbdd4a814397f81adf0129288f32b615bd1f94f) | 34,933 | 431,749 |
|
|
92
|
-
| [react](https://github.com/antvinni/gitmole/blob/main/docs/examples/react.md) | [`2b19aecd`](https://github.com/facebook/react/commit/2b19aecd0e9111b774fad0fad9862e50bcb5bc8a) | 21,703 | 681,078 |
|
|
95
|
+
| [curl](https://github.com/antvinni/gitmole/blob/main/docs/examples/curl.md) | [`540ee5b5`](https://github.com/curl/curl/commit/540ee5b560cc6e775e11317048a13cc7e355bf91) | 39,758 | 247,179 | 57 s |
|
|
96
|
+
| [django](https://github.com/antvinni/gitmole/blob/main/docs/examples/django.md) | [`8cbdd4a8`](https://github.com/django/django/commit/8cbdd4a814397f81adf0129288f32b615bd1f94f) | 34,933 | 431,749 | 132 s |
|
|
97
|
+
| [react](https://github.com/antvinni/gitmole/blob/main/docs/examples/react.md) | [`2b19aecd`](https://github.com/facebook/react/commit/2b19aecd0e9111b774fad0fad9862e50bcb5bc8a) | 21,703 | 681,078 | 151 s |
|
|
93
98
|
|
|
94
99
|
Run times are one `gitmole CLONE` with every default step, on a MacBook Pro (M4, 16 GB).
|
|
95
100
|
|
|
@@ -110,6 +115,7 @@ you about a clone.
|
|
|
110
115
|
| Which blocks of code appear more than once | [jscpd](https://github.com/kucherenko/jscpd) | brew |
|
|
111
116
|
| Have secrets ever been committed | [betterleaks](https://github.com/betterleaks/betterleaks) | brew |
|
|
112
117
|
| Do the dependencies have known vulnerabilities | [osv-scanner](https://github.com/google/osv-scanner), offline against a local copy of the OSV database | brew, plus a one-time database download |
|
|
118
|
+
| How deeply nested is the code, what did the authors flag, what imports what | [tree-sitter](https://github.com/tree-sitter/py-tree-sitter) grammars for eleven languages | pip, opt-in with `gitmole[structure]` |
|
|
113
119
|
|
|
114
120
|
Why these and not others: [docs/tools.md](https://github.com/antvinni/gitmole/blob/main/docs/tools.md).
|
|
115
121
|
|
|
@@ -121,6 +127,7 @@ Why these and not others: [docs/tools.md](https://github.com/antvinni/gitmole/bl
|
|
|
121
127
|
- [Example reports](https://github.com/antvinni/gitmole/tree/main/docs/examples): curl, django and react at pinned commits, regenerated by `bin/render-examples`.
|
|
122
128
|
- [Validation](https://github.com/antvinni/gitmole/blob/main/docs/validation.md): the watch list against other ways of ranking the same files at six cut-offs on three repositories.
|
|
123
129
|
- [Why these tools](https://github.com/antvinni/gitmole/blob/main/docs/tools.md): the rationale, what was left out, licences.
|
|
130
|
+
- [References](https://github.com/antvinni/gitmole/blob/main/docs/references.md): the research and tools gitmole's rules are built on.
|
|
124
131
|
- [Development](https://github.com/antvinni/gitmole/blob/main/docs/development.md): setup, tests, releases, code layout.
|
|
125
132
|
- [Contributing](https://github.com/antvinni/gitmole/blob/main/CONTRIBUTING.md): bugs, ideas, pull requests, security reports.
|
|
126
133
|
|
|
@@ -11,7 +11,7 @@ Free. Any Stack. Local. Offline. Deterministic. Fast.
|
|
|
11
11
|
- **Free.** MIT licence, no paid tier, no account, no token. A local clone needs no credentials, and a public `owner/repo` is cloned with plain git. Your `gh` login is only used for private repositories and for `owner/*`, and only when you ask for them. The tools it runs are open source too.
|
|
12
12
|
- **Any stack.** It reads what every repository has: the git log, git blame and the files themselves.
|
|
13
13
|
- **Local & Offline.** Everything runs against a clone on your machine. Nothing is uploaded, nothing phones home; the vulnerability database is a copy you download once.
|
|
14
|
-
- **Deterministic.** No AI at runtime. Every finding is a plain rule over counts you can recompute by hand. The JSON export carries each finding's rule
|
|
14
|
+
- **Deterministic.** No AI at runtime. Every finding is a plain rule over counts you can recompute by hand. The JSON export carries each finding's rule, the numbers it fired on and, where a rule rests on a paper, the citation. The same commit gives the same bytes: gitmole's own CI runs it twice on every commit, compares the exports and attests the report.
|
|
15
15
|
|
|
16
16
|
## Install
|
|
17
17
|
|
|
@@ -40,12 +40,17 @@ gitmole . --markdown report.md # the same report as a Markdown document
|
|
|
40
40
|
gitmole . --json report.json # every table, the watch list and the findings
|
|
41
41
|
gitmole . --fail-on warning # exit 3 if any finding is a warning or worse
|
|
42
42
|
gitmole . --risk main --risk-threshold 10 # exit 3 if the files changed since main hold over 10% of the risk
|
|
43
|
+
gitmole . --sarif gitmole.sarif # the findings for GitHub code scanning or GitLab
|
|
44
|
+
gitmole . --compare last.json # what changed since an earlier --json export
|
|
45
|
+
gitmole analysis-repo --no-run --hook # a coding agent's edit hook: history's view of the files it just touched
|
|
43
46
|
gitmole . --since 2y --full # the current team, every row and column
|
|
44
47
|
gitmole --clean # list what gitmole left behind, delete on a yes
|
|
45
48
|
```
|
|
46
49
|
|
|
47
50
|
A CI job that runs `gitmole . --fail-on critical --markdown - >> "$GITHUB_STEP_SUMMARY"`
|
|
48
|
-
blocks on secrets in source files and still posts the report.
|
|
51
|
+
blocks on secrets in source files and still posts the report. The same
|
|
52
|
+
scoring wires into Claude Code, Cursor, Gemini CLI and pre-commit as a hook
|
|
53
|
+
that exits 2 over a threshold. Every option:
|
|
49
54
|
[docs/cli.md](https://github.com/antvinni/gitmole/blob/main/docs/cli.md).
|
|
50
55
|
|
|
51
56
|
## What you get
|
|
@@ -54,9 +59,9 @@ Reports on repositories you know, each at a pinned commit, published as gitmole
|
|
|
54
59
|
|
|
55
60
|
| Repository | Commit | Commits | Lines | gitmole run |
|
|
56
61
|
|---|---|---:|---:|---:|
|
|
57
|
-
| [curl](https://github.com/antvinni/gitmole/blob/main/docs/examples/curl.md) | [`540ee5b5`](https://github.com/curl/curl/commit/540ee5b560cc6e775e11317048a13cc7e355bf91) | 39,758 | 247,179 |
|
|
58
|
-
| [django](https://github.com/antvinni/gitmole/blob/main/docs/examples/django.md) | [`8cbdd4a8`](https://github.com/django/django/commit/8cbdd4a814397f81adf0129288f32b615bd1f94f) | 34,933 | 431,749 |
|
|
59
|
-
| [react](https://github.com/antvinni/gitmole/blob/main/docs/examples/react.md) | [`2b19aecd`](https://github.com/facebook/react/commit/2b19aecd0e9111b774fad0fad9862e50bcb5bc8a) | 21,703 | 681,078 |
|
|
62
|
+
| [curl](https://github.com/antvinni/gitmole/blob/main/docs/examples/curl.md) | [`540ee5b5`](https://github.com/curl/curl/commit/540ee5b560cc6e775e11317048a13cc7e355bf91) | 39,758 | 247,179 | 57 s |
|
|
63
|
+
| [django](https://github.com/antvinni/gitmole/blob/main/docs/examples/django.md) | [`8cbdd4a8`](https://github.com/django/django/commit/8cbdd4a814397f81adf0129288f32b615bd1f94f) | 34,933 | 431,749 | 132 s |
|
|
64
|
+
| [react](https://github.com/antvinni/gitmole/blob/main/docs/examples/react.md) | [`2b19aecd`](https://github.com/facebook/react/commit/2b19aecd0e9111b774fad0fad9862e50bcb5bc8a) | 21,703 | 681,078 | 151 s |
|
|
60
65
|
|
|
61
66
|
Run times are one `gitmole CLONE` with every default step, on a MacBook Pro (M4, 16 GB).
|
|
62
67
|
|
|
@@ -77,6 +82,7 @@ you about a clone.
|
|
|
77
82
|
| Which blocks of code appear more than once | [jscpd](https://github.com/kucherenko/jscpd) | brew |
|
|
78
83
|
| Have secrets ever been committed | [betterleaks](https://github.com/betterleaks/betterleaks) | brew |
|
|
79
84
|
| Do the dependencies have known vulnerabilities | [osv-scanner](https://github.com/google/osv-scanner), offline against a local copy of the OSV database | brew, plus a one-time database download |
|
|
85
|
+
| How deeply nested is the code, what did the authors flag, what imports what | [tree-sitter](https://github.com/tree-sitter/py-tree-sitter) grammars for eleven languages | pip, opt-in with `gitmole[structure]` |
|
|
80
86
|
|
|
81
87
|
Why these and not others: [docs/tools.md](https://github.com/antvinni/gitmole/blob/main/docs/tools.md).
|
|
82
88
|
|
|
@@ -88,6 +94,7 @@ Why these and not others: [docs/tools.md](https://github.com/antvinni/gitmole/bl
|
|
|
88
94
|
- [Example reports](https://github.com/antvinni/gitmole/tree/main/docs/examples): curl, django and react at pinned commits, regenerated by `bin/render-examples`.
|
|
89
95
|
- [Validation](https://github.com/antvinni/gitmole/blob/main/docs/validation.md): the watch list against other ways of ranking the same files at six cut-offs on three repositories.
|
|
90
96
|
- [Why these tools](https://github.com/antvinni/gitmole/blob/main/docs/tools.md): the rationale, what was left out, licences.
|
|
97
|
+
- [References](https://github.com/antvinni/gitmole/blob/main/docs/references.md): the research and tools gitmole's rules are built on.
|
|
91
98
|
- [Development](https://github.com/antvinni/gitmole/blob/main/docs/development.md): setup, tests, releases, code layout.
|
|
92
99
|
- [Contributing](https://github.com/antvinni/gitmole/blob/main/CONTRIBUTING.md): bugs, ideas, pull requests, security reports.
|
|
93
100
|
|
|
@@ -592,7 +592,7 @@ def _render(out_dir: str, console: Console, ui: Console, args, err: Console) ->
|
|
|
592
592
|
return 2
|
|
593
593
|
comparison = _compare.compare(before, report, found)
|
|
594
594
|
if args.json:
|
|
595
|
-
_write(
|
|
595
|
+
_write(render.dumps_json(report, found, risk=risk, compare=comparison), args.json, console)
|
|
596
596
|
if args.markdown:
|
|
597
597
|
_write(render.markdown(report, found, full=args.full, risk=risk, base=args.risk, compare=comparison), args.markdown, console)
|
|
598
598
|
if args.sarif:
|
|
@@ -65,4 +65,13 @@ def compare(before: dict, report: dict, found: list, top: int = watch.WATCH_TOP)
|
|
|
65
65
|
"watch_entered": [f for f in after_watch if f not in before_watch], "watch_left": [f for f in before_watch if f not in after_watch],
|
|
66
66
|
"tally": {"before": _tally(before.get("findings") or []), "after": _tally(found)},
|
|
67
67
|
"before": {"commit": (meta_b.get("run") or {}).get("commit"), "date": meta_b.get("last_date"),
|
|
68
|
-
"options_differ": _options_differ(meta_b, meta_a)}}
|
|
68
|
+
"options_differ": _options_differ(meta_b, meta_a), "database": _database_changed(before, report)}}
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _database_changed(before: dict, report: dict):
|
|
72
|
+
"""{before, after} dates when the two runs scanned against different snapshots of the OSV database,
|
|
73
|
+
else None: a new advisory changes the vulnerable-dependency finding without any change to the code."""
|
|
74
|
+
b, a = before.get("dependencies") or {}, report.get("dependencies") or {}
|
|
75
|
+
if b.get("database_digest") and a.get("database_digest") and b["database_digest"] != a["database_digest"]:
|
|
76
|
+
return {"before": b.get("database_date"), "after": a.get("database_date")}
|
|
77
|
+
return None
|
|
@@ -58,6 +58,28 @@ def database_date(base: str = None) -> str | None:
|
|
|
58
58
|
return dt.datetime.fromtimestamp(newest).date().isoformat() if newest else None
|
|
59
59
|
|
|
60
60
|
|
|
61
|
+
def database_digest(base: str = None) -> str | None:
|
|
62
|
+
"""A digest of the local database snapshot: every file under the cache directories, by path, size
|
|
63
|
+
and modification time. Two runs against the same snapshot agree; a refresh changes it, which is how
|
|
64
|
+
--compare can say a dependency finding moved because the database did. None when there is none."""
|
|
65
|
+
import hashlib
|
|
66
|
+
base = cache_dir() if base is None else base
|
|
67
|
+
entries = []
|
|
68
|
+
for name in CACHE_DIRS:
|
|
69
|
+
root = os.path.join(base, name)
|
|
70
|
+
for dirpath, _, files in os.walk(root):
|
|
71
|
+
for f in files:
|
|
72
|
+
full = os.path.join(dirpath, f)
|
|
73
|
+
try:
|
|
74
|
+
st = os.stat(full)
|
|
75
|
+
except OSError:
|
|
76
|
+
continue
|
|
77
|
+
entries.append(f"{os.path.relpath(full, base)}\0{st.st_size}\0{int(st.st_mtime)}")
|
|
78
|
+
if not entries:
|
|
79
|
+
return None
|
|
80
|
+
return hashlib.sha256("\n".join(sorted(entries)).encode("utf-8", "surrogateescape")).hexdigest()[:16]
|
|
81
|
+
|
|
82
|
+
|
|
61
83
|
def _key(version: str) -> tuple:
|
|
62
84
|
return tuple(int(n) for n in _NUMBER.findall(version or ""))
|
|
63
85
|
|
|
@@ -94,7 +116,7 @@ def _score(groups: list, vulns: list) -> float | None:
|
|
|
94
116
|
|
|
95
117
|
|
|
96
118
|
_WORDS = {"CRITICAL": 9.5, "HIGH": 8.0, "MODERATE": 5.5, "MEDIUM": 5.5, "LOW": 2.0}
|
|
97
|
-
MALICIOUS_PREFIX = "MAL-" # OpenSSF malicious-packages records, in the same OSV database; they carry no CVSS
|
|
119
|
+
MALICIOUS_PREFIX = "MAL-" # OpenSSF malicious-packages records (ossf/malicious-packages), in the same OSV database; they carry no CVSS
|
|
98
120
|
|
|
99
121
|
|
|
100
122
|
def is_malicious(vulns: list) -> bool:
|
|
@@ -185,6 +207,7 @@ def main(argv=None) -> int:
|
|
|
185
207
|
return 1
|
|
186
208
|
result = summarise(data, os.getcwd())
|
|
187
209
|
result["database_date"] = database_date()
|
|
210
|
+
result["database_digest"] = database_digest()
|
|
188
211
|
else:
|
|
189
212
|
print(f"deps.py: osv-scanner exited {proc.returncode}; no report written", file=sys.stderr)
|
|
190
213
|
return proc.returncode
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
"""Heuristics that turn a loaded report into a short list of flagged findings.
|
|
1
|
+
"""Heuristics that turn a loaded report into a short list of flagged findings. A rule that rests on a paper
|
|
2
|
+
carries the short citation in its rule dict's `ref` (see REFS); docs/references.md has the full entries."""
|
|
2
3
|
from __future__ import annotations
|
|
3
4
|
|
|
4
5
|
import re
|
|
@@ -11,11 +12,23 @@ PLACEHOLDER_NAMES = {"your name", "unknown", "root", "user"}
|
|
|
11
12
|
PLACEHOLDER_EMAIL = re.compile(r"(@example\.(com|org|net)$|^you@|^user@|^root@|@localhost$)")
|
|
12
13
|
|
|
13
14
|
|
|
15
|
+
# The paper a rule rests on, as the short citation the rule dict carries in `ref`; the full entries are in
|
|
16
|
+
# docs/references.md. A rule that is gitmole's own heuristic has none.
|
|
17
|
+
REFS = {"minor_contributors": "Bird et al., FSE 2011", "tangled_commits": "Herzig and Zeller, MSR 2013",
|
|
18
|
+
"brain_methods": "Lanza and Marinescu, 2006", "tight_coupling": "Gall, Hajek and Jazayeri, ICSM 1998",
|
|
19
|
+
"hotspot_dominance": "Tornhill, Your Code as a Crime Scene, 2024", "trojan_source": "Boucher and Anderson, USENIX Security 2023",
|
|
20
|
+
"debt_in_hotspots": "Maldonado and Shihab, MTD 2015", "hidden_coupling": "Ajienka and Capiluppi, JSS 2017",
|
|
21
|
+
"unreferenced_files": "Romano et al., TSE 2020", "signoff_by_co_author": "Linux kernel, Documentation/process/coding-assistants.rst",
|
|
22
|
+
"deep_nesting": "SonarSource cognitive complexity; CodeScene code health"}
|
|
23
|
+
|
|
24
|
+
|
|
14
25
|
def _f(severity: str, title: str, statement: str, advice: str, rule: dict, evidence: dict) -> dict:
|
|
15
26
|
"""A finding: the facts, then the next step. `detail` is the two joined for anyone reading the
|
|
16
27
|
JSON; `advice` says which part is the step so the report can show it on its own line. `rule` is
|
|
17
28
|
the rule's id and the thresholds it fired on, `evidence` the numbers they were compared with:
|
|
18
29
|
between them a reader of the JSON can check the finding without reading this file."""
|
|
30
|
+
if rule.get("id") in REFS and "ref" not in rule:
|
|
31
|
+
rule = {**rule, "ref": REFS[rule["id"]]}
|
|
19
32
|
return {"severity": severity, "title": title, "detail": f"{statement.rstrip()} {advice}", "advice": advice,
|
|
20
33
|
"rule": rule, "evidence": evidence}
|
|
21
34
|
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
"""The agent-hook gate: `gitmole OUT_DIR --no-run --hook [--risk-threshold N] [-- FILE...]
|
|
1
|
+
"""The agent-hook gate: `gitmole OUT_DIR --no-run --hook [--risk-threshold N] [-- FILE...]`, after Codacy's
|
|
2
|
+
argument for deterministic findings before inference and Zimmermann et al.'s co-change recommendations.
|
|
2
3
|
|
|
3
4
|
Deterministic findings should run before inference, so the model reasons over a short list rather
|
|
4
5
|
than rediscovering known issues. A linter says what is wrong in the diff; gitmole says what history
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"""Repository hygiene from the clone alone: the checks OpenSSF Scorecard and the OSPS Baseline make
|
|
2
|
-
through the GitHub API, done with file and git reads
|
|
2
|
+
through the GitHub API, done with file and git reads; Trojan Source after Boucher and Anderson (USENIX
|
|
3
|
+
Security 2023).
|
|
3
4
|
|
|
4
5
|
Runs as a pipeline step, `python -m gitmole.hygiene OUT_DIR`, from inside the repository, and writes
|
|
5
6
|
hygiene.json; findings.py turns it into findings. Each check keys on a convention of the ecosystem (a
|
|
@@ -228,6 +228,9 @@ def scan_unreachable(repo: str, out_dir: str, found: dict) -> list:
|
|
|
228
228
|
sha = os.path.basename(r.get("File") or "")
|
|
229
229
|
r["File"] = f"(unreachable blob {sha[:12]})"
|
|
230
230
|
r["Commit"] = ""
|
|
231
|
+
# betterleaks' own fingerprint names the scratch path the blob was written to; this one names the blob,
|
|
232
|
+
# so it is the same in every run and can go into .betterleaksignore
|
|
233
|
+
r["Fingerprint"] = f"unreachable:{sha}:{r.get('RuleID', '')}:{r.get('StartLine', '')}"
|
|
231
234
|
return rows
|
|
232
235
|
|
|
233
236
|
|
|
@@ -182,6 +182,7 @@ def parse_functions(text: str) -> list:
|
|
|
182
182
|
anonymous = name in ("", "(anonymous)")
|
|
183
183
|
rows.append({"file": _rel(r[6]), "function": textfmt.cut(label if anonymous and label else name, NAME_CAP) or "(anonymous)", "anonymous": anonymous,
|
|
184
184
|
"ccn": _num(r[1]), "nloc": _num(r[0]), "params": _num(r[3]), "start": _num(r[9]), "end": _num(r[10]), "suspect": suspect})
|
|
185
|
+
rows.sort(key=lambda f: (f["file"], f["start"], f["end"], f["function"])) # the function step works in parallel; the order is this one
|
|
185
186
|
return rows
|
|
186
187
|
|
|
187
188
|
|
|
@@ -239,6 +240,8 @@ def parse_secrets(text: str) -> list:
|
|
|
239
240
|
value, placeholder = None, False
|
|
240
241
|
out.append({"rule": r.get("RuleID", ""), "file": r.get("File", ""), "commit": r.get("Commit", "")[:7], "line": r.get("StartLine"),
|
|
241
242
|
"fingerprint": r.get("Fingerprint", ""), "value": value, "placeholder": placeholder})
|
|
243
|
+
# betterleaks scans in parallel and does not promise an order; everything downstream reads the rows in this one
|
|
244
|
+
out.sort(key=lambda r: (r["file"], r["commit"], r["line"] or 0, r["rule"], r["fingerprint"]))
|
|
242
245
|
return out
|
|
243
246
|
|
|
244
247
|
|
|
@@ -251,6 +254,8 @@ def parse_dependencies(data) -> dict:
|
|
|
251
254
|
if data["status"] == "scanned":
|
|
252
255
|
out.update({"sources": data.get("sources") or [], "packages": _num(data.get("packages")),
|
|
253
256
|
"vulnerable": data.get("vulnerable") or [], "database_date": data.get("database_date")})
|
|
257
|
+
if data.get("database_digest"):
|
|
258
|
+
out["database_digest"] = data["database_digest"]
|
|
254
259
|
elif data["status"] == "no-database":
|
|
255
260
|
out["download"] = data.get("download") or ""
|
|
256
261
|
return out
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
|
-
"""Change analysis over a git log export, in the layout code-maat produced
|
|
2
|
+
"""Change analysis over a git log export, in the layout code-maat produced: revisions, coupling and sum of
|
|
3
|
+
coupling after Adam Tornhill's code-maat, minor contributors after Bird et al. (FSE 2011), change entropy
|
|
4
|
+
after Hassan (ICSE 2009), oversized fixes after Herzig and Zeller (MSR 2013); see docs/references.md.
|
|
3
5
|
|
|
4
6
|
Standalone on purpose: gitmole runs it as a pipeline step with
|
|
5
7
|
`python3 maat.py LOG OUT_DIR [--aliases META_JSON]` and it must not need the
|
|
@@ -4,6 +4,8 @@ The default report is the tighter one: the columns you actually read, capped row
|
|
|
4
4
|
paths. `full` restores every column and row (Markdown export is always full)."""
|
|
5
5
|
from __future__ import annotations
|
|
6
6
|
|
|
7
|
+
import json
|
|
8
|
+
|
|
7
9
|
from rich import box
|
|
8
10
|
from rich.columns import Columns
|
|
9
11
|
from rich.console import Console, Group
|
|
@@ -758,6 +760,10 @@ def compare_section(result: dict) -> dict:
|
|
|
758
760
|
lines = []
|
|
759
761
|
if before.get("options_differ"):
|
|
760
762
|
lines.append(f"options differ: {', '.join(before['options_differ'])}; the changes partly reflect them")
|
|
763
|
+
db = before.get("database")
|
|
764
|
+
if db:
|
|
765
|
+
lines.append(f"the vulnerability database changed between the runs ({db.get('before') or '?'} to {db.get('after') or '?'}), "
|
|
766
|
+
"so a dependency finding can move with no change to the code")
|
|
761
767
|
lines.append(f"{against}, {before.get('date') or '?'} · {_tally_words(result['tally']['before'])} → {_tally_words(result['tally']['after'])}")
|
|
762
768
|
columns = [("change", {}), ("what", {"overflow": "fold", "ratio": 3})]
|
|
763
769
|
# an empty section prints heading + note and drops the caption (section_block, _md_section), so when
|
|
@@ -1093,6 +1099,35 @@ def markdown(report: dict, findings: list, full: bool = False, risk: dict = None
|
|
|
1093
1099
|
return "\n".join(out)
|
|
1094
1100
|
|
|
1095
1101
|
|
|
1102
|
+
def _envelope(out: dict) -> dict:
|
|
1103
|
+
"""Move what differs between two runs of the same clone into `envelope`: the blame pass's measured
|
|
1104
|
+
projection, the machine-local output directory, the structure cache's hits. Everything outside
|
|
1105
|
+
it is the same bytes for the same commit and options, which the CI determinism job checks."""
|
|
1106
|
+
import copy
|
|
1107
|
+
out = copy.deepcopy(out)
|
|
1108
|
+
env = {"out_dir": out.pop("out_dir", None)}
|
|
1109
|
+
age = (out.get("meta") or {}).get("age")
|
|
1110
|
+
if isinstance(age, dict) and "projected_seconds" in age:
|
|
1111
|
+
env["projected_seconds"] = age.pop("projected_seconds")
|
|
1112
|
+
struct = out.get("structure")
|
|
1113
|
+
if isinstance(struct, dict) and "cached" in struct:
|
|
1114
|
+
env["structure_cached"] = struct.pop("cached")
|
|
1115
|
+
# the value hashes use a key made for each run; the same value gets the same label within an export
|
|
1116
|
+
labels = {}
|
|
1117
|
+
if isinstance(out.get("secrets"), list):
|
|
1118
|
+
out["secrets"].sort(key=lambda r: (r.get("file") or "", r.get("commit") or "", r.get("line") or 0, r.get("rule") or "", r.get("fingerprint") or ""))
|
|
1119
|
+
for row in out.get("secrets") or []:
|
|
1120
|
+
if row.get("value"):
|
|
1121
|
+
row["value"] = labels.setdefault(row["value"], f"v{len(labels) + 1}")
|
|
1122
|
+
out["envelope"] = env
|
|
1123
|
+
return out
|
|
1124
|
+
|
|
1125
|
+
|
|
1126
|
+
def dumps_json(report: dict, findings: list, risk: dict = None, compare: dict = None) -> str:
|
|
1127
|
+
"""The --json export as text: keys sorted, the run-specific values in `envelope`."""
|
|
1128
|
+
return json.dumps(_envelope(to_json(report, findings, risk=risk, compare=compare)), indent=2, sort_keys=True, ensure_ascii=False) + "\n"
|
|
1129
|
+
|
|
1130
|
+
|
|
1096
1131
|
def to_json(report: dict, findings: list, risk: dict = None, compare: dict = None) -> dict:
|
|
1097
1132
|
out = {**{k: v for k, v in report.items() if k != "backtest"}, "findings": findings, # the sub-report is a report of its own
|
|
1098
1133
|
"watch": [{k: v for k, v in r.items() if k != "function"} | {"function": r["function"]["function"] if r["function"] else None}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""The findings as SARIF 2.1.0, for GitHub code scanning and GitLab.
|
|
1
|
+
"""The findings as SARIF 2.1.0, for GitHub code scanning and GitLab, as their SARIF documentation describes.
|
|
2
2
|
|
|
3
3
|
`--json` already carries a rule id, thresholds and evidence per finding, which is most of a SARIF
|
|
4
4
|
result. What the uploaders need on top: `version "2.1.0"`, `tool.driver` with `rules[]` (id,
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
"""Commit signing coverage, read from the commit objects, with no keyring and no network
|
|
1
|
+
"""Commit signing coverage, read from the commit objects, with no keyring and no network (the measure the
|
|
2
|
+
EASE 2025 study of commit signing on GitHub reports).
|
|
2
3
|
|
|
3
4
|
`%G?` depends on the local keyring: on a fresh clone nearly every signed commit comes back E, cannot
|
|
4
5
|
check. The keyring-independent reading is the `gpgsig` (or `gpgsig-sha256`) header in the commit
|
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
"""The watch list: the source files most likely to be fixed next, and what to look at in each.
|
|
1
|
+
"""The watch list: the source files most likely to be fixed next, and what to look at in each. Hotspots by
|
|
2
|
+
revisions × lines of code, after Tornhill (Your Code as a Crime Scene); the change factors on --risk after
|
|
3
|
+
Kamei et al. (TSE 2013), the coupling gaps after Zimmermann et al. (TSE 2005).
|
|
2
4
|
|
|
3
5
|
Each source file that is still in the tree and changed more than once is ranked by revisions × lines
|
|
4
6
|
of code, the product the Hotspots table uses: measured at six cut-offs on three repositories
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gitmole
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.18.0
|
|
4
4
|
Summary: Offline git repository analysis with a terminal report: hotspots, coupling, ownership, code age, secrets, repo health.
|
|
5
5
|
License: MIT
|
|
6
6
|
Project-URL: Homepage, https://github.com/antvinni/gitmole
|
|
@@ -44,7 +44,7 @@ Free. Any Stack. Local. Offline. Deterministic. Fast.
|
|
|
44
44
|
- **Free.** MIT licence, no paid tier, no account, no token. A local clone needs no credentials, and a public `owner/repo` is cloned with plain git. Your `gh` login is only used for private repositories and for `owner/*`, and only when you ask for them. The tools it runs are open source too.
|
|
45
45
|
- **Any stack.** It reads what every repository has: the git log, git blame and the files themselves.
|
|
46
46
|
- **Local & Offline.** Everything runs against a clone on your machine. Nothing is uploaded, nothing phones home; the vulnerability database is a copy you download once.
|
|
47
|
-
- **Deterministic.** No AI at runtime. Every finding is a plain rule over counts you can recompute by hand. The JSON export carries each finding's rule
|
|
47
|
+
- **Deterministic.** No AI at runtime. Every finding is a plain rule over counts you can recompute by hand. The JSON export carries each finding's rule, the numbers it fired on and, where a rule rests on a paper, the citation. The same commit gives the same bytes: gitmole's own CI runs it twice on every commit, compares the exports and attests the report.
|
|
48
48
|
|
|
49
49
|
## Install
|
|
50
50
|
|
|
@@ -73,12 +73,17 @@ gitmole . --markdown report.md # the same report as a Markdown document
|
|
|
73
73
|
gitmole . --json report.json # every table, the watch list and the findings
|
|
74
74
|
gitmole . --fail-on warning # exit 3 if any finding is a warning or worse
|
|
75
75
|
gitmole . --risk main --risk-threshold 10 # exit 3 if the files changed since main hold over 10% of the risk
|
|
76
|
+
gitmole . --sarif gitmole.sarif # the findings for GitHub code scanning or GitLab
|
|
77
|
+
gitmole . --compare last.json # what changed since an earlier --json export
|
|
78
|
+
gitmole analysis-repo --no-run --hook # a coding agent's edit hook: history's view of the files it just touched
|
|
76
79
|
gitmole . --since 2y --full # the current team, every row and column
|
|
77
80
|
gitmole --clean # list what gitmole left behind, delete on a yes
|
|
78
81
|
```
|
|
79
82
|
|
|
80
83
|
A CI job that runs `gitmole . --fail-on critical --markdown - >> "$GITHUB_STEP_SUMMARY"`
|
|
81
|
-
blocks on secrets in source files and still posts the report.
|
|
84
|
+
blocks on secrets in source files and still posts the report. The same
|
|
85
|
+
scoring wires into Claude Code, Cursor, Gemini CLI and pre-commit as a hook
|
|
86
|
+
that exits 2 over a threshold. Every option:
|
|
82
87
|
[docs/cli.md](https://github.com/antvinni/gitmole/blob/main/docs/cli.md).
|
|
83
88
|
|
|
84
89
|
## What you get
|
|
@@ -87,9 +92,9 @@ Reports on repositories you know, each at a pinned commit, published as gitmole
|
|
|
87
92
|
|
|
88
93
|
| Repository | Commit | Commits | Lines | gitmole run |
|
|
89
94
|
|---|---|---:|---:|---:|
|
|
90
|
-
| [curl](https://github.com/antvinni/gitmole/blob/main/docs/examples/curl.md) | [`540ee5b5`](https://github.com/curl/curl/commit/540ee5b560cc6e775e11317048a13cc7e355bf91) | 39,758 | 247,179 |
|
|
91
|
-
| [django](https://github.com/antvinni/gitmole/blob/main/docs/examples/django.md) | [`8cbdd4a8`](https://github.com/django/django/commit/8cbdd4a814397f81adf0129288f32b615bd1f94f) | 34,933 | 431,749 |
|
|
92
|
-
| [react](https://github.com/antvinni/gitmole/blob/main/docs/examples/react.md) | [`2b19aecd`](https://github.com/facebook/react/commit/2b19aecd0e9111b774fad0fad9862e50bcb5bc8a) | 21,703 | 681,078 |
|
|
95
|
+
| [curl](https://github.com/antvinni/gitmole/blob/main/docs/examples/curl.md) | [`540ee5b5`](https://github.com/curl/curl/commit/540ee5b560cc6e775e11317048a13cc7e355bf91) | 39,758 | 247,179 | 57 s |
|
|
96
|
+
| [django](https://github.com/antvinni/gitmole/blob/main/docs/examples/django.md) | [`8cbdd4a8`](https://github.com/django/django/commit/8cbdd4a814397f81adf0129288f32b615bd1f94f) | 34,933 | 431,749 | 132 s |
|
|
97
|
+
| [react](https://github.com/antvinni/gitmole/blob/main/docs/examples/react.md) | [`2b19aecd`](https://github.com/facebook/react/commit/2b19aecd0e9111b774fad0fad9862e50bcb5bc8a) | 21,703 | 681,078 | 151 s |
|
|
93
98
|
|
|
94
99
|
Run times are one `gitmole CLONE` with every default step, on a MacBook Pro (M4, 16 GB).
|
|
95
100
|
|
|
@@ -110,6 +115,7 @@ you about a clone.
|
|
|
110
115
|
| Which blocks of code appear more than once | [jscpd](https://github.com/kucherenko/jscpd) | brew |
|
|
111
116
|
| Have secrets ever been committed | [betterleaks](https://github.com/betterleaks/betterleaks) | brew |
|
|
112
117
|
| Do the dependencies have known vulnerabilities | [osv-scanner](https://github.com/google/osv-scanner), offline against a local copy of the OSV database | brew, plus a one-time database download |
|
|
118
|
+
| How deeply nested is the code, what did the authors flag, what imports what | [tree-sitter](https://github.com/tree-sitter/py-tree-sitter) grammars for eleven languages | pip, opt-in with `gitmole[structure]` |
|
|
113
119
|
|
|
114
120
|
Why these and not others: [docs/tools.md](https://github.com/antvinni/gitmole/blob/main/docs/tools.md).
|
|
115
121
|
|
|
@@ -121,6 +127,7 @@ Why these and not others: [docs/tools.md](https://github.com/antvinni/gitmole/bl
|
|
|
121
127
|
- [Example reports](https://github.com/antvinni/gitmole/tree/main/docs/examples): curl, django and react at pinned commits, regenerated by `bin/render-examples`.
|
|
122
128
|
- [Validation](https://github.com/antvinni/gitmole/blob/main/docs/validation.md): the watch list against other ways of ranking the same files at six cut-offs on three repositories.
|
|
123
129
|
- [Why these tools](https://github.com/antvinni/gitmole/blob/main/docs/tools.md): the rationale, what was left out, licences.
|
|
130
|
+
- [References](https://github.com/antvinni/gitmole/blob/main/docs/references.md): the research and tools gitmole's rules are built on.
|
|
124
131
|
- [Development](https://github.com/antvinni/gitmole/blob/main/docs/development.md): setup, tests, releases, code layout.
|
|
125
132
|
- [Contributing](https://github.com/antvinni/gitmole/blob/main/CONTRIBUTING.md): bugs, ideas, pull requests, security reports.
|
|
126
133
|
|
|
@@ -36,7 +36,7 @@ class Compare(unittest.TestCase):
|
|
|
36
36
|
[("Repo health", "warning", "warning"), ("Bug magnets", "info", "warning")], "the after copy, with the severity it had")
|
|
37
37
|
self.assertEqual((out["watch_entered"], out["watch_left"]), (["c.py"], ["b.py"]))
|
|
38
38
|
self.assertEqual(out["tally"], {"before": {"critical": 0, "warning": 2, "info": 2}, "after": {"critical": 0, "warning": 2, "info": 1}})
|
|
39
|
-
self.assertEqual(out["before"], {"commit": "540ee5b560cc6e775e11317048a13cc7e355bf91", "date": "2026-09-10", "options_differ": ["ignore_data"]})
|
|
39
|
+
self.assertEqual(out["before"], {"commit": "540ee5b560cc6e775e11317048a13cc7e355bf91", "date": "2026-09-10", "options_differ": ["ignore_data"], "database": None})
|
|
40
40
|
|
|
41
41
|
def test_compare_skips_watch_rows_without_a_file(self):
|
|
42
42
|
self.before["watch"] = [{"file": "a.py"}, {"note": "no file field"}, {"file": "b.py"}]
|
|
@@ -46,7 +46,7 @@ class Compare(unittest.TestCase):
|
|
|
46
46
|
def test_an_export_without_a_manifest_compares_findings_only(self):
|
|
47
47
|
self.before["meta"].pop("run")
|
|
48
48
|
out = compare.compare(self.before, self.after, self.after_findings)
|
|
49
|
-
self.assertEqual(out["before"], {"commit": None, "date": "2026-09-10", "options_differ": []})
|
|
49
|
+
self.assertEqual(out["before"], {"commit": None, "date": "2026-09-10", "options_differ": [], "database": None})
|
|
50
50
|
|
|
51
51
|
def test_is_export(self):
|
|
52
52
|
self.assertTrue(compare.is_export(self.before))
|
|
@@ -101,6 +101,20 @@ class DatabaseDate(unittest.TestCase):
|
|
|
101
101
|
os.utime(path, (1_600_000_000, 1_600_000_000))
|
|
102
102
|
self.assertEqual(deps.database_date(d), "2020-09-13")
|
|
103
103
|
|
|
104
|
+
def test_the_digest_names_the_snapshot_and_changes_when_it_does(self):
|
|
105
|
+
with tempfile.TemporaryDirectory() as d:
|
|
106
|
+
self.assertIsNone(deps.database_digest(d))
|
|
107
|
+
os.makedirs(os.path.join(d, "osv-scalibr", "npm"))
|
|
108
|
+
path = os.path.join(d, "osv-scalibr", "npm", "all.zip")
|
|
109
|
+
with open(path, "w") as fh:
|
|
110
|
+
fh.write("a")
|
|
111
|
+
os.utime(path, (1_600_000_000, 1_600_000_000))
|
|
112
|
+
first = deps.database_digest(d)
|
|
113
|
+
self.assertRegex(first, r"^[0-9a-f]{16}$")
|
|
114
|
+
self.assertEqual(deps.database_digest(d), first)
|
|
115
|
+
os.utime(path, (1_600_000_100, 1_600_000_100))
|
|
116
|
+
self.assertNotEqual(deps.database_digest(d), first, "a refreshed copy is another snapshot")
|
|
117
|
+
|
|
104
118
|
def test_the_explicit_variable_wins_over_the_platform_cache(self):
|
|
105
119
|
from unittest.mock import patch
|
|
106
120
|
with patch.dict(os.environ, {"OSV_SCANNER_LOCAL_DB_CACHE_DIRECTORY": "/x/y"}):
|
|
@@ -271,7 +271,7 @@ class MinorContributors(unittest.TestCase):
|
|
|
271
271
|
self.assertNotIn("src/f10.py", f["detail"], "outside the top ten")
|
|
272
272
|
self.assertEqual(f["advice"], "Have Ann, who wrote most of src/f0.py, review changes to it from anyone else; "
|
|
273
273
|
"Bird et al. found the count of minor contributors the strongest ownership predictor of defects.")
|
|
274
|
-
self.assertEqual(f["rule"], {"id": "minor_contributors", "min_minor": 5, "warn_at": 10, "minor_share": 0.05, "top_n": 10})
|
|
274
|
+
self.assertEqual(f["rule"], {"id": "minor_contributors", "min_minor": 5, "warn_at": 10, "minor_share": 0.05, "top_n": 10, "ref": "Bird et al., FSE 2011"})
|
|
275
275
|
self.assertEqual(f["evidence"]["files"][0], {"file": "src/f0.py", "minor": 12, "authors": 15, "owner": "Ann"})
|
|
276
276
|
|
|
277
277
|
def test_info_below_ten_and_nothing_below_five(self):
|
|
@@ -302,7 +302,8 @@ class TangledCommits(unittest.TestCase):
|
|
|
302
302
|
"t1 (34 files, 9 directories, Fix the parser, add a cache and rename the helpers); t2 (12 files, 4 directories, Add x; fix y). "
|
|
303
303
|
"A fix among them credits every file it touched, so 3 fixes over the repository's 99th percentile of lines changed are already left out of the fix counts.", f["detail"])
|
|
304
304
|
self.assertEqual(f["advice"], "Split a change that does several things before merge; the fix history stays readable and the coupling stays real.")
|
|
305
|
-
self.assertEqual(f["rule"], {"id": "tangled_commits", "min_files": 10, "min_dirs": 4, "min_clauses": 2, "min_share": 0.02, "min_count": 5
|
|
305
|
+
self.assertEqual(f["rule"], {"id": "tangled_commits", "min_files": 10, "min_dirs": 4, "min_clauses": 2, "min_share": 0.02, "min_count": 5,
|
|
306
|
+
"ref": "Herzig and Zeller, MSR 2013"})
|
|
306
307
|
self.assertEqual(f["evidence"]["count"], 12)
|
|
307
308
|
self.assertEqual(f["evidence"]["oversized_fixes"], 3)
|
|
308
309
|
|
|
@@ -1247,3 +1248,23 @@ class AgentSurface(unittest.TestCase):
|
|
|
1247
1248
|
self.assertEqual(self.by_id(self.rep()).get("signoff_by_co_author"), None)
|
|
1248
1249
|
one = self.rep(trailers={"never_author": [], "signoff_by_co_author": [{"name": "Ghost", "email": "ghost@x.com", "commits": 1}]})
|
|
1249
1250
|
self.assertIsNone(self.by_id(one).get("signoff_by_co_author"), "one commit is not a habit")
|
|
1251
|
+
|
|
1252
|
+
|
|
1253
|
+
class References(unittest.TestCase):
|
|
1254
|
+
def test_every_rule_resting_on_a_paper_names_it_where_the_numbers_are(self):
|
|
1255
|
+
expected = {"minor_contributors": "Bird et al., FSE 2011", "tangled_commits": "Herzig and Zeller, MSR 2013",
|
|
1256
|
+
"brain_methods": "Lanza and Marinescu, 2006", "tight_coupling": "Gall, Hajek and Jazayeri, ICSM 1998",
|
|
1257
|
+
"hotspot_dominance": "Tornhill, Your Code as a Crime Scene, 2024", "trojan_source": "Boucher and Anderson, USENIX Security 2023",
|
|
1258
|
+
"debt_in_hotspots": "Maldonado and Shihab, MTD 2015", "hidden_coupling": "Ajienka and Capiluppi, JSS 2017",
|
|
1259
|
+
"unreferenced_files": "Romano et al., TSE 2020"}
|
|
1260
|
+
for rule, ref in expected.items():
|
|
1261
|
+
self.assertEqual(findings.REFS[rule], ref, rule)
|
|
1262
|
+
import os
|
|
1263
|
+
page = open(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "docs", "references.md")).read()
|
|
1264
|
+
for ref in expected.values():
|
|
1265
|
+
surname = ref.split(",")[0].split(" and ")[0].split(" et al.")[0]
|
|
1266
|
+
self.assertIn(surname, page, f"{ref} is on the references page")
|
|
1267
|
+
|
|
1268
|
+
def test_the_ref_reaches_the_rule_dict(self):
|
|
1269
|
+
found = findings.tight_coupling(report(coupling=[{"entity": "a.py", "coupled": "b.py", "degree": 90, "average-revs": 10}]))
|
|
1270
|
+
self.assertEqual(found[0]["rule"]["ref"], "Gall, Hajek and Jazayeri, ICSM 1998")
|
|
@@ -96,5 +96,25 @@ class Golden(unittest.TestCase):
|
|
|
96
96
|
self.fail("report differs from tests/golden/report.txt (UPDATE_GOLDEN=1 to accept):\n" + diff)
|
|
97
97
|
|
|
98
98
|
|
|
99
|
+
def test_two_runs_of_the_same_commit_export_the_same_json_outside_the_envelope(self):
|
|
100
|
+
import json
|
|
101
|
+
with tempfile.TemporaryDirectory() as work:
|
|
102
|
+
repo = os.path.join(work, "demo")
|
|
103
|
+
os.makedirs(repo)
|
|
104
|
+
build_repo(repo)
|
|
105
|
+
exports = []
|
|
106
|
+
for i in range(2):
|
|
107
|
+
out = os.path.join(work, "out") # the same output directory, as a CI job reruns into
|
|
108
|
+
c = Console(file=io.StringIO(), width=100, record=True, force_terminal=False, color_system=None)
|
|
109
|
+
target = os.path.join(work, f"report{i}.json")
|
|
110
|
+
with patch.dict(os.environ, HERMETIC_ENV):
|
|
111
|
+
self.assertEqual(cli.main([repo, "--out", out, "--json", target], console=c), 0)
|
|
112
|
+
with open(target) as fh:
|
|
113
|
+
data = json.load(fh)
|
|
114
|
+
del data["envelope"]
|
|
115
|
+
exports.append(json.dumps(data, sort_keys=True))
|
|
116
|
+
self.assertEqual(exports[0], exports[1], "same commit, same options, same bytes: the claim no inference-based reviewer can make")
|
|
117
|
+
|
|
118
|
+
|
|
99
119
|
if __name__ == "__main__":
|
|
100
120
|
unittest.main()
|
|
@@ -297,6 +297,7 @@ fi
|
|
|
297
297
|
[row] = rows
|
|
298
298
|
self.assertRegex(row["File"], r"^\(unreachable blob [0-9a-f]{12}\)$")
|
|
299
299
|
self.assertEqual(row["Commit"], "")
|
|
300
|
+
self.assertRegex(row["Fingerprint"], r"^unreachable:[0-9a-f]{40}:generic-api-key:1$", "no scratch path, the same in every run")
|
|
300
301
|
self.assertNotIn(FAKE, json.dumps(rows))
|
|
301
302
|
self.assertEqual((record["blobs"], record["scanned"], record["findings"]), (2, 2, 1))
|
|
302
303
|
self.assertEqual(leftovers, ["secrets.json", "unreachable.json"], "the blobs written for the scan are gone")
|
|
@@ -167,8 +167,8 @@ class ParseFunctions(unittest.TestCase):
|
|
|
167
167
|
rows = load.parse_functions('136,47,926,1,270,"@316-585@completions.go","completions.go",""," c * Command",316,585,"Run: func(c *Command) {",""\n'
|
|
168
168
|
'4,2,36,0,4,"(anonymous)@1-4@routes.js","routes.js","(anonymous)","(anonymous)",1,4,"app.post(""/api/x"", async (req, res) => {",""\n'
|
|
169
169
|
'4,2,14,2,4,"tracked@1-4@app.py","app.py","tracked","tracked( a , b )",1,4,"",""\n')
|
|
170
|
-
self.assertEqual(
|
|
171
|
-
[("Run: func(c *Command) {", True), ('app.post("/api/x", async (req, res) => {', True), ("tracked", False)])
|
|
170
|
+
self.assertEqual(sorted((r["function"], r["anonymous"]) for r in rows),
|
|
171
|
+
sorted([("Run: func(c *Command) {", True), ('app.post("/api/x", async (req, res) => {', True), ("tracked", False)]))
|
|
172
172
|
|
|
173
173
|
def test_a_suspect_span_carries_its_reason(self):
|
|
174
174
|
rows = load.parse_functions('9,1,21,1,9,"tpl@1-9@tpl.js","tpl.js","tpl","tpl ( name )",1,9,"","opens a block at line 8 no deeper than its own start"\n')
|
|
@@ -177,7 +177,8 @@ class ParseFunctions(unittest.TestCase):
|
|
|
177
177
|
def test_a_row_cut_short_by_a_killed_step_does_not_abort_the_report(self):
|
|
178
178
|
rows = load.parse_functions(self.CSV + '5,3,40,1,5,"g@1-5@a.py","a.py","g","g( )",1,\n')
|
|
179
179
|
self.assertEqual(len(rows), 3)
|
|
180
|
-
|
|
180
|
+
g = next(r for r in rows if r["function"] == "g")
|
|
181
|
+
self.assertEqual(g["end"], 0)
|
|
181
182
|
|
|
182
183
|
|
|
183
184
|
class ParseDuplicates(unittest.TestCase):
|
|
@@ -372,6 +373,12 @@ class LoadReport(unittest.TestCase):
|
|
|
372
373
|
self.assertEqual(d["then"], {"date": "2025-09-17", "rev": "abc", "files": 9, "rate": 4.2})
|
|
373
374
|
self.assertNotIn("then", load.parse_duplicates_json({"rate": 6.1, "blocks": []}))
|
|
374
375
|
|
|
376
|
+
def test_function_rows_come_back_in_file_and_line_order_whatever_the_step_wrote(self):
|
|
377
|
+
text = ('3,2,20,1,3,"g@9-11@b.py","b.py","g","g( x )",9,11\n'
|
|
378
|
+
'3,2,20,1,3,"f@1-3@b.py","b.py","f","f( x )",1,3\n'
|
|
379
|
+
'3,2,20,1,3,"h@5-7@a.py","a.py","h","h( x )",5,7\n')
|
|
380
|
+
self.assertEqual([(f["file"], f["function"]) for f in load.parse_functions(text)], [("a.py", "h"), ("b.py", "f"), ("b.py", "g")])
|
|
381
|
+
|
|
375
382
|
def test_no_signing_file_is_an_empty_record(self):
|
|
376
383
|
import os, tempfile
|
|
377
384
|
with tempfile.TemporaryDirectory() as out:
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import json
|
|
1
2
|
import io
|
|
2
3
|
import unittest
|
|
3
4
|
|
|
@@ -283,6 +284,38 @@ class Report(unittest.TestCase):
|
|
|
283
284
|
self.assertIn("12% of commits land in bursts of five or more within ten minutes; 80% have conventional-commit subjects; commits come in 20 hours of the day", caption)
|
|
284
285
|
self.assertIn("## Trailers", render.markdown(r, []))
|
|
285
286
|
|
|
287
|
+
def test_the_comparison_says_when_the_vulnerability_database_changed(self):
|
|
288
|
+
result = {"new": [], "resolved": [], "persisting": [], "watch_entered": [], "watch_left": [],
|
|
289
|
+
"tally": {"before": {"critical": 0, "warning": 0, "info": 0}, "after": {"critical": 0, "warning": 0, "info": 0}},
|
|
290
|
+
"before": {"commit": "abc12345", "date": "2026-09-01", "options_differ": [], "database": {"before": "2026-09-01", "after": "2026-09-17"}}}
|
|
291
|
+
sec = render.compare_section(result)
|
|
292
|
+
text = (sec.get("caption") or "") + (sec.get("note") or "")
|
|
293
|
+
self.assertIn("the vulnerability database changed between the runs (2026-09-01 to 2026-09-17), so a dependency finding can move with no change to the code", text)
|
|
294
|
+
|
|
295
|
+
def test_the_json_is_the_same_bytes_for_the_same_clone_whatever_the_run(self):
|
|
296
|
+
import copy
|
|
297
|
+
a = sample_report()
|
|
298
|
+
a["secrets"] = [{"rule": "k", "file": "b.py", "commit": "c2", "line": 3, "fingerprint": "f2", "value": "9f00aa11bb22", "placeholder": False},
|
|
299
|
+
{"rule": "k", "file": "a.py", "commit": "c1", "line": 1, "fingerprint": "f1", "value": "12ab34cd56ef", "placeholder": False},
|
|
300
|
+
{"rule": "k", "file": "c.py", "commit": "c3", "line": 2, "fingerprint": "f3", "value": "12ab34cd56ef", "placeholder": False}]
|
|
301
|
+
a["meta"]["age"] = {"status": "run", "projected_seconds": 3.14159, "files": 10}
|
|
302
|
+
a["structure"] = {"status": "run", "cached": 0, "files": {}}
|
|
303
|
+
b = copy.deepcopy(a)
|
|
304
|
+
for row, other in zip(b["secrets"], ("0000ffff1111", "aaaabbbbcccc", "aaaabbbbcccc")):
|
|
305
|
+
row["value"] = other # another run: another random key, the same grouping
|
|
306
|
+
b["secrets"].reverse() # and betterleaks free to report in another order
|
|
307
|
+
b["meta"]["age"]["projected_seconds"] = 2.71828
|
|
308
|
+
b["structure"]["cached"] = 40
|
|
309
|
+
b["out_dir"] = "/somewhere/else"
|
|
310
|
+
first, second = json.loads(render.dumps_json(a, [])), json.loads(render.dumps_json(b, []))
|
|
311
|
+
self.assertNotEqual(first["envelope"], second["envelope"], "timings, the output path and cache hits vary, and live in the envelope")
|
|
312
|
+
del first["envelope"], second["envelope"]
|
|
313
|
+
self.assertEqual(json.dumps(first, sort_keys=True), json.dumps(second, sort_keys=True))
|
|
314
|
+
self.assertEqual([r["value"] for r in first["secrets"]], ["v1", "v2", "v1"], "the same value, the same label, by the rows' sorted order")
|
|
315
|
+
text = render.dumps_json(a, [])
|
|
316
|
+
self.assertTrue(text.endswith("}\n"))
|
|
317
|
+
self.assertEqual(text, render.dumps_json(a, []))
|
|
318
|
+
|
|
286
319
|
def test_the_secrets_line_says_what_lay_outside_reachable_history(self):
|
|
287
320
|
r = sample_report()
|
|
288
321
|
r["unreachable"] = {"objects": 0, "blobs": 0, "scanned": 0, "findings": 0}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|