gitlab-harvester 1.0.0__tar.gz → 1.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/PKG-INFO +23 -36
- {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/README.md +22 -35
- {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/gitlab_harvester.egg-info/PKG-INFO +23 -36
- {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/gitlab_harvester.egg-info/SOURCES.txt +4 -0
- {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/glh/cli.py +106 -11
- {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/glh/config.py +6 -4
- {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/glh/harvester.py +310 -74
- {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/glh/modes.py +81 -7
- gitlab_harvester-1.2.2/glh/postprocess/split.py +123 -0
- gitlab_harvester-1.2.2/glh/ratelimit.py +194 -0
- gitlab_harvester-1.2.2/glh/tokens.py +117 -0
- gitlab_harvester-1.2.2/glh/user_agents.py +82 -0
- {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/glh/utils/terminal_utils.py +11 -2
- {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/glh/workers.py +104 -2
- {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/pyproject.toml +1 -1
- {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/LICENSE +0 -0
- {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/gitlab_harvester.egg-info/dependency_links.txt +0 -0
- {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/gitlab_harvester.egg-info/entry_points.txt +0 -0
- {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/gitlab_harvester.egg-info/requires.txt +0 -0
- {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/gitlab_harvester.egg-info/top_level.txt +0 -0
- {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/gitlab_harvester.py +0 -0
- {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/glh/__init__.py +0 -0
- {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/glh/exceptions.py +0 -0
- {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/glh/planner.py +0 -0
- {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/glh/postprocess/__init__.py +0 -0
- {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/glh/postprocess/convert.py +0 -0
- {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/glh/postprocess/dedup.py +0 -0
- {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/glh/postprocess/pipeline.py +0 -0
- {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/glh/session.py +0 -0
- {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/glh/utils/__init__.py +0 -0
- {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/glh/utils/logging_utils.py +0 -0
- {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gitlab-harvester
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.2.2
|
|
4
4
|
Summary: Build a GitLab instance project index and search repositories for sensitive keywords (API-only, no cloning).
|
|
5
5
|
Author-email: Cur1iosity <cur1iosity@protonmail.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -53,15 +53,17 @@ Dynamic: license-file
|
|
|
53
53
|
It searches every repository of a GitLab instance for keywords and secrets through the API,
|
|
54
54
|
without cloning, including GitLab CE, which has no instance-wide code search.
|
|
55
55
|
|
|
56
|
-
- **Red team engagements.**
|
|
57
|
-
|
|
58
|
-
|
|
56
|
+
- **Red team engagements.** One token, the whole instance: credentials, internal hosts
|
|
57
|
+
and pivots hiding in the code. Loud and fast or low and slow, your call: rate limits,
|
|
58
|
+
jitter and User-Agent rotation keep the noise and the signature under control. Captured
|
|
59
|
+
several accounts' tokens? Hand them all to one run and beat a per-user rate limit.
|
|
59
60
|
- **Penetration tests.** Instance-wide discovery of credentials, tokens and keys as part
|
|
60
61
|
of the assessment.
|
|
61
|
-
- **Blue teams.** Secret hunting and audits:
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
62
|
+
- **Blue teams and security research.** Secret hunting and hygiene audits: a quick first
|
|
63
|
+
look, full baselines, regular re-scans of recent activity, deduplicated reports, and a
|
|
64
|
+
history of runs in the log file.
|
|
65
|
+
- **Developers, analysts and architects.** Not for secrets, but for code: fast search
|
|
66
|
+
across a large codebase, even on GitLab CE.
|
|
65
67
|
|
|
66
68
|
<p align="center">
|
|
67
69
|
<img src="https://raw.githubusercontent.com/Cur1iosity/GitlabHarvester/demo-assets/demo.gif"
|
|
@@ -78,40 +80,21 @@ Requires Python 3.10+ and a GitLab token with the `read_api` scope.
|
|
|
78
80
|
|
|
79
81
|
## Quick start
|
|
80
82
|
|
|
81
|
-
|
|
82
|
-
export GITLAB_TOKEN=...
|
|
83
|
+
Search one term in the default branch of every project:
|
|
83
84
|
|
|
84
|
-
|
|
85
|
+
```bash
|
|
86
|
+
export GITLAB_TOKEN='your-token-here'
|
|
85
87
|
gitlab-harvester -u https://gitlab.example.com -t "$GITLAB_TOKEN" -s password
|
|
86
|
-
|
|
87
|
-
# Search a list of terms in all branches, skipping branches forks share with their upstream.
|
|
88
|
-
# --dump-projects rebuilds the index so it includes all branch names.
|
|
89
|
-
gitlab-harvester -u https://gitlab.example.com -t "$GITLAB_TOKEN" -f words.txt -b all --forks branch-diff --dump-projects --session audit
|
|
90
|
-
|
|
91
|
-
# Search only repositories with activity on or after a date, e.g. a weekly re-scan.
|
|
92
|
-
# Projects inactive since then are skipped without API calls; in active ones every file is searched.
|
|
93
|
-
gitlab-harvester -u https://gitlab.example.com -t "$GITLAB_TOKEN" -f words.txt -b all --started-from 2024-06-01 --session since-june
|
|
94
|
-
|
|
95
|
-
# Continue after an interruption (reuses the index)
|
|
96
|
-
gitlab-harvester -u https://gitlab.example.com -t "$GITLAB_TOKEN" -f words.txt -b all --forks branch-diff --session audit --resume
|
|
97
|
-
|
|
98
|
-
# Optional: options can also come from a config file (see config.example.ini); CLI arguments override it
|
|
99
|
-
gitlab-harvester -c config.ini -s token
|
|
100
|
-
|
|
101
|
-
# Remove duplicate hits and export as a JSON array
|
|
102
|
-
gitlab-harvester -m dedup --input-file audit.jsonl --output-file clean.jsonl
|
|
103
|
-
gitlab-harvester -m convert --input-file clean.jsonl --output-file clean.json
|
|
104
88
|
```
|
|
105
89
|
|
|
106
|
-
More scenarios, and when to use which options (index-only export, fork strategies,
|
|
107
|
-
fresh findings only, parallel requests, baseline and re-scans):
|
|
108
|
-
**[docs/USE_CASES.md](docs/USE_CASES.md)**.
|
|
109
|
-
|
|
110
90
|
The first run builds a project index (`<host>_project_index.jsonl`), and later runs reuse it.
|
|
111
|
-
Branch options take effect when the index is built; add `--dump-projects` after changing them.
|
|
112
91
|
Hits are written to a JSONL session file as they are found.
|
|
113
92
|
|
|
114
|
-
|
|
93
|
+
Lists of terms, all branches and forks, fresh findings only, resuming, parallel requests
|
|
94
|
+
and rate limits, deduplicated reports, config files:
|
|
95
|
+
**[docs/USE_CASES.md](docs/USE_CASES.md)**.
|
|
96
|
+
|
|
97
|
+
## Common options
|
|
115
98
|
|
|
116
99
|
| Option | Purpose |
|
|
117
100
|
|---|---|
|
|
@@ -119,7 +102,11 @@ Common options:
|
|
|
119
102
|
| `--forks include\|skip\|branch-diff\|all-branches` | how forks are handled |
|
|
120
103
|
| `--time-after DATE`, `--time-before DATE` | limit the index, its projects and the files with hits to a time window |
|
|
121
104
|
| `--started-from 2024-06-01` | search only indexed projects with activity since the date |
|
|
122
|
-
| `-
|
|
105
|
+
| `-t "tok1;tok2"`, `--token-file FILE` | several tokens run in parallel, each with its own workers and rate-limit budget |
|
|
106
|
+
| `-w, --workers N` | parallel API requests per token, for instances without rate limits |
|
|
107
|
+
| `--rpm N`, `--index-rpm N`, `--search-rpm N` | requests per minute per worker: all, non-search, search |
|
|
108
|
+
| `--jitter 0.01-0.05` | random delay before every request (seconds, or `10ms-50ms`) |
|
|
109
|
+
| `-A STRING`, `--user-agent-file FILE` | fixed User-Agent, or a random one from the file for every request |
|
|
123
110
|
| `-m dump-index`, `--dump-projects` | build or rebuild the project index |
|
|
124
111
|
| `--session NAME`, `--resume` | name a session and continue it later |
|
|
125
112
|
| `--post-process dedup,convert`, `--final-only` | deduplicate and convert the session right after the search |
|
|
@@ -15,15 +15,17 @@
|
|
|
15
15
|
It searches every repository of a GitLab instance for keywords and secrets through the API,
|
|
16
16
|
without cloning, including GitLab CE, which has no instance-wide code search.
|
|
17
17
|
|
|
18
|
-
- **Red team engagements.**
|
|
19
|
-
|
|
20
|
-
|
|
18
|
+
- **Red team engagements.** One token, the whole instance: credentials, internal hosts
|
|
19
|
+
and pivots hiding in the code. Loud and fast or low and slow, your call: rate limits,
|
|
20
|
+
jitter and User-Agent rotation keep the noise and the signature under control. Captured
|
|
21
|
+
several accounts' tokens? Hand them all to one run and beat a per-user rate limit.
|
|
21
22
|
- **Penetration tests.** Instance-wide discovery of credentials, tokens and keys as part
|
|
22
23
|
of the assessment.
|
|
23
|
-
- **Blue teams.** Secret hunting and audits:
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
24
|
+
- **Blue teams and security research.** Secret hunting and hygiene audits: a quick first
|
|
25
|
+
look, full baselines, regular re-scans of recent activity, deduplicated reports, and a
|
|
26
|
+
history of runs in the log file.
|
|
27
|
+
- **Developers, analysts and architects.** Not for secrets, but for code: fast search
|
|
28
|
+
across a large codebase, even on GitLab CE.
|
|
27
29
|
|
|
28
30
|
<p align="center">
|
|
29
31
|
<img src="https://raw.githubusercontent.com/Cur1iosity/GitlabHarvester/demo-assets/demo.gif"
|
|
@@ -40,40 +42,21 @@ Requires Python 3.10+ and a GitLab token with the `read_api` scope.
|
|
|
40
42
|
|
|
41
43
|
## Quick start
|
|
42
44
|
|
|
43
|
-
|
|
44
|
-
export GITLAB_TOKEN=...
|
|
45
|
+
Search one term in the default branch of every project:
|
|
45
46
|
|
|
46
|
-
|
|
47
|
+
```bash
|
|
48
|
+
export GITLAB_TOKEN='your-token-here'
|
|
47
49
|
gitlab-harvester -u https://gitlab.example.com -t "$GITLAB_TOKEN" -s password
|
|
48
|
-
|
|
49
|
-
# Search a list of terms in all branches, skipping branches forks share with their upstream.
|
|
50
|
-
# --dump-projects rebuilds the index so it includes all branch names.
|
|
51
|
-
gitlab-harvester -u https://gitlab.example.com -t "$GITLAB_TOKEN" -f words.txt -b all --forks branch-diff --dump-projects --session audit
|
|
52
|
-
|
|
53
|
-
# Search only repositories with activity on or after a date, e.g. a weekly re-scan.
|
|
54
|
-
# Projects inactive since then are skipped without API calls; in active ones every file is searched.
|
|
55
|
-
gitlab-harvester -u https://gitlab.example.com -t "$GITLAB_TOKEN" -f words.txt -b all --started-from 2024-06-01 --session since-june
|
|
56
|
-
|
|
57
|
-
# Continue after an interruption (reuses the index)
|
|
58
|
-
gitlab-harvester -u https://gitlab.example.com -t "$GITLAB_TOKEN" -f words.txt -b all --forks branch-diff --session audit --resume
|
|
59
|
-
|
|
60
|
-
# Optional: options can also come from a config file (see config.example.ini); CLI arguments override it
|
|
61
|
-
gitlab-harvester -c config.ini -s token
|
|
62
|
-
|
|
63
|
-
# Remove duplicate hits and export as a JSON array
|
|
64
|
-
gitlab-harvester -m dedup --input-file audit.jsonl --output-file clean.jsonl
|
|
65
|
-
gitlab-harvester -m convert --input-file clean.jsonl --output-file clean.json
|
|
66
50
|
```
|
|
67
51
|
|
|
68
|
-
More scenarios, and when to use which options (index-only export, fork strategies,
|
|
69
|
-
fresh findings only, parallel requests, baseline and re-scans):
|
|
70
|
-
**[docs/USE_CASES.md](docs/USE_CASES.md)**.
|
|
71
|
-
|
|
72
52
|
The first run builds a project index (`<host>_project_index.jsonl`), and later runs reuse it.
|
|
73
|
-
Branch options take effect when the index is built; add `--dump-projects` after changing them.
|
|
74
53
|
Hits are written to a JSONL session file as they are found.
|
|
75
54
|
|
|
76
|
-
|
|
55
|
+
Lists of terms, all branches and forks, fresh findings only, resuming, parallel requests
|
|
56
|
+
and rate limits, deduplicated reports, config files:
|
|
57
|
+
**[docs/USE_CASES.md](docs/USE_CASES.md)**.
|
|
58
|
+
|
|
59
|
+
## Common options
|
|
77
60
|
|
|
78
61
|
| Option | Purpose |
|
|
79
62
|
|---|---|
|
|
@@ -81,7 +64,11 @@ Common options:
|
|
|
81
64
|
| `--forks include\|skip\|branch-diff\|all-branches` | how forks are handled |
|
|
82
65
|
| `--time-after DATE`, `--time-before DATE` | limit the index, its projects and the files with hits to a time window |
|
|
83
66
|
| `--started-from 2024-06-01` | search only indexed projects with activity since the date |
|
|
84
|
-
| `-
|
|
67
|
+
| `-t "tok1;tok2"`, `--token-file FILE` | several tokens run in parallel, each with its own workers and rate-limit budget |
|
|
68
|
+
| `-w, --workers N` | parallel API requests per token, for instances without rate limits |
|
|
69
|
+
| `--rpm N`, `--index-rpm N`, `--search-rpm N` | requests per minute per worker: all, non-search, search |
|
|
70
|
+
| `--jitter 0.01-0.05` | random delay before every request (seconds, or `10ms-50ms`) |
|
|
71
|
+
| `-A STRING`, `--user-agent-file FILE` | fixed User-Agent, or a random one from the file for every request |
|
|
85
72
|
| `-m dump-index`, `--dump-projects` | build or rebuild the project index |
|
|
86
73
|
| `--session NAME`, `--resume` | name a session and continue it later |
|
|
87
74
|
| `--post-process dedup,convert`, `--final-only` | deduplicate and convert the session right after the search |
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gitlab-harvester
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.2.2
|
|
4
4
|
Summary: Build a GitLab instance project index and search repositories for sensitive keywords (API-only, no cloning).
|
|
5
5
|
Author-email: Cur1iosity <cur1iosity@protonmail.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -53,15 +53,17 @@ Dynamic: license-file
|
|
|
53
53
|
It searches every repository of a GitLab instance for keywords and secrets through the API,
|
|
54
54
|
without cloning, including GitLab CE, which has no instance-wide code search.
|
|
55
55
|
|
|
56
|
-
- **Red team engagements.**
|
|
57
|
-
|
|
58
|
-
|
|
56
|
+
- **Red team engagements.** One token, the whole instance: credentials, internal hosts
|
|
57
|
+
and pivots hiding in the code. Loud and fast or low and slow, your call: rate limits,
|
|
58
|
+
jitter and User-Agent rotation keep the noise and the signature under control. Captured
|
|
59
|
+
several accounts' tokens? Hand them all to one run and beat a per-user rate limit.
|
|
59
60
|
- **Penetration tests.** Instance-wide discovery of credentials, tokens and keys as part
|
|
60
61
|
of the assessment.
|
|
61
|
-
- **Blue teams.** Secret hunting and audits:
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
62
|
+
- **Blue teams and security research.** Secret hunting and hygiene audits: a quick first
|
|
63
|
+
look, full baselines, regular re-scans of recent activity, deduplicated reports, and a
|
|
64
|
+
history of runs in the log file.
|
|
65
|
+
- **Developers, analysts and architects.** Not for secrets, but for code: fast search
|
|
66
|
+
across a large codebase, even on GitLab CE.
|
|
65
67
|
|
|
66
68
|
<p align="center">
|
|
67
69
|
<img src="https://raw.githubusercontent.com/Cur1iosity/GitlabHarvester/demo-assets/demo.gif"
|
|
@@ -78,40 +80,21 @@ Requires Python 3.10+ and a GitLab token with the `read_api` scope.
|
|
|
78
80
|
|
|
79
81
|
## Quick start
|
|
80
82
|
|
|
81
|
-
|
|
82
|
-
export GITLAB_TOKEN=...
|
|
83
|
+
Search one term in the default branch of every project:
|
|
83
84
|
|
|
84
|
-
|
|
85
|
+
```bash
|
|
86
|
+
export GITLAB_TOKEN='your-token-here'
|
|
85
87
|
gitlab-harvester -u https://gitlab.example.com -t "$GITLAB_TOKEN" -s password
|
|
86
|
-
|
|
87
|
-
# Search a list of terms in all branches, skipping branches forks share with their upstream.
|
|
88
|
-
# --dump-projects rebuilds the index so it includes all branch names.
|
|
89
|
-
gitlab-harvester -u https://gitlab.example.com -t "$GITLAB_TOKEN" -f words.txt -b all --forks branch-diff --dump-projects --session audit
|
|
90
|
-
|
|
91
|
-
# Search only repositories with activity on or after a date, e.g. a weekly re-scan.
|
|
92
|
-
# Projects inactive since then are skipped without API calls; in active ones every file is searched.
|
|
93
|
-
gitlab-harvester -u https://gitlab.example.com -t "$GITLAB_TOKEN" -f words.txt -b all --started-from 2024-06-01 --session since-june
|
|
94
|
-
|
|
95
|
-
# Continue after an interruption (reuses the index)
|
|
96
|
-
gitlab-harvester -u https://gitlab.example.com -t "$GITLAB_TOKEN" -f words.txt -b all --forks branch-diff --session audit --resume
|
|
97
|
-
|
|
98
|
-
# Optional: options can also come from a config file (see config.example.ini); CLI arguments override it
|
|
99
|
-
gitlab-harvester -c config.ini -s token
|
|
100
|
-
|
|
101
|
-
# Remove duplicate hits and export as a JSON array
|
|
102
|
-
gitlab-harvester -m dedup --input-file audit.jsonl --output-file clean.jsonl
|
|
103
|
-
gitlab-harvester -m convert --input-file clean.jsonl --output-file clean.json
|
|
104
88
|
```
|
|
105
89
|
|
|
106
|
-
More scenarios, and when to use which options (index-only export, fork strategies,
|
|
107
|
-
fresh findings only, parallel requests, baseline and re-scans):
|
|
108
|
-
**[docs/USE_CASES.md](docs/USE_CASES.md)**.
|
|
109
|
-
|
|
110
90
|
The first run builds a project index (`<host>_project_index.jsonl`), and later runs reuse it.
|
|
111
|
-
Branch options take effect when the index is built; add `--dump-projects` after changing them.
|
|
112
91
|
Hits are written to a JSONL session file as they are found.
|
|
113
92
|
|
|
114
|
-
|
|
93
|
+
Lists of terms, all branches and forks, fresh findings only, resuming, parallel requests
|
|
94
|
+
and rate limits, deduplicated reports, config files:
|
|
95
|
+
**[docs/USE_CASES.md](docs/USE_CASES.md)**.
|
|
96
|
+
|
|
97
|
+
## Common options
|
|
115
98
|
|
|
116
99
|
| Option | Purpose |
|
|
117
100
|
|---|---|
|
|
@@ -119,7 +102,11 @@ Common options:
|
|
|
119
102
|
| `--forks include\|skip\|branch-diff\|all-branches` | how forks are handled |
|
|
120
103
|
| `--time-after DATE`, `--time-before DATE` | limit the index, its projects and the files with hits to a time window |
|
|
121
104
|
| `--started-from 2024-06-01` | search only indexed projects with activity since the date |
|
|
122
|
-
| `-
|
|
105
|
+
| `-t "tok1;tok2"`, `--token-file FILE` | several tokens run in parallel, each with its own workers and rate-limit budget |
|
|
106
|
+
| `-w, --workers N` | parallel API requests per token, for instances without rate limits |
|
|
107
|
+
| `--rpm N`, `--index-rpm N`, `--search-rpm N` | requests per minute per worker: all, non-search, search |
|
|
108
|
+
| `--jitter 0.01-0.05` | random delay before every request (seconds, or `10ms-50ms`) |
|
|
109
|
+
| `-A STRING`, `--user-agent-file FILE` | fixed User-Agent, or a random one from the file for every request |
|
|
123
110
|
| `-m dump-index`, `--dump-projects` | build or rebuild the project index |
|
|
124
111
|
| `--session NAME`, `--resume` | name a session and continue it later |
|
|
125
112
|
| `--post-process dedup,convert`, `--final-only` | deduplicate and convert the session right after the search |
|
|
@@ -15,12 +15,16 @@ glh/exceptions.py
|
|
|
15
15
|
glh/harvester.py
|
|
16
16
|
glh/modes.py
|
|
17
17
|
glh/planner.py
|
|
18
|
+
glh/ratelimit.py
|
|
18
19
|
glh/session.py
|
|
20
|
+
glh/tokens.py
|
|
21
|
+
glh/user_agents.py
|
|
19
22
|
glh/workers.py
|
|
20
23
|
glh/postprocess/__init__.py
|
|
21
24
|
glh/postprocess/convert.py
|
|
22
25
|
glh/postprocess/dedup.py
|
|
23
26
|
glh/postprocess/pipeline.py
|
|
27
|
+
glh/postprocess/split.py
|
|
24
28
|
glh/utils/__init__.py
|
|
25
29
|
glh/utils/logging_utils.py
|
|
26
30
|
glh/utils/terminal_utils.py
|
|
@@ -17,6 +17,7 @@ from dataclasses import dataclass
|
|
|
17
17
|
from datetime import datetime
|
|
18
18
|
from importlib.metadata import version, PackageNotFoundError
|
|
19
19
|
|
|
20
|
+
from glh import ratelimit, user_agents
|
|
20
21
|
from glh.config import TERMS_DEST, ConfigError, load_config
|
|
21
22
|
from glh.planner import parse_timestamp
|
|
22
23
|
from glh.postprocess.pipeline import normalize_steps
|
|
@@ -108,6 +109,37 @@ def parse_workers(value: str) -> int:
|
|
|
108
109
|
raise argparse.ArgumentTypeError(f"Use an integer from 1 to {MAX_WORKERS}.")
|
|
109
110
|
|
|
110
111
|
|
|
112
|
+
def parse_rpm(value: str) -> int:
|
|
113
|
+
"""Parse --rpm/--index-rpm/--search-rpm: a non-negative integer (0 = no limit)."""
|
|
114
|
+
v = value.strip()
|
|
115
|
+
if v.isdigit():
|
|
116
|
+
return int(v)
|
|
117
|
+
raise argparse.ArgumentTypeError("Use a non-negative integer (requests per minute, 0 = no limit).")
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def parse_jitter(value: str) -> tuple[float, float]:
|
|
121
|
+
"""Parse --jitter: MAX or MIN-MAX seconds, optionally with ms/s (0.03, 0.01-0.05, 10ms-50ms)."""
|
|
122
|
+
try:
|
|
123
|
+
return ratelimit.parse_jitter(value)
|
|
124
|
+
except ValueError as e:
|
|
125
|
+
raise argparse.ArgumentTypeError(
|
|
126
|
+
f"{e}. Use MAX or MIN-MAX in seconds, e.g. 0.03, 0.01-0.05 or 10ms-50ms (0 = off).") from None
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def parse_user_agent(value: str) -> str:
|
|
130
|
+
"""Parse -A/--user-agent: a User-Agent that can be sent as an HTTP header."""
|
|
131
|
+
try:
|
|
132
|
+
return user_agents.check_user_agent(value)
|
|
133
|
+
except user_agents.UserAgentError as e:
|
|
134
|
+
raise argparse.ArgumentTypeError(str(e)) from None
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
# (specific dest, group dest) for every request rate limit; resolved like TIME_BOUNDS.
|
|
138
|
+
RATE_LIMITS: tuple[tuple[str, str], ...] = (
|
|
139
|
+
("index_rpm", "rpm"),
|
|
140
|
+
("search_rpm", "rpm"),
|
|
141
|
+
)
|
|
142
|
+
|
|
111
143
|
# (specific dest, group dest) for every time bound; see CliParser.parse().
|
|
112
144
|
TIME_BOUNDS: tuple[tuple[str, str], ...] = (
|
|
113
145
|
("index_active_after", "time_after"),
|
|
@@ -150,11 +182,17 @@ class CliArgs:
|
|
|
150
182
|
config: str | None
|
|
151
183
|
url: str | None
|
|
152
184
|
token: str | None
|
|
185
|
+
token_file: str | None
|
|
153
186
|
proxy: str | None
|
|
154
187
|
timeout: int
|
|
155
188
|
insecure: bool
|
|
156
189
|
ca_bundle: str | None
|
|
157
190
|
workers: int
|
|
191
|
+
index_rpm: int
|
|
192
|
+
search_rpm: int
|
|
193
|
+
jitter: tuple[float, float]
|
|
194
|
+
user_agent: str | None
|
|
195
|
+
user_agent_file: str | None
|
|
158
196
|
|
|
159
197
|
log_file: str | None
|
|
160
198
|
log_level: str
|
|
@@ -183,6 +221,7 @@ class CliArgs:
|
|
|
183
221
|
session: str | None
|
|
184
222
|
session_file: str | None
|
|
185
223
|
resume: bool
|
|
224
|
+
output_separated: bool
|
|
186
225
|
|
|
187
226
|
input_file: str | None
|
|
188
227
|
output_file: str | None
|
|
@@ -226,7 +265,17 @@ class CliParser:
|
|
|
226
265
|
default=None,
|
|
227
266
|
help="GitLab base URL (e.g., https://gitlab.example.com). Alias: -H/--host (deprecated).",
|
|
228
267
|
)
|
|
229
|
-
|
|
268
|
+
tok = parser.add_mutually_exclusive_group()
|
|
269
|
+
tok.add_argument(
|
|
270
|
+
"-t", "--token", default=None,
|
|
271
|
+
help="GitLab token(s), required for search/dump-index. Several tokens separated by ';' run in "
|
|
272
|
+
"parallel, each with its own --workers clients and its own rate-limit budget.",
|
|
273
|
+
)
|
|
274
|
+
tok.add_argument(
|
|
275
|
+
"--token-file", default=None, metavar="FILE",
|
|
276
|
+
help="File with one GitLab token per line (# comments allowed); each token runs in parallel, "
|
|
277
|
+
"like several tokens in -t.",
|
|
278
|
+
)
|
|
230
279
|
parser.add_argument("-p", "--proxy", default=None, help="HTTP(S) proxy URL for GitLab API traffic.",)
|
|
231
280
|
parser.add_argument("--timeout", default=60, type=int, help="Timeout in seconds for GitLab API request.",)
|
|
232
281
|
parser.add_argument(
|
|
@@ -250,6 +299,31 @@ class CliParser:
|
|
|
250
299
|
help=f"Parallel API requests (1-{MAX_WORKERS}, default 1 = sequential). Speeds up project listing, "
|
|
251
300
|
"branch listing and search; use more than 1 only if the instance has no API rate limits.",
|
|
252
301
|
)
|
|
302
|
+
parser.add_argument(
|
|
303
|
+
"--rpm", type=parse_rpm, default=None, metavar="N",
|
|
304
|
+
help="Shortcut for --index-rpm and --search-rpm (a specific option overrides it).",
|
|
305
|
+
)
|
|
306
|
+
parser.add_argument(
|
|
307
|
+
"--index-rpm", type=parse_rpm, default=None, metavar="N",
|
|
308
|
+
help="Max API requests per minute per worker, except search requests: project and branch "
|
|
309
|
+
"listing, file-change checks (default 0 = no limit).",
|
|
310
|
+
)
|
|
311
|
+
parser.add_argument(
|
|
312
|
+
"--search-rpm", type=parse_rpm, default=None, metavar="N",
|
|
313
|
+
help="Max search API requests per minute per worker; GitLab rate-limits search separately "
|
|
314
|
+
"(default 0 = no limit).",
|
|
315
|
+
)
|
|
316
|
+
parser.add_argument(
|
|
317
|
+
"--jitter", type=parse_jitter, default=None, metavar="RANGE",
|
|
318
|
+
help="Random delay before every API request, drawn anew for each one: MAX or MIN-MAX seconds, "
|
|
319
|
+
"e.g. 0.03, 0.01-0.05 or 10ms-50ms (default: off). Adds to the --*-rpm spacing.",
|
|
320
|
+
)
|
|
321
|
+
ua = parser.add_mutually_exclusive_group()
|
|
322
|
+
ua.add_argument("-A", "--user-agent", type=parse_user_agent, default=None, metavar="STRING",
|
|
323
|
+
help="User-Agent of every API request (default: python-gitlab's).")
|
|
324
|
+
ua.add_argument("--user-agent-file", default=None, metavar="FILE",
|
|
325
|
+
help="File with one User-Agent per line (# comments allowed): every API request "
|
|
326
|
+
"gets one chosen at random.")
|
|
253
327
|
|
|
254
328
|
# Logging
|
|
255
329
|
parser.add_argument("--log-file", default=None, help="Write logs to a file (in addition to stderr).")
|
|
@@ -334,6 +408,14 @@ class CliParser:
|
|
|
334
408
|
parser.add_argument("--resume", action=argparse.BooleanOptionalAction, default=False,
|
|
335
409
|
help="Resume from existing session file.")
|
|
336
410
|
|
|
411
|
+
parser.add_argument(
|
|
412
|
+
"--output-separated",
|
|
413
|
+
action=argparse.BooleanOptionalAction,
|
|
414
|
+
default=False,
|
|
415
|
+
help="With several search terms, also write each term's hits to its own file "
|
|
416
|
+
"<session>.<term>.jsonl next to the session (ignored for a single term).",
|
|
417
|
+
)
|
|
418
|
+
|
|
337
419
|
# Postprocess
|
|
338
420
|
parser.add_argument(
|
|
339
421
|
"--post-process",
|
|
@@ -382,15 +464,16 @@ class CliParser:
|
|
|
382
464
|
return set(vars(probe.parse_args(argv)))
|
|
383
465
|
|
|
384
466
|
@staticmethod
|
|
385
|
-
def
|
|
467
|
+
def _resolve_group_options(ns: argparse.Namespace, given: set[str], config: dict) -> None:
|
|
386
468
|
"""
|
|
387
|
-
Resolve every
|
|
469
|
+
Resolve every option that has a group shortcut (time bounds, rate limits)
|
|
470
|
+
in place and drop their keys from ``config``.
|
|
388
471
|
|
|
389
|
-
Order: specific option on the CLI > --time-after
|
|
390
|
-
specific key in the config file >
|
|
472
|
+
Order: specific option on the CLI > group option (--time-after, --rpm, ...)
|
|
473
|
+
on the CLI > specific key in the config file > group key in the config file.
|
|
391
474
|
"""
|
|
392
475
|
resolved: dict[str, object] = {}
|
|
393
|
-
for specific, group in TIME_BOUNDS:
|
|
476
|
+
for specific, group in TIME_BOUNDS + RATE_LIMITS:
|
|
394
477
|
if specific in given:
|
|
395
478
|
resolved[specific] = getattr(ns, specific)
|
|
396
479
|
elif group in given:
|
|
@@ -401,7 +484,7 @@ class CliParser:
|
|
|
401
484
|
resolved[specific] = config[group]
|
|
402
485
|
else:
|
|
403
486
|
resolved[specific] = None
|
|
404
|
-
for dest in {d for pair in TIME_BOUNDS for d in pair}:
|
|
487
|
+
for dest in {d for pair in TIME_BOUNDS + RATE_LIMITS for d in pair}:
|
|
405
488
|
config.pop(dest, None)
|
|
406
489
|
for dest, value in resolved.items():
|
|
407
490
|
setattr(ns, dest, value)
|
|
@@ -427,7 +510,8 @@ class CliParser:
|
|
|
427
510
|
parser.error(str(e))
|
|
428
511
|
given = cls._given_dests(argv)
|
|
429
512
|
|
|
430
|
-
exclusive = (("search", "terms_file", TERMS_DEST), ("session", "session_file")
|
|
513
|
+
exclusive = (("search", "terms_file", TERMS_DEST), ("session", "session_file"),
|
|
514
|
+
("user_agent", "user_agent_file"), ("token", "token_file"))
|
|
431
515
|
for group in exclusive:
|
|
432
516
|
if given & set(group):
|
|
433
517
|
for dest in group:
|
|
@@ -439,14 +523,18 @@ class CliParser:
|
|
|
439
523
|
if dest not in given:
|
|
440
524
|
config.pop(dest, None)
|
|
441
525
|
|
|
442
|
-
cls.
|
|
526
|
+
cls._resolve_group_options(ns, given, config)
|
|
443
527
|
|
|
444
528
|
for dest, value in config.items():
|
|
445
529
|
if dest not in given:
|
|
446
530
|
setattr(ns, dest, value)
|
|
447
531
|
else:
|
|
448
|
-
|
|
449
|
-
|
|
532
|
+
grouped = TIME_BOUNDS + RATE_LIMITS + (("time_after", ""), ("time_before", ""), ("rpm", ""))
|
|
533
|
+
cls._resolve_group_options(ns, {d for d, _ in grouped if getattr(ns, d) is not None}, {})
|
|
534
|
+
|
|
535
|
+
for dest, _ in RATE_LIMITS:
|
|
536
|
+
if getattr(ns, dest) is None:
|
|
537
|
+
setattr(ns, dest, 0)
|
|
450
538
|
|
|
451
539
|
if ns.insecure and ns.ca_bundle:
|
|
452
540
|
parser.error("--insecure and --ca-bundle exclude each other.")
|
|
@@ -473,11 +561,17 @@ class CliParser:
|
|
|
473
561
|
config=ns.config,
|
|
474
562
|
url=ns.url,
|
|
475
563
|
token=ns.token,
|
|
564
|
+
token_file=ns.token_file,
|
|
476
565
|
proxy=ns.proxy,
|
|
477
566
|
timeout=ns.timeout,
|
|
478
567
|
insecure=ns.insecure,
|
|
479
568
|
ca_bundle=ns.ca_bundle,
|
|
480
569
|
workers=ns.workers,
|
|
570
|
+
index_rpm=ns.index_rpm,
|
|
571
|
+
search_rpm=ns.search_rpm,
|
|
572
|
+
jitter=ns.jitter or ratelimit.NO_JITTER,
|
|
573
|
+
user_agent=ns.user_agent,
|
|
574
|
+
user_agent_file=ns.user_agent_file,
|
|
481
575
|
log_file=ns.log_file,
|
|
482
576
|
log_level=ns.log_level,
|
|
483
577
|
debug=ns.debug,
|
|
@@ -502,6 +596,7 @@ class CliParser:
|
|
|
502
596
|
session=ns.session,
|
|
503
597
|
session_file=ns.session_file,
|
|
504
598
|
resume=ns.resume,
|
|
599
|
+
output_separated=ns.output_separated,
|
|
505
600
|
input_file=ns.input_file,
|
|
506
601
|
output_file=ns.output_file,
|
|
507
602
|
post_process=tuple(ns.post_process),
|
|
@@ -16,8 +16,9 @@ Extras that have no CLI counterpart:
|
|
|
16
16
|
- ``token = env:NAME`` reads the token from the environment variable NAME.
|
|
17
17
|
|
|
18
18
|
Relative paths (``index-file``, ``terms-file``, ``session-file``, ``log-file``,
|
|
19
|
-
``input-file``, ``output-file``, ``sqlite-path``, ``ca-bundle``
|
|
20
|
-
directory of the config file, so the
|
|
19
|
+
``input-file``, ``output-file``, ``sqlite-path``, ``ca-bundle``, ``user-agent-file``,
|
|
20
|
+
``token-file``) are resolved against the directory of the config file, so the
|
|
21
|
+
config works from any working directory.
|
|
21
22
|
"""
|
|
22
23
|
|
|
23
24
|
from __future__ import annotations
|
|
@@ -30,7 +31,7 @@ from typing import Any
|
|
|
30
31
|
|
|
31
32
|
# Keys (argparse dests) that hold file system paths.
|
|
32
33
|
PATH_DESTS = frozenset({"index_file", "terms_file", "session_file", "log_file", "input_file", "output_file",
|
|
33
|
-
"sqlite_path", "ca_bundle"})
|
|
34
|
+
"sqlite_path", "ca_bundle", "user_agent_file", "token_file"})
|
|
34
35
|
# Config-only keys (no CLI option).
|
|
35
36
|
TERMS_DEST = "terms"
|
|
36
37
|
# Options that make no sense in a config file.
|
|
@@ -156,7 +157,8 @@ def load_config(path: str | os.PathLike[str], parser: argparse.ArgumentParser) -
|
|
|
156
157
|
values[action.dest] = value
|
|
157
158
|
|
|
158
159
|
# The same exclusivity rules as on the command line.
|
|
159
|
-
for group in (("search", "terms_file", TERMS_DEST), ("session", "session_file")
|
|
160
|
+
for group in (("search", "terms_file", TERMS_DEST), ("session", "session_file"),
|
|
161
|
+
("user_agent", "user_agent_file"), ("token", "token_file")):
|
|
160
162
|
present = [d for d in group if d in values]
|
|
161
163
|
if len(present) > 1:
|
|
162
164
|
names = ", ".join(d.replace("_", "-") for d in present)
|