gitlab-harvester 1.0.0__tar.gz → 1.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/PKG-INFO +23 -36
  2. {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/README.md +22 -35
  3. {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/gitlab_harvester.egg-info/PKG-INFO +23 -36
  4. {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/gitlab_harvester.egg-info/SOURCES.txt +4 -0
  5. {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/glh/cli.py +106 -11
  6. {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/glh/config.py +6 -4
  7. {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/glh/harvester.py +310 -74
  8. {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/glh/modes.py +81 -7
  9. gitlab_harvester-1.2.2/glh/postprocess/split.py +123 -0
  10. gitlab_harvester-1.2.2/glh/ratelimit.py +194 -0
  11. gitlab_harvester-1.2.2/glh/tokens.py +117 -0
  12. gitlab_harvester-1.2.2/glh/user_agents.py +82 -0
  13. {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/glh/utils/terminal_utils.py +11 -2
  14. {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/glh/workers.py +104 -2
  15. {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/pyproject.toml +1 -1
  16. {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/LICENSE +0 -0
  17. {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/gitlab_harvester.egg-info/dependency_links.txt +0 -0
  18. {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/gitlab_harvester.egg-info/entry_points.txt +0 -0
  19. {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/gitlab_harvester.egg-info/requires.txt +0 -0
  20. {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/gitlab_harvester.egg-info/top_level.txt +0 -0
  21. {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/gitlab_harvester.py +0 -0
  22. {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/glh/__init__.py +0 -0
  23. {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/glh/exceptions.py +0 -0
  24. {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/glh/planner.py +0 -0
  25. {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/glh/postprocess/__init__.py +0 -0
  26. {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/glh/postprocess/convert.py +0 -0
  27. {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/glh/postprocess/dedup.py +0 -0
  28. {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/glh/postprocess/pipeline.py +0 -0
  29. {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/glh/session.py +0 -0
  30. {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/glh/utils/__init__.py +0 -0
  31. {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/glh/utils/logging_utils.py +0 -0
  32. {gitlab_harvester-1.0.0 → gitlab_harvester-1.2.2}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: gitlab-harvester
3
- Version: 1.0.0
3
+ Version: 1.2.2
4
4
  Summary: Build a GitLab instance project index and search repositories for sensitive keywords (API-only, no cloning).
5
5
  Author-email: Cur1iosity <cur1iosity@protonmail.com>
6
6
  License-Expression: MIT
@@ -53,15 +53,17 @@ Dynamic: license-file
53
53
  It searches every repository of a GitLab instance for keywords and secrets through the API,
54
54
  without cloning, including GitLab CE, which has no instance-wide code search.
55
55
 
56
- - **Red team engagements.** A quick PoC: install, point it at an instance, get findings.
57
- Fine-grained settings (branch and fork strategies, time windows, parallel requests)
58
- control how many requests are sent and how much noise they make.
56
+ - **Red team engagements.** One token, the whole instance: credentials, internal hosts
57
+ and pivots hiding in the code. Loud and fast or low and slow, your call: rate limits,
58
+ jitter and User-Agent rotation keep the noise and the signature under control. Captured
59
+ several accounts' tokens? Hand them all to one run and beat a per-user rate limit.
59
60
  - **Penetration tests.** Instance-wide discovery of credentials, tokens and keys as part
60
61
  of the assessment.
61
- - **Blue teams.** Secret hunting and audits: full baselines, regular re-scans of recent
62
- activity, deduplicated reports, and a history of runs in the log file.
63
- - **Developers, analysts and architects.** Surprisingly handy as plain instance-wide code
64
- search: find where an API, a config key or a library is used across all projects.
62
+ - **Blue teams and security research.** Secret hunting and hygiene audits: a quick first
63
+ look, full baselines, regular re-scans of recent activity, deduplicated reports, and a
64
+ history of runs in the log file.
65
+ - **Developers, analysts and architects.** Not for secrets, but for code: fast search
66
+ across a large codebase, even on GitLab CE.
65
67
 
66
68
  <p align="center">
67
69
  <img src="https://raw.githubusercontent.com/Cur1iosity/GitlabHarvester/demo-assets/demo.gif"
@@ -78,40 +80,21 @@ Requires Python 3.10+ and a GitLab token with the `read_api` scope.
78
80
 
79
81
  ## Quick start
80
82
 
81
- ```bash
82
- export GITLAB_TOKEN=...
83
+ Search one term in the default branch of every project:
83
84
 
84
- # Search one term in the default branch of every project
85
+ ```bash
86
+ export GITLAB_TOKEN='your-token-here'
85
87
  gitlab-harvester -u https://gitlab.example.com -t "$GITLAB_TOKEN" -s password
86
-
87
- # Search a list of terms in all branches, skipping branches forks share with their upstream.
88
- # --dump-projects rebuilds the index so it includes all branch names.
89
- gitlab-harvester -u https://gitlab.example.com -t "$GITLAB_TOKEN" -f words.txt -b all --forks branch-diff --dump-projects --session audit
90
-
91
- # Search only repositories with activity on or after a date, e.g. a weekly re-scan.
92
- # Projects inactive since then are skipped without API calls; in active ones every file is searched.
93
- gitlab-harvester -u https://gitlab.example.com -t "$GITLAB_TOKEN" -f words.txt -b all --started-from 2024-06-01 --session since-june
94
-
95
- # Continue after an interruption (reuses the index)
96
- gitlab-harvester -u https://gitlab.example.com -t "$GITLAB_TOKEN" -f words.txt -b all --forks branch-diff --session audit --resume
97
-
98
- # Optional: options can also come from a config file (see config.example.ini); CLI arguments override it
99
- gitlab-harvester -c config.ini -s token
100
-
101
- # Remove duplicate hits and export as a JSON array
102
- gitlab-harvester -m dedup --input-file audit.jsonl --output-file clean.jsonl
103
- gitlab-harvester -m convert --input-file clean.jsonl --output-file clean.json
104
88
  ```
105
89
 
106
- More scenarios, and when to use which options (index-only export, fork strategies,
107
- fresh findings only, parallel requests, baseline and re-scans):
108
- **[docs/USE_CASES.md](docs/USE_CASES.md)**.
109
-
110
90
  The first run builds a project index (`<host>_project_index.jsonl`), and later runs reuse it.
111
- Branch options take effect when the index is built; add `--dump-projects` after changing them.
112
91
  Hits are written to a JSONL session file as they are found.
113
92
 
114
- Common options:
93
+ Lists of terms, all branches and forks, fresh findings only, resuming, parallel requests
94
+ and rate limits, deduplicated reports, config files:
95
+ **[docs/USE_CASES.md](docs/USE_CASES.md)**.
96
+
97
+ ## Common options
115
98
 
116
99
  | Option | Purpose |
117
100
  |---|---|
@@ -119,7 +102,11 @@ Common options:
119
102
  | `--forks include\|skip\|branch-diff\|all-branches` | how forks are handled |
120
103
  | `--time-after DATE`, `--time-before DATE` | limit the index, its projects and the files with hits to a time window |
121
104
  | `--started-from 2024-06-01` | search only indexed projects with activity since the date |
122
- | `-w, --workers N` | parallel API requests, for instances without rate limits |
105
+ | `-t "tok1;tok2"`, `--token-file FILE` | several tokens run in parallel, each with its own workers and rate-limit budget |
106
+ | `-w, --workers N` | parallel API requests per token, for instances without rate limits |
107
+ | `--rpm N`, `--index-rpm N`, `--search-rpm N` | requests per minute per worker: all, non-search, search |
108
+ | `--jitter 0.01-0.05` | random delay before every request (seconds, or `10ms-50ms`) |
109
+ | `-A STRING`, `--user-agent-file FILE` | fixed User-Agent, or a random one from the file for every request |
123
110
  | `-m dump-index`, `--dump-projects` | build or rebuild the project index |
124
111
  | `--session NAME`, `--resume` | name a session and continue it later |
125
112
  | `--post-process dedup,convert`, `--final-only` | deduplicate and convert the session right after the search |
@@ -15,15 +15,17 @@
15
15
  It searches every repository of a GitLab instance for keywords and secrets through the API,
16
16
  without cloning, including GitLab CE, which has no instance-wide code search.
17
17
 
18
- - **Red team engagements.** A quick PoC: install, point it at an instance, get findings.
19
- Fine-grained settings (branch and fork strategies, time windows, parallel requests)
20
- control how many requests are sent and how much noise they make.
18
+ - **Red team engagements.** One token, the whole instance: credentials, internal hosts
19
+ and pivots hiding in the code. Loud and fast or low and slow, your call: rate limits,
20
+ jitter and User-Agent rotation keep the noise and the signature under control. Captured
21
+ several accounts' tokens? Hand them all to one run and beat a per-user rate limit.
21
22
  - **Penetration tests.** Instance-wide discovery of credentials, tokens and keys as part
22
23
  of the assessment.
23
- - **Blue teams.** Secret hunting and audits: full baselines, regular re-scans of recent
24
- activity, deduplicated reports, and a history of runs in the log file.
25
- - **Developers, analysts and architects.** Surprisingly handy as plain instance-wide code
26
- search: find where an API, a config key or a library is used across all projects.
24
+ - **Blue teams and security research.** Secret hunting and hygiene audits: a quick first
25
+ look, full baselines, regular re-scans of recent activity, deduplicated reports, and a
26
+ history of runs in the log file.
27
+ - **Developers, analysts and architects.** Not for secrets, but for code: fast search
28
+ across a large codebase, even on GitLab CE.
27
29
 
28
30
  <p align="center">
29
31
  <img src="https://raw.githubusercontent.com/Cur1iosity/GitlabHarvester/demo-assets/demo.gif"
@@ -40,40 +42,21 @@ Requires Python 3.10+ and a GitLab token with the `read_api` scope.
40
42
 
41
43
  ## Quick start
42
44
 
43
- ```bash
44
- export GITLAB_TOKEN=...
45
+ Search one term in the default branch of every project:
45
46
 
46
- # Search one term in the default branch of every project
47
+ ```bash
48
+ export GITLAB_TOKEN='your-token-here'
47
49
  gitlab-harvester -u https://gitlab.example.com -t "$GITLAB_TOKEN" -s password
48
-
49
- # Search a list of terms in all branches, skipping branches forks share with their upstream.
50
- # --dump-projects rebuilds the index so it includes all branch names.
51
- gitlab-harvester -u https://gitlab.example.com -t "$GITLAB_TOKEN" -f words.txt -b all --forks branch-diff --dump-projects --session audit
52
-
53
- # Search only repositories with activity on or after a date, e.g. a weekly re-scan.
54
- # Projects inactive since then are skipped without API calls; in active ones every file is searched.
55
- gitlab-harvester -u https://gitlab.example.com -t "$GITLAB_TOKEN" -f words.txt -b all --started-from 2024-06-01 --session since-june
56
-
57
- # Continue after an interruption (reuses the index)
58
- gitlab-harvester -u https://gitlab.example.com -t "$GITLAB_TOKEN" -f words.txt -b all --forks branch-diff --session audit --resume
59
-
60
- # Optional: options can also come from a config file (see config.example.ini); CLI arguments override it
61
- gitlab-harvester -c config.ini -s token
62
-
63
- # Remove duplicate hits and export as a JSON array
64
- gitlab-harvester -m dedup --input-file audit.jsonl --output-file clean.jsonl
65
- gitlab-harvester -m convert --input-file clean.jsonl --output-file clean.json
66
50
  ```
67
51
 
68
- More scenarios, and when to use which options (index-only export, fork strategies,
69
- fresh findings only, parallel requests, baseline and re-scans):
70
- **[docs/USE_CASES.md](docs/USE_CASES.md)**.
71
-
72
52
  The first run builds a project index (`<host>_project_index.jsonl`), and later runs reuse it.
73
- Branch options take effect when the index is built; add `--dump-projects` after changing them.
74
53
  Hits are written to a JSONL session file as they are found.
75
54
 
76
- Common options:
55
+ Lists of terms, all branches and forks, fresh findings only, resuming, parallel requests
56
+ and rate limits, deduplicated reports, config files:
57
+ **[docs/USE_CASES.md](docs/USE_CASES.md)**.
58
+
59
+ ## Common options
77
60
 
78
61
  | Option | Purpose |
79
62
  |---|---|
@@ -81,7 +64,11 @@ Common options:
81
64
  | `--forks include\|skip\|branch-diff\|all-branches` | how forks are handled |
82
65
  | `--time-after DATE`, `--time-before DATE` | limit the index, its projects and the files with hits to a time window |
83
66
  | `--started-from 2024-06-01` | search only indexed projects with activity since the date |
84
- | `-w, --workers N` | parallel API requests, for instances without rate limits |
67
+ | `-t "tok1;tok2"`, `--token-file FILE` | several tokens run in parallel, each with its own workers and rate-limit budget |
68
+ | `-w, --workers N` | parallel API requests per token, for instances without rate limits |
69
+ | `--rpm N`, `--index-rpm N`, `--search-rpm N` | requests per minute per worker: all, non-search, search |
70
+ | `--jitter 0.01-0.05` | random delay before every request (seconds, or `10ms-50ms`) |
71
+ | `-A STRING`, `--user-agent-file FILE` | fixed User-Agent, or a random one from the file for every request |
85
72
  | `-m dump-index`, `--dump-projects` | build or rebuild the project index |
86
73
  | `--session NAME`, `--resume` | name a session and continue it later |
87
74
  | `--post-process dedup,convert`, `--final-only` | deduplicate and convert the session right after the search |
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: gitlab-harvester
3
- Version: 1.0.0
3
+ Version: 1.2.2
4
4
  Summary: Build a GitLab instance project index and search repositories for sensitive keywords (API-only, no cloning).
5
5
  Author-email: Cur1iosity <cur1iosity@protonmail.com>
6
6
  License-Expression: MIT
@@ -53,15 +53,17 @@ Dynamic: license-file
53
53
  It searches every repository of a GitLab instance for keywords and secrets through the API,
54
54
  without cloning, including GitLab CE, which has no instance-wide code search.
55
55
 
56
- - **Red team engagements.** A quick PoC: install, point it at an instance, get findings.
57
- Fine-grained settings (branch and fork strategies, time windows, parallel requests)
58
- control how many requests are sent and how much noise they make.
56
+ - **Red team engagements.** One token, the whole instance: credentials, internal hosts
57
+ and pivots hiding in the code. Loud and fast or low and slow, your call: rate limits,
58
+ jitter and User-Agent rotation keep the noise and the signature under control. Captured
59
+ several accounts' tokens? Hand them all to one run and beat a per-user rate limit.
59
60
  - **Penetration tests.** Instance-wide discovery of credentials, tokens and keys as part
60
61
  of the assessment.
61
- - **Blue teams.** Secret hunting and audits: full baselines, regular re-scans of recent
62
- activity, deduplicated reports, and a history of runs in the log file.
63
- - **Developers, analysts and architects.** Surprisingly handy as plain instance-wide code
64
- search: find where an API, a config key or a library is used across all projects.
62
+ - **Blue teams and security research.** Secret hunting and hygiene audits: a quick first
63
+ look, full baselines, regular re-scans of recent activity, deduplicated reports, and a
64
+ history of runs in the log file.
65
+ - **Developers, analysts and architects.** Not for secrets, but for code: fast search
66
+ across a large codebase, even on GitLab CE.
65
67
 
66
68
  <p align="center">
67
69
  <img src="https://raw.githubusercontent.com/Cur1iosity/GitlabHarvester/demo-assets/demo.gif"
@@ -78,40 +80,21 @@ Requires Python 3.10+ and a GitLab token with the `read_api` scope.
78
80
 
79
81
  ## Quick start
80
82
 
81
- ```bash
82
- export GITLAB_TOKEN=...
83
+ Search one term in the default branch of every project:
83
84
 
84
- # Search one term in the default branch of every project
85
+ ```bash
86
+ export GITLAB_TOKEN='your-token-here'
85
87
  gitlab-harvester -u https://gitlab.example.com -t "$GITLAB_TOKEN" -s password
86
-
87
- # Search a list of terms in all branches, skipping branches forks share with their upstream.
88
- # --dump-projects rebuilds the index so it includes all branch names.
89
- gitlab-harvester -u https://gitlab.example.com -t "$GITLAB_TOKEN" -f words.txt -b all --forks branch-diff --dump-projects --session audit
90
-
91
- # Search only repositories with activity on or after a date, e.g. a weekly re-scan.
92
- # Projects inactive since then are skipped without API calls; in active ones every file is searched.
93
- gitlab-harvester -u https://gitlab.example.com -t "$GITLAB_TOKEN" -f words.txt -b all --started-from 2024-06-01 --session since-june
94
-
95
- # Continue after an interruption (reuses the index)
96
- gitlab-harvester -u https://gitlab.example.com -t "$GITLAB_TOKEN" -f words.txt -b all --forks branch-diff --session audit --resume
97
-
98
- # Optional: options can also come from a config file (see config.example.ini); CLI arguments override it
99
- gitlab-harvester -c config.ini -s token
100
-
101
- # Remove duplicate hits and export as a JSON array
102
- gitlab-harvester -m dedup --input-file audit.jsonl --output-file clean.jsonl
103
- gitlab-harvester -m convert --input-file clean.jsonl --output-file clean.json
104
88
  ```
105
89
 
106
- More scenarios, and when to use which options (index-only export, fork strategies,
107
- fresh findings only, parallel requests, baseline and re-scans):
108
- **[docs/USE_CASES.md](docs/USE_CASES.md)**.
109
-
110
90
  The first run builds a project index (`<host>_project_index.jsonl`), and later runs reuse it.
111
- Branch options take effect when the index is built; add `--dump-projects` after changing them.
112
91
  Hits are written to a JSONL session file as they are found.
113
92
 
114
- Common options:
93
+ Lists of terms, all branches and forks, fresh findings only, resuming, parallel requests
94
+ and rate limits, deduplicated reports, config files:
95
+ **[docs/USE_CASES.md](docs/USE_CASES.md)**.
96
+
97
+ ## Common options
115
98
 
116
99
  | Option | Purpose |
117
100
  |---|---|
@@ -119,7 +102,11 @@ Common options:
119
102
  | `--forks include\|skip\|branch-diff\|all-branches` | how forks are handled |
120
103
  | `--time-after DATE`, `--time-before DATE` | limit the index, its projects and the files with hits to a time window |
121
104
  | `--started-from 2024-06-01` | search only indexed projects with activity since the date |
122
- | `-w, --workers N` | parallel API requests, for instances without rate limits |
105
+ | `-t "tok1;tok2"`, `--token-file FILE` | several tokens run in parallel, each with its own workers and rate-limit budget |
106
+ | `-w, --workers N` | parallel API requests per token, for instances without rate limits |
107
+ | `--rpm N`, `--index-rpm N`, `--search-rpm N` | requests per minute per worker: all, non-search, search |
108
+ | `--jitter 0.01-0.05` | random delay before every request (seconds, or `10ms-50ms`) |
109
+ | `-A STRING`, `--user-agent-file FILE` | fixed User-Agent, or a random one from the file for every request |
123
110
  | `-m dump-index`, `--dump-projects` | build or rebuild the project index |
124
111
  | `--session NAME`, `--resume` | name a session and continue it later |
125
112
  | `--post-process dedup,convert`, `--final-only` | deduplicate and convert the session right after the search |
@@ -15,12 +15,16 @@ glh/exceptions.py
15
15
  glh/harvester.py
16
16
  glh/modes.py
17
17
  glh/planner.py
18
+ glh/ratelimit.py
18
19
  glh/session.py
20
+ glh/tokens.py
21
+ glh/user_agents.py
19
22
  glh/workers.py
20
23
  glh/postprocess/__init__.py
21
24
  glh/postprocess/convert.py
22
25
  glh/postprocess/dedup.py
23
26
  glh/postprocess/pipeline.py
27
+ glh/postprocess/split.py
24
28
  glh/utils/__init__.py
25
29
  glh/utils/logging_utils.py
26
30
  glh/utils/terminal_utils.py
@@ -17,6 +17,7 @@ from dataclasses import dataclass
17
17
  from datetime import datetime
18
18
  from importlib.metadata import version, PackageNotFoundError
19
19
 
20
+ from glh import ratelimit, user_agents
20
21
  from glh.config import TERMS_DEST, ConfigError, load_config
21
22
  from glh.planner import parse_timestamp
22
23
  from glh.postprocess.pipeline import normalize_steps
@@ -108,6 +109,37 @@ def parse_workers(value: str) -> int:
108
109
  raise argparse.ArgumentTypeError(f"Use an integer from 1 to {MAX_WORKERS}.")
109
110
 
110
111
 
112
+ def parse_rpm(value: str) -> int:
113
+ """Parse --rpm/--index-rpm/--search-rpm: a non-negative integer (0 = no limit)."""
114
+ v = value.strip()
115
+ if v.isdigit():
116
+ return int(v)
117
+ raise argparse.ArgumentTypeError("Use a non-negative integer (requests per minute, 0 = no limit).")
118
+
119
+
120
+ def parse_jitter(value: str) -> tuple[float, float]:
121
+ """Parse --jitter: MAX or MIN-MAX seconds, optionally with ms/s (0.03, 0.01-0.05, 10ms-50ms)."""
122
+ try:
123
+ return ratelimit.parse_jitter(value)
124
+ except ValueError as e:
125
+ raise argparse.ArgumentTypeError(
126
+ f"{e}. Use MAX or MIN-MAX in seconds, e.g. 0.03, 0.01-0.05 or 10ms-50ms (0 = off).") from None
127
+
128
+
129
+ def parse_user_agent(value: str) -> str:
130
+ """Parse -A/--user-agent: a User-Agent that can be sent as an HTTP header."""
131
+ try:
132
+ return user_agents.check_user_agent(value)
133
+ except user_agents.UserAgentError as e:
134
+ raise argparse.ArgumentTypeError(str(e)) from None
135
+
136
+
137
+ # (specific dest, group dest) for every request rate limit; resolved like TIME_BOUNDS.
138
+ RATE_LIMITS: tuple[tuple[str, str], ...] = (
139
+ ("index_rpm", "rpm"),
140
+ ("search_rpm", "rpm"),
141
+ )
142
+
111
143
  # (specific dest, group dest) for every time bound; see CliParser.parse().
112
144
  TIME_BOUNDS: tuple[tuple[str, str], ...] = (
113
145
  ("index_active_after", "time_after"),
@@ -150,11 +182,17 @@ class CliArgs:
150
182
  config: str | None
151
183
  url: str | None
152
184
  token: str | None
185
+ token_file: str | None
153
186
  proxy: str | None
154
187
  timeout: int
155
188
  insecure: bool
156
189
  ca_bundle: str | None
157
190
  workers: int
191
+ index_rpm: int
192
+ search_rpm: int
193
+ jitter: tuple[float, float]
194
+ user_agent: str | None
195
+ user_agent_file: str | None
158
196
 
159
197
  log_file: str | None
160
198
  log_level: str
@@ -183,6 +221,7 @@ class CliArgs:
183
221
  session: str | None
184
222
  session_file: str | None
185
223
  resume: bool
224
+ output_separated: bool
186
225
 
187
226
  input_file: str | None
188
227
  output_file: str | None
@@ -226,7 +265,17 @@ class CliParser:
226
265
  default=None,
227
266
  help="GitLab base URL (e.g., https://gitlab.example.com). Alias: -H/--host (deprecated).",
228
267
  )
229
- parser.add_argument("-t", "--token", default=None, help="GitLab token (required for search/dump-index).",)
268
+ tok = parser.add_mutually_exclusive_group()
269
+ tok.add_argument(
270
+ "-t", "--token", default=None,
271
+ help="GitLab token(s), required for search/dump-index. Several tokens separated by ';' run in "
272
+ "parallel, each with its own --workers clients and its own rate-limit budget.",
273
+ )
274
+ tok.add_argument(
275
+ "--token-file", default=None, metavar="FILE",
276
+ help="File with one GitLab token per line (# comments allowed); each token runs in parallel, "
277
+ "like several tokens in -t.",
278
+ )
230
279
  parser.add_argument("-p", "--proxy", default=None, help="HTTP(S) proxy URL for GitLab API traffic.",)
231
280
  parser.add_argument("--timeout", default=60, type=int, help="Timeout in seconds for GitLab API request.",)
232
281
  parser.add_argument(
@@ -250,6 +299,31 @@ class CliParser:
250
299
  help=f"Parallel API requests (1-{MAX_WORKERS}, default 1 = sequential). Speeds up project listing, "
251
300
  "branch listing and search; use more than 1 only if the instance has no API rate limits.",
252
301
  )
302
+ parser.add_argument(
303
+ "--rpm", type=parse_rpm, default=None, metavar="N",
304
+ help="Shortcut for --index-rpm and --search-rpm (a specific option overrides it).",
305
+ )
306
+ parser.add_argument(
307
+ "--index-rpm", type=parse_rpm, default=None, metavar="N",
308
+ help="Max API requests per minute per worker, except search requests: project and branch "
309
+ "listing, file-change checks (default 0 = no limit).",
310
+ )
311
+ parser.add_argument(
312
+ "--search-rpm", type=parse_rpm, default=None, metavar="N",
313
+ help="Max search API requests per minute per worker; GitLab rate-limits search separately "
314
+ "(default 0 = no limit).",
315
+ )
316
+ parser.add_argument(
317
+ "--jitter", type=parse_jitter, default=None, metavar="RANGE",
318
+ help="Random delay before every API request, drawn anew for each one: MAX or MIN-MAX seconds, "
319
+ "e.g. 0.03, 0.01-0.05 or 10ms-50ms (default: off). Adds to the --*-rpm spacing.",
320
+ )
321
+ ua = parser.add_mutually_exclusive_group()
322
+ ua.add_argument("-A", "--user-agent", type=parse_user_agent, default=None, metavar="STRING",
323
+ help="User-Agent of every API request (default: python-gitlab's).")
324
+ ua.add_argument("--user-agent-file", default=None, metavar="FILE",
325
+ help="File with one User-Agent per line (# comments allowed): every API request "
326
+ "gets one chosen at random.")
253
327
 
254
328
  # Logging
255
329
  parser.add_argument("--log-file", default=None, help="Write logs to a file (in addition to stderr).")
@@ -334,6 +408,14 @@ class CliParser:
334
408
  parser.add_argument("--resume", action=argparse.BooleanOptionalAction, default=False,
335
409
  help="Resume from existing session file.")
336
410
 
411
+ parser.add_argument(
412
+ "--output-separated",
413
+ action=argparse.BooleanOptionalAction,
414
+ default=False,
415
+ help="With several search terms, also write each term's hits to its own file "
416
+ "<session>.<term>.jsonl next to the session (ignored for a single term).",
417
+ )
418
+
337
419
  # Postprocess
338
420
  parser.add_argument(
339
421
  "--post-process",
@@ -382,15 +464,16 @@ class CliParser:
382
464
  return set(vars(probe.parse_args(argv)))
383
465
 
384
466
  @staticmethod
385
- def _resolve_time_bounds(ns: argparse.Namespace, given: set[str], config: dict) -> None:
467
+ def _resolve_group_options(ns: argparse.Namespace, given: set[str], config: dict) -> None:
386
468
  """
387
- Resolve every time bound in place and drop time keys from ``config``.
469
+ Resolve every option that has a group shortcut (time bounds, rate limits)
470
+ in place and drop their keys from ``config``.
388
471
 
389
- Order: specific option on the CLI > --time-after/--time-before on the CLI >
390
- specific key in the config file > time-after/time-before in the config file.
472
+ Order: specific option on the CLI > group option (--time-after, --rpm, ...)
473
+ on the CLI > specific key in the config file > group key in the config file.
391
474
  """
392
475
  resolved: dict[str, object] = {}
393
- for specific, group in TIME_BOUNDS:
476
+ for specific, group in TIME_BOUNDS + RATE_LIMITS:
394
477
  if specific in given:
395
478
  resolved[specific] = getattr(ns, specific)
396
479
  elif group in given:
@@ -401,7 +484,7 @@ class CliParser:
401
484
  resolved[specific] = config[group]
402
485
  else:
403
486
  resolved[specific] = None
404
- for dest in {d for pair in TIME_BOUNDS for d in pair}:
487
+ for dest in {d for pair in TIME_BOUNDS + RATE_LIMITS for d in pair}:
405
488
  config.pop(dest, None)
406
489
  for dest, value in resolved.items():
407
490
  setattr(ns, dest, value)
@@ -427,7 +510,8 @@ class CliParser:
427
510
  parser.error(str(e))
428
511
  given = cls._given_dests(argv)
429
512
 
430
- exclusive = (("search", "terms_file", TERMS_DEST), ("session", "session_file"))
513
+ exclusive = (("search", "terms_file", TERMS_DEST), ("session", "session_file"),
514
+ ("user_agent", "user_agent_file"), ("token", "token_file"))
431
515
  for group in exclusive:
432
516
  if given & set(group):
433
517
  for dest in group:
@@ -439,14 +523,18 @@ class CliParser:
439
523
  if dest not in given:
440
524
  config.pop(dest, None)
441
525
 
442
- cls._resolve_time_bounds(ns, given, config)
526
+ cls._resolve_group_options(ns, given, config)
443
527
 
444
528
  for dest, value in config.items():
445
529
  if dest not in given:
446
530
  setattr(ns, dest, value)
447
531
  else:
448
- cls._resolve_time_bounds(ns, {d for d, _ in TIME_BOUNDS + (("time_after", ""), ("time_before", ""))
449
- if getattr(ns, d) is not None}, {})
532
+ grouped = TIME_BOUNDS + RATE_LIMITS + (("time_after", ""), ("time_before", ""), ("rpm", ""))
533
+ cls._resolve_group_options(ns, {d for d, _ in grouped if getattr(ns, d) is not None}, {})
534
+
535
+ for dest, _ in RATE_LIMITS:
536
+ if getattr(ns, dest) is None:
537
+ setattr(ns, dest, 0)
450
538
 
451
539
  if ns.insecure and ns.ca_bundle:
452
540
  parser.error("--insecure and --ca-bundle exclude each other.")
@@ -473,11 +561,17 @@ class CliParser:
473
561
  config=ns.config,
474
562
  url=ns.url,
475
563
  token=ns.token,
564
+ token_file=ns.token_file,
476
565
  proxy=ns.proxy,
477
566
  timeout=ns.timeout,
478
567
  insecure=ns.insecure,
479
568
  ca_bundle=ns.ca_bundle,
480
569
  workers=ns.workers,
570
+ index_rpm=ns.index_rpm,
571
+ search_rpm=ns.search_rpm,
572
+ jitter=ns.jitter or ratelimit.NO_JITTER,
573
+ user_agent=ns.user_agent,
574
+ user_agent_file=ns.user_agent_file,
481
575
  log_file=ns.log_file,
482
576
  log_level=ns.log_level,
483
577
  debug=ns.debug,
@@ -502,6 +596,7 @@ class CliParser:
502
596
  session=ns.session,
503
597
  session_file=ns.session_file,
504
598
  resume=ns.resume,
599
+ output_separated=ns.output_separated,
505
600
  input_file=ns.input_file,
506
601
  output_file=ns.output_file,
507
602
  post_process=tuple(ns.post_process),
@@ -16,8 +16,9 @@ Extras that have no CLI counterpart:
16
16
  - ``token = env:NAME`` reads the token from the environment variable NAME.
17
17
 
18
18
  Relative paths (``index-file``, ``terms-file``, ``session-file``, ``log-file``,
19
- ``input-file``, ``output-file``, ``sqlite-path``, ``ca-bundle``) are resolved against the
20
- directory of the config file, so the config works from any working directory.
19
+ ``input-file``, ``output-file``, ``sqlite-path``, ``ca-bundle``, ``user-agent-file``,
20
+ ``token-file``) are resolved against the directory of the config file, so the
21
+ config works from any working directory.
21
22
  """
22
23
 
23
24
  from __future__ import annotations
@@ -30,7 +31,7 @@ from typing import Any
30
31
 
31
32
  # Keys (argparse dests) that hold file system paths.
32
33
  PATH_DESTS = frozenset({"index_file", "terms_file", "session_file", "log_file", "input_file", "output_file",
33
- "sqlite_path", "ca_bundle"})
34
+ "sqlite_path", "ca_bundle", "user_agent_file", "token_file"})
34
35
  # Config-only keys (no CLI option).
35
36
  TERMS_DEST = "terms"
36
37
  # Options that make no sense in a config file.
@@ -156,7 +157,8 @@ def load_config(path: str | os.PathLike[str], parser: argparse.ArgumentParser) -
156
157
  values[action.dest] = value
157
158
 
158
159
  # The same exclusivity rules as on the command line.
159
- for group in (("search", "terms_file", TERMS_DEST), ("session", "session_file")):
160
+ for group in (("search", "terms_file", TERMS_DEST), ("session", "session_file"),
161
+ ("user_agent", "user_agent_file"), ("token", "token_file")):
160
162
  present = [d for d in group if d in values]
161
163
  if len(present) > 1:
162
164
  names = ", ".join(d.replace("_", "-") for d in present)