gitlab-harvester 0.2.5__tar.gz → 0.2.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gitlab_harvester-0.2.5 → gitlab_harvester-0.2.6}/PKG-INFO +11 -11
- {gitlab_harvester-0.2.5 → gitlab_harvester-0.2.6}/README.md +10 -10
- {gitlab_harvester-0.2.5 → gitlab_harvester-0.2.6}/gitlab_harvester.egg-info/PKG-INFO +11 -11
- {gitlab_harvester-0.2.5 → gitlab_harvester-0.2.6}/gitlab_harvester.egg-info/SOURCES.txt +5 -1
- gitlab_harvester-0.2.6/gitlab_harvester.py +39 -0
- gitlab_harvester-0.2.6/glh/cli.py +249 -0
- gitlab_harvester-0.2.6/glh/exceptions.py +9 -0
- {gitlab_harvester-0.2.5 → gitlab_harvester-0.2.6}/glh/harvester.py +177 -351
- {gitlab_harvester-0.2.5 → gitlab_harvester-0.2.6}/glh/modes.py +41 -16
- gitlab_harvester-0.2.6/glh/utils/__init__.py +0 -0
- gitlab_harvester-0.2.6/glh/utils/logging_utils.py +134 -0
- gitlab_harvester-0.2.6/glh/utils/terminal_utils.py +137 -0
- {gitlab_harvester-0.2.5 → gitlab_harvester-0.2.6}/pyproject.toml +1 -1
- gitlab_harvester-0.2.5/gitlab_harvester.py +0 -26
- gitlab_harvester-0.2.5/glh/cli.py +0 -189
- {gitlab_harvester-0.2.5 → gitlab_harvester-0.2.6}/LICENSE +0 -0
- {gitlab_harvester-0.2.5 → gitlab_harvester-0.2.6}/gitlab_harvester.egg-info/dependency_links.txt +0 -0
- {gitlab_harvester-0.2.5 → gitlab_harvester-0.2.6}/gitlab_harvester.egg-info/entry_points.txt +0 -0
- {gitlab_harvester-0.2.5 → gitlab_harvester-0.2.6}/gitlab_harvester.egg-info/requires.txt +0 -0
- {gitlab_harvester-0.2.5 → gitlab_harvester-0.2.6}/gitlab_harvester.egg-info/top_level.txt +0 -0
- {gitlab_harvester-0.2.5 → gitlab_harvester-0.2.6}/glh/__init__.py +0 -0
- {gitlab_harvester-0.2.5 → gitlab_harvester-0.2.6}/glh/planner.py +0 -0
- {gitlab_harvester-0.2.5 → gitlab_harvester-0.2.6}/glh/postprocess/__init__.py +0 -0
- {gitlab_harvester-0.2.5 → gitlab_harvester-0.2.6}/glh/postprocess/convert.py +0 -0
- {gitlab_harvester-0.2.5 → gitlab_harvester-0.2.6}/glh/postprocess/dedup.py +0 -0
- {gitlab_harvester-0.2.5 → gitlab_harvester-0.2.6}/glh/session.py +0 -0
- {gitlab_harvester-0.2.5 → gitlab_harvester-0.2.6}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gitlab-harvester
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.6
|
|
4
4
|
Summary: Build a GitLab instance project index and search repositories for sensitive keywords (API-only, no cloning).
|
|
5
5
|
Author-email: Cur1iosity <cur1iosity@protonmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -51,19 +51,19 @@ Built for **security audits, secret discovery, compliance checks, and large-scal
|
|
|
51
51
|
Search a keyword:
|
|
52
52
|
|
|
53
53
|
```bash
|
|
54
|
-
gitlab-harvester -
|
|
54
|
+
gitlab-harvester -u https://gitlab.example.com -t $TOKEN --search password
|
|
55
55
|
```
|
|
56
56
|
|
|
57
57
|
Search from file:
|
|
58
58
|
|
|
59
59
|
```bash
|
|
60
|
-
gitlab-harvester -
|
|
60
|
+
gitlab-harvester -u https://gitlab.example.com -t $TOKEN --terms-file words.txt
|
|
61
61
|
```
|
|
62
62
|
|
|
63
63
|
Build project index only:
|
|
64
64
|
|
|
65
65
|
```bash
|
|
66
|
-
gitlab-harvester -
|
|
66
|
+
gitlab-harvester -u https://gitlab.example.com -t $TOKEN -m dump-index
|
|
67
67
|
```
|
|
68
68
|
|
|
69
69
|
Deduplicate results:
|
|
@@ -132,7 +132,7 @@ pipx install gitlab-harvester
|
|
|
132
132
|
Run:
|
|
133
133
|
|
|
134
134
|
```bash
|
|
135
|
-
gitlab-harvester --
|
|
135
|
+
gitlab-harvester --uelp
|
|
136
136
|
```
|
|
137
137
|
|
|
138
138
|
---
|
|
@@ -186,13 +186,13 @@ Two independent controls:
|
|
|
186
186
|
Example:
|
|
187
187
|
|
|
188
188
|
```bash
|
|
189
|
-
gitlab-harvester -
|
|
189
|
+
gitlab-harvester -u ... -t ... --scan-branches 10
|
|
190
190
|
```
|
|
191
191
|
|
|
192
192
|
Store all + scan all:
|
|
193
193
|
|
|
194
194
|
```bash
|
|
195
|
-
gitlab-harvester -
|
|
195
|
+
gitlab-harvester -u ... -t ... --index-branches all --scan-branches all
|
|
196
196
|
```
|
|
197
197
|
|
|
198
198
|
Shortcut:
|
|
@@ -225,13 +225,13 @@ Recommended → **branch-diff**
|
|
|
225
225
|
Create session:
|
|
226
226
|
|
|
227
227
|
```bash
|
|
228
|
-
gitlab-harvester -
|
|
228
|
+
gitlab-harvester -u ... -t ... --terms-file words.txt --session audit
|
|
229
229
|
```
|
|
230
230
|
|
|
231
231
|
Resume:
|
|
232
232
|
|
|
233
233
|
```bash
|
|
234
|
-
gitlab-harvester -
|
|
234
|
+
gitlab-harvester -u ... -t ... --session-file audit.jsonl --resume
|
|
235
235
|
```
|
|
236
236
|
|
|
237
237
|
---
|
|
@@ -264,8 +264,8 @@ gitlab-harvester -m dedup \
|
|
|
264
264
|
Options:
|
|
265
265
|
|
|
266
266
|
* `--sqlite-path file.sqlite`
|
|
267
|
-
* `--
|
|
268
|
-
* `--no-normalize-
|
|
267
|
+
* `--uash-algo blake2b|sha1|sha256`
|
|
268
|
+
* `--no-normalize-uits`
|
|
269
269
|
|
|
270
270
|
---
|
|
271
271
|
|
|
@@ -17,19 +17,19 @@ Built for **security audits, secret discovery, compliance checks, and large-scal
|
|
|
17
17
|
Search a keyword:
|
|
18
18
|
|
|
19
19
|
```bash
|
|
20
|
-
gitlab-harvester -
|
|
20
|
+
gitlab-harvester -u https://gitlab.example.com -t $TOKEN --search password
|
|
21
21
|
```
|
|
22
22
|
|
|
23
23
|
Search from file:
|
|
24
24
|
|
|
25
25
|
```bash
|
|
26
|
-
gitlab-harvester -
|
|
26
|
+
gitlab-harvester -u https://gitlab.example.com -t $TOKEN --terms-file words.txt
|
|
27
27
|
```
|
|
28
28
|
|
|
29
29
|
Build project index only:
|
|
30
30
|
|
|
31
31
|
```bash
|
|
32
|
-
gitlab-harvester -
|
|
32
|
+
gitlab-harvester -u https://gitlab.example.com -t $TOKEN -m dump-index
|
|
33
33
|
```
|
|
34
34
|
|
|
35
35
|
Deduplicate results:
|
|
@@ -98,7 +98,7 @@ pipx install gitlab-harvester
|
|
|
98
98
|
Run:
|
|
99
99
|
|
|
100
100
|
```bash
|
|
101
|
-
gitlab-harvester --
|
|
101
|
+
gitlab-harvester --uelp
|
|
102
102
|
```
|
|
103
103
|
|
|
104
104
|
---
|
|
@@ -152,13 +152,13 @@ Two independent controls:
|
|
|
152
152
|
Example:
|
|
153
153
|
|
|
154
154
|
```bash
|
|
155
|
-
gitlab-harvester -
|
|
155
|
+
gitlab-harvester -u ... -t ... --scan-branches 10
|
|
156
156
|
```
|
|
157
157
|
|
|
158
158
|
Store all + scan all:
|
|
159
159
|
|
|
160
160
|
```bash
|
|
161
|
-
gitlab-harvester -
|
|
161
|
+
gitlab-harvester -u ... -t ... --index-branches all --scan-branches all
|
|
162
162
|
```
|
|
163
163
|
|
|
164
164
|
Shortcut:
|
|
@@ -191,13 +191,13 @@ Recommended → **branch-diff**
|
|
|
191
191
|
Create session:
|
|
192
192
|
|
|
193
193
|
```bash
|
|
194
|
-
gitlab-harvester -
|
|
194
|
+
gitlab-harvester -u ... -t ... --terms-file words.txt --session audit
|
|
195
195
|
```
|
|
196
196
|
|
|
197
197
|
Resume:
|
|
198
198
|
|
|
199
199
|
```bash
|
|
200
|
-
gitlab-harvester -
|
|
200
|
+
gitlab-harvester -u ... -t ... --session-file audit.jsonl --resume
|
|
201
201
|
```
|
|
202
202
|
|
|
203
203
|
---
|
|
@@ -230,8 +230,8 @@ gitlab-harvester -m dedup \
|
|
|
230
230
|
Options:
|
|
231
231
|
|
|
232
232
|
* `--sqlite-path file.sqlite`
|
|
233
|
-
* `--
|
|
234
|
-
* `--no-normalize-
|
|
233
|
+
* `--uash-algo blake2b|sha1|sha256`
|
|
234
|
+
* `--no-normalize-uits`
|
|
235
235
|
|
|
236
236
|
---
|
|
237
237
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gitlab-harvester
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.6
|
|
4
4
|
Summary: Build a GitLab instance project index and search repositories for sensitive keywords (API-only, no cloning).
|
|
5
5
|
Author-email: Cur1iosity <cur1iosity@protonmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -51,19 +51,19 @@ Built for **security audits, secret discovery, compliance checks, and large-scal
|
|
|
51
51
|
Search a keyword:
|
|
52
52
|
|
|
53
53
|
```bash
|
|
54
|
-
gitlab-harvester -
|
|
54
|
+
gitlab-harvester -u https://gitlab.example.com -t $TOKEN --search password
|
|
55
55
|
```
|
|
56
56
|
|
|
57
57
|
Search from file:
|
|
58
58
|
|
|
59
59
|
```bash
|
|
60
|
-
gitlab-harvester -
|
|
60
|
+
gitlab-harvester -u https://gitlab.example.com -t $TOKEN --terms-file words.txt
|
|
61
61
|
```
|
|
62
62
|
|
|
63
63
|
Build project index only:
|
|
64
64
|
|
|
65
65
|
```bash
|
|
66
|
-
gitlab-harvester -
|
|
66
|
+
gitlab-harvester -u https://gitlab.example.com -t $TOKEN -m dump-index
|
|
67
67
|
```
|
|
68
68
|
|
|
69
69
|
Deduplicate results:
|
|
@@ -132,7 +132,7 @@ pipx install gitlab-harvester
|
|
|
132
132
|
Run:
|
|
133
133
|
|
|
134
134
|
```bash
|
|
135
|
-
gitlab-harvester --
|
|
135
|
+
gitlab-harvester --uelp
|
|
136
136
|
```
|
|
137
137
|
|
|
138
138
|
---
|
|
@@ -186,13 +186,13 @@ Two independent controls:
|
|
|
186
186
|
Example:
|
|
187
187
|
|
|
188
188
|
```bash
|
|
189
|
-
gitlab-harvester -
|
|
189
|
+
gitlab-harvester -u ... -t ... --scan-branches 10
|
|
190
190
|
```
|
|
191
191
|
|
|
192
192
|
Store all + scan all:
|
|
193
193
|
|
|
194
194
|
```bash
|
|
195
|
-
gitlab-harvester -
|
|
195
|
+
gitlab-harvester -u ... -t ... --index-branches all --scan-branches all
|
|
196
196
|
```
|
|
197
197
|
|
|
198
198
|
Shortcut:
|
|
@@ -225,13 +225,13 @@ Recommended → **branch-diff**
|
|
|
225
225
|
Create session:
|
|
226
226
|
|
|
227
227
|
```bash
|
|
228
|
-
gitlab-harvester -
|
|
228
|
+
gitlab-harvester -u ... -t ... --terms-file words.txt --session audit
|
|
229
229
|
```
|
|
230
230
|
|
|
231
231
|
Resume:
|
|
232
232
|
|
|
233
233
|
```bash
|
|
234
|
-
gitlab-harvester -
|
|
234
|
+
gitlab-harvester -u ... -t ... --session-file audit.jsonl --resume
|
|
235
235
|
```
|
|
236
236
|
|
|
237
237
|
---
|
|
@@ -264,8 +264,8 @@ gitlab-harvester -m dedup \
|
|
|
264
264
|
Options:
|
|
265
265
|
|
|
266
266
|
* `--sqlite-path file.sqlite`
|
|
267
|
-
* `--
|
|
268
|
-
* `--no-normalize-
|
|
267
|
+
* `--uash-algo blake2b|sha1|sha256`
|
|
268
|
+
* `--no-normalize-uits`
|
|
269
269
|
|
|
270
270
|
---
|
|
271
271
|
|
|
@@ -10,10 +10,14 @@ gitlab_harvester.egg-info/requires.txt
|
|
|
10
10
|
gitlab_harvester.egg-info/top_level.txt
|
|
11
11
|
glh/__init__.py
|
|
12
12
|
glh/cli.py
|
|
13
|
+
glh/exceptions.py
|
|
13
14
|
glh/harvester.py
|
|
14
15
|
glh/modes.py
|
|
15
16
|
glh/planner.py
|
|
16
17
|
glh/session.py
|
|
17
18
|
glh/postprocess/__init__.py
|
|
18
19
|
glh/postprocess/convert.py
|
|
19
|
-
glh/postprocess/dedup.py
|
|
20
|
+
glh/postprocess/dedup.py
|
|
21
|
+
glh/utils/__init__.py
|
|
22
|
+
glh/utils/logging_utils.py
|
|
23
|
+
glh/utils/terminal_utils.py
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import signal
|
|
6
|
+
import sys
|
|
7
|
+
|
|
8
|
+
from glh.cli import CliParser, get_banner
|
|
9
|
+
from glh.modes import run_mode
|
|
10
|
+
|
|
11
|
+
__all__ = ["main"]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _handle_exit(signum: int, _frame) -> None: # noqa: ANN001
|
|
15
|
+
"""Handle SIGINT/SIGTERM with a clean exit code and no traceback."""
|
|
16
|
+
print("\n[!] Interrupted — exiting.", file=sys.stderr)
|
|
17
|
+
code = 130 if signum == signal.SIGINT else 143 # 128 + SIGINT(2) / SIGTERM(15)
|
|
18
|
+
raise SystemExit(code)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def main(argv: list[str] | None = None) -> None:
|
|
22
|
+
"""CLI entrypoint."""
|
|
23
|
+
signal.signal(signal.SIGINT, _handle_exit)
|
|
24
|
+
signal.signal(signal.SIGTERM, _handle_exit)
|
|
25
|
+
|
|
26
|
+
argv = list(sys.argv[1:] if argv is None else argv)
|
|
27
|
+
|
|
28
|
+
if not any(a in {"-h", "--help", "-V", "--version"} for a in argv):
|
|
29
|
+
print(get_banner(), file=sys.stderr)
|
|
30
|
+
|
|
31
|
+
try:
|
|
32
|
+
args = CliParser.parse(argv)
|
|
33
|
+
run_mode(args)
|
|
34
|
+
except KeyboardInterrupt:
|
|
35
|
+
_handle_exit(signal.SIGINT, None)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
if __name__ == "__main__":
|
|
39
|
+
main()
|
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
# glh/cli.py
|
|
2
|
+
"""
|
|
3
|
+
Command-line interface parsing for GitlabHarvester.
|
|
4
|
+
|
|
5
|
+
This module provides:
|
|
6
|
+
- Version resolution via importlib.metadata
|
|
7
|
+
- Branches mode parsing helpers
|
|
8
|
+
- A typed CliArgs container
|
|
9
|
+
- Argparse builder and parser
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import argparse
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
from importlib.metadata import version, PackageNotFoundError
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
BranchesMode = str | int # "default" | "all" | int
|
|
20
|
+
|
|
21
|
+
DEFAULT_INDEX_BRANCHES: BranchesMode = "default"
|
|
22
|
+
DEFAULT_SCAN_BRANCHES: BranchesMode = "default"
|
|
23
|
+
_BANNER = r"""
|
|
24
|
+
┏┓• ┓ ┓ ┓┏
|
|
25
|
+
┃┓┓╋┃┏┓┣┓┣┫┏┓┏┓┓┏┏┓┏╋┏┓┏┓
|
|
26
|
+
┗┛┗┗┗┗┻┗┛┛┗┗┻┛ ┗┛┗ ┛┗┗ ┛
|
|
27
|
+
""".rstrip()
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _get_version() -> str:
|
|
31
|
+
"""Return installed package version (or 'unknown' if not installed)."""
|
|
32
|
+
try:
|
|
33
|
+
return version("gitlab-harvester")
|
|
34
|
+
except PackageNotFoundError:
|
|
35
|
+
return "unknown"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def get_banner() -> str:
|
|
39
|
+
"""Return the CLI banner with version suffix."""
|
|
40
|
+
return f"{_BANNER} v{_get_version()}"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def parse_branches(value: str) -> BranchesMode:
|
|
44
|
+
"""Parse branches mode argument.
|
|
45
|
+
|
|
46
|
+
Allowed values:
|
|
47
|
+
- 'default'
|
|
48
|
+
- 'all'
|
|
49
|
+
- positive integer (e.g. '10')
|
|
50
|
+
|
|
51
|
+
Args:
|
|
52
|
+
value: Raw CLI value.
|
|
53
|
+
|
|
54
|
+
Returns:
|
|
55
|
+
Parsed branches mode.
|
|
56
|
+
|
|
57
|
+
Raises:
|
|
58
|
+
argparse.ArgumentTypeError: If value is invalid.
|
|
59
|
+
"""
|
|
60
|
+
v = value.strip().lower()
|
|
61
|
+
if v in {"default", "all"}:
|
|
62
|
+
return v
|
|
63
|
+
if v.isdigit() and int(v) > 0:
|
|
64
|
+
return int(v)
|
|
65
|
+
raise argparse.ArgumentTypeError("Use 'default', 'all', or a positive integer (e.g., 10).")
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@dataclass(frozen=True, slots=True)
|
|
69
|
+
class CliArgs:
|
|
70
|
+
"""Typed container for parsed CLI arguments."""
|
|
71
|
+
|
|
72
|
+
url: str | None
|
|
73
|
+
token: str | None
|
|
74
|
+
proxy: str | None
|
|
75
|
+
|
|
76
|
+
log_file: str | None
|
|
77
|
+
log_level: str
|
|
78
|
+
debug: bool
|
|
79
|
+
|
|
80
|
+
mode: str
|
|
81
|
+
|
|
82
|
+
batch_size: int
|
|
83
|
+
dump_projects: bool
|
|
84
|
+
index_file: str | None
|
|
85
|
+
index_branches: BranchesMode
|
|
86
|
+
branches_per_page: int
|
|
87
|
+
|
|
88
|
+
scan_branches: BranchesMode
|
|
89
|
+
forks: str
|
|
90
|
+
fork_diff_bases: str
|
|
91
|
+
search: str | None
|
|
92
|
+
terms_file: str | None
|
|
93
|
+
session: str | None
|
|
94
|
+
session_file: str | None
|
|
95
|
+
resume: bool
|
|
96
|
+
|
|
97
|
+
input_file: str | None
|
|
98
|
+
output_file: str | None
|
|
99
|
+
|
|
100
|
+
sqlite_path: str | None
|
|
101
|
+
hash_algo: str
|
|
102
|
+
normalize_hits: bool
|
|
103
|
+
|
|
104
|
+
sort_keys: bool
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
class CliParser:
|
|
108
|
+
"""Argparse wrapper for building and parsing GitlabHarvester CLI."""
|
|
109
|
+
|
|
110
|
+
@staticmethod
|
|
111
|
+
def build() -> argparse.ArgumentParser:
|
|
112
|
+
"""Build and return the configured ArgumentParser."""
|
|
113
|
+
parser = argparse.ArgumentParser(
|
|
114
|
+
prog="gitlab-harvester",
|
|
115
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
116
|
+
description=f"{get_banner()}\nSearch GitLab instances for sensitive data, secrets, and patterns "
|
|
117
|
+
f"across projects and branches using a local project index.",
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
# Core connectivity
|
|
121
|
+
# NOTE: -H/--host is kept for backward compatibility and maps to --url.
|
|
122
|
+
parser.add_argument(
|
|
123
|
+
"-u", "--url", "-H", "--host",
|
|
124
|
+
dest="url",
|
|
125
|
+
default=None,
|
|
126
|
+
help="GitLab base URL (e.g., https://gitlab.example.com). Alias: -H/--host (deprecated).",
|
|
127
|
+
)
|
|
128
|
+
parser.add_argument("-t", "--token", default=None, help="GitLab token (required for search/dump-index).",)
|
|
129
|
+
parser.add_argument("-p", "--proxy", default=None, help="HTTP(S) proxy URL for GitLab API traffic.",)
|
|
130
|
+
|
|
131
|
+
# Logging
|
|
132
|
+
parser.add_argument("--log-file", default=None, help="Write logs to a file (in addition to stderr).")
|
|
133
|
+
parser.add_argument("--log-level", choices=("ERROR", "WARN", "INFO"), default="INFO", help="Console log level.")
|
|
134
|
+
parser.add_argument("--debug", action="store_true", help="Enable debug logging (overrides --log-level).")
|
|
135
|
+
|
|
136
|
+
# Mode
|
|
137
|
+
parser.add_argument(
|
|
138
|
+
"-m",
|
|
139
|
+
"--mode",
|
|
140
|
+
choices=("search", "dump-index", "dedup", "convert"),
|
|
141
|
+
default="search",
|
|
142
|
+
help="Execution mode.",
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
# Index
|
|
146
|
+
parser.add_argument("-bs", "--batch-size", type=int, default=100, help="API pagination batch size.")
|
|
147
|
+
parser.add_argument("--index-file", default=None, help="Path to the Instance Project Index file (JSONL).")
|
|
148
|
+
parser.add_argument("--dump-projects", action="store_true", help="Force rebuilding the project index.")
|
|
149
|
+
|
|
150
|
+
# Branches
|
|
151
|
+
parser.add_argument(
|
|
152
|
+
"-b",
|
|
153
|
+
"--branches",
|
|
154
|
+
dest="branches",
|
|
155
|
+
type=parse_branches,
|
|
156
|
+
default=None,
|
|
157
|
+
help="Convenience: set both --index-branches and --scan-branches.",
|
|
158
|
+
)
|
|
159
|
+
parser.add_argument("--index-branches", type=parse_branches, default=None, help="Branches mode for index build.")
|
|
160
|
+
parser.add_argument("--scan-branches", type=parse_branches, default=None, help="Branches mode for scanning.")
|
|
161
|
+
parser.add_argument("--branches-per-page", type=int, default=100, help="Branches API pagination size.")
|
|
162
|
+
|
|
163
|
+
# Forks
|
|
164
|
+
parser.add_argument(
|
|
165
|
+
"--forks",
|
|
166
|
+
choices=("skip", "include", "branch-diff", "all-branches"),
|
|
167
|
+
default="include",
|
|
168
|
+
help="Fork handling strategy.",
|
|
169
|
+
)
|
|
170
|
+
parser.add_argument("--fork-diff-bases", default="main,master,develop,dev", help="Comma-separated base branches.")
|
|
171
|
+
|
|
172
|
+
# Search
|
|
173
|
+
group = parser.add_mutually_exclusive_group()
|
|
174
|
+
group.add_argument("-s", "--search", default=None, help="Single search term.")
|
|
175
|
+
group.add_argument("-f", "--terms-file", default=None, help="Path to file with one term per line.")
|
|
176
|
+
|
|
177
|
+
sess = parser.add_mutually_exclusive_group()
|
|
178
|
+
sess.add_argument("--session", default=None, help="Session name (creates results/<name>.jsonl).")
|
|
179
|
+
sess.add_argument("--session-file", default=None, help="Explicit session file path (JSONL).")
|
|
180
|
+
|
|
181
|
+
parser.add_argument("--resume", action="store_true", help="Resume from existing session file.")
|
|
182
|
+
|
|
183
|
+
# Postprocess
|
|
184
|
+
parser.add_argument("--input-file", default=None, help="Input file for postprocess modes.")
|
|
185
|
+
parser.add_argument("--output-file", default=None, help="Output file for postprocess modes.")
|
|
186
|
+
|
|
187
|
+
# Dedup
|
|
188
|
+
parser.add_argument("--sqlite-path", default=None, help="Optional sqlite cache path for dedup.")
|
|
189
|
+
parser.add_argument(
|
|
190
|
+
"--hash-algo",
|
|
191
|
+
choices=("blake2b", "sha1", "sha256"),
|
|
192
|
+
default="blake2b",
|
|
193
|
+
help="Hash algorithm for deduplication.",
|
|
194
|
+
)
|
|
195
|
+
parser.add_argument("--no-normalize-hits", action="store_true", help="Disable hit normalization.")
|
|
196
|
+
|
|
197
|
+
# Convert
|
|
198
|
+
parser.add_argument("--sort-keys", action="store_true", help="Sort JSON keys in convert mode output.")
|
|
199
|
+
|
|
200
|
+
parser.add_argument("-V", "--version", action="version", version=f"%(prog)s {_get_version()}")
|
|
201
|
+
|
|
202
|
+
return parser
|
|
203
|
+
|
|
204
|
+
@classmethod
|
|
205
|
+
def parse(cls, argv: list[str] | None = None) -> CliArgs:
|
|
206
|
+
"""Parse argv into CliArgs."""
|
|
207
|
+
ns = cls.build().parse_args(argv)
|
|
208
|
+
|
|
209
|
+
if ns.debug:
|
|
210
|
+
ns.log_level = "DEBUG"
|
|
211
|
+
|
|
212
|
+
if ns.index_branches is None and ns.branches is not None:
|
|
213
|
+
ns.index_branches = ns.branches
|
|
214
|
+
if ns.scan_branches is None and ns.branches is not None:
|
|
215
|
+
ns.scan_branches = ns.branches
|
|
216
|
+
|
|
217
|
+
if ns.index_branches is None:
|
|
218
|
+
ns.index_branches = DEFAULT_INDEX_BRANCHES
|
|
219
|
+
if ns.scan_branches is None:
|
|
220
|
+
ns.scan_branches = DEFAULT_SCAN_BRANCHES
|
|
221
|
+
|
|
222
|
+
return CliArgs(
|
|
223
|
+
url=ns.url,
|
|
224
|
+
token=ns.token,
|
|
225
|
+
proxy=ns.proxy,
|
|
226
|
+
log_file=ns.log_file,
|
|
227
|
+
log_level=ns.log_level,
|
|
228
|
+
debug=ns.debug,
|
|
229
|
+
mode=ns.mode,
|
|
230
|
+
batch_size=ns.batch_size,
|
|
231
|
+
dump_projects=ns.dump_projects,
|
|
232
|
+
index_file=ns.index_file,
|
|
233
|
+
index_branches=ns.index_branches,
|
|
234
|
+
branches_per_page=ns.branches_per_page,
|
|
235
|
+
scan_branches=ns.scan_branches,
|
|
236
|
+
forks=ns.forks,
|
|
237
|
+
fork_diff_bases=ns.fork_diff_bases,
|
|
238
|
+
search=ns.search,
|
|
239
|
+
terms_file=ns.terms_file,
|
|
240
|
+
session=ns.session,
|
|
241
|
+
session_file=ns.session_file,
|
|
242
|
+
resume=ns.resume,
|
|
243
|
+
input_file=ns.input_file,
|
|
244
|
+
output_file=ns.output_file,
|
|
245
|
+
sqlite_path=ns.sqlite_path,
|
|
246
|
+
hash_algo=ns.hash_algo,
|
|
247
|
+
normalize_hits=not ns.no_normalize_hits,
|
|
248
|
+
sort_keys=ns.sort_keys,
|
|
249
|
+
)
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
class GitlabHarvesterError(Exception):
|
|
2
|
+
"""
|
|
3
|
+
Base exception for GitlabHarvester-specific errors.
|
|
4
|
+
|
|
5
|
+
Raised to indicate incorrect usage or unrecoverable conditions
|
|
6
|
+
within the harvester workflow, such as attempting to access the
|
|
7
|
+
GitLab API before client initialization.
|
|
8
|
+
"""
|
|
9
|
+
pass
|