gitlab-harvester 0.2.7__tar.gz → 0.2.9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gitlab_harvester-0.2.7 → gitlab_harvester-0.2.9}/PKG-INFO +4 -4
- {gitlab_harvester-0.2.7 → gitlab_harvester-0.2.9}/README.md +3 -3
- {gitlab_harvester-0.2.7 → gitlab_harvester-0.2.9}/gitlab_harvester.egg-info/PKG-INFO +4 -4
- {gitlab_harvester-0.2.7 → gitlab_harvester-0.2.9}/glh/cli.py +4 -1
- {gitlab_harvester-0.2.7 → gitlab_harvester-0.2.9}/glh/harvester.py +11 -5
- {gitlab_harvester-0.2.7 → gitlab_harvester-0.2.9}/glh/modes.py +1 -0
- {gitlab_harvester-0.2.7 → gitlab_harvester-0.2.9}/glh/utils/terminal_utils.py +31 -2
- {gitlab_harvester-0.2.7 → gitlab_harvester-0.2.9}/pyproject.toml +1 -1
- {gitlab_harvester-0.2.7 → gitlab_harvester-0.2.9}/LICENSE +0 -0
- {gitlab_harvester-0.2.7 → gitlab_harvester-0.2.9}/gitlab_harvester.egg-info/SOURCES.txt +0 -0
- {gitlab_harvester-0.2.7 → gitlab_harvester-0.2.9}/gitlab_harvester.egg-info/dependency_links.txt +0 -0
- {gitlab_harvester-0.2.7 → gitlab_harvester-0.2.9}/gitlab_harvester.egg-info/entry_points.txt +0 -0
- {gitlab_harvester-0.2.7 → gitlab_harvester-0.2.9}/gitlab_harvester.egg-info/requires.txt +0 -0
- {gitlab_harvester-0.2.7 → gitlab_harvester-0.2.9}/gitlab_harvester.egg-info/top_level.txt +0 -0
- {gitlab_harvester-0.2.7 → gitlab_harvester-0.2.9}/gitlab_harvester.py +0 -0
- {gitlab_harvester-0.2.7 → gitlab_harvester-0.2.9}/glh/__init__.py +0 -0
- {gitlab_harvester-0.2.7 → gitlab_harvester-0.2.9}/glh/exceptions.py +0 -0
- {gitlab_harvester-0.2.7 → gitlab_harvester-0.2.9}/glh/planner.py +0 -0
- {gitlab_harvester-0.2.7 → gitlab_harvester-0.2.9}/glh/postprocess/__init__.py +0 -0
- {gitlab_harvester-0.2.7 → gitlab_harvester-0.2.9}/glh/postprocess/convert.py +0 -0
- {gitlab_harvester-0.2.7 → gitlab_harvester-0.2.9}/glh/postprocess/dedup.py +0 -0
- {gitlab_harvester-0.2.7 → gitlab_harvester-0.2.9}/glh/session.py +0 -0
- {gitlab_harvester-0.2.7 → gitlab_harvester-0.2.9}/glh/utils/__init__.py +0 -0
- {gitlab_harvester-0.2.7 → gitlab_harvester-0.2.9}/glh/utils/logging_utils.py +0 -0
- {gitlab_harvester-0.2.7 → gitlab_harvester-0.2.9}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gitlab-harvester
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.9
|
|
4
4
|
Summary: Build a GitLab instance project index and search repositories for sensitive keywords (API-only, no cloning).
|
|
5
5
|
Author-email: Cur1iosity <cur1iosity@protonmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -132,7 +132,7 @@ pipx install gitlab-harvester
|
|
|
132
132
|
Run:
|
|
133
133
|
|
|
134
134
|
```bash
|
|
135
|
-
gitlab-harvester --
|
|
135
|
+
gitlab-harvester --help
|
|
136
136
|
```
|
|
137
137
|
|
|
138
138
|
---
|
|
@@ -264,8 +264,8 @@ gitlab-harvester -m dedup \
|
|
|
264
264
|
Options:
|
|
265
265
|
|
|
266
266
|
* `--sqlite-path file.sqlite`
|
|
267
|
-
* `--
|
|
268
|
-
* `--no-normalize-
|
|
267
|
+
* `--hash-algo blake2b|sha1|sha256`
|
|
268
|
+
* `--no-normalize-hits`
|
|
269
269
|
|
|
270
270
|
---
|
|
271
271
|
|
|
@@ -98,7 +98,7 @@ pipx install gitlab-harvester
|
|
|
98
98
|
Run:
|
|
99
99
|
|
|
100
100
|
```bash
|
|
101
|
-
gitlab-harvester --
|
|
101
|
+
gitlab-harvester --help
|
|
102
102
|
```
|
|
103
103
|
|
|
104
104
|
---
|
|
@@ -230,8 +230,8 @@ gitlab-harvester -m dedup \
|
|
|
230
230
|
Options:
|
|
231
231
|
|
|
232
232
|
* `--sqlite-path file.sqlite`
|
|
233
|
-
* `--
|
|
234
|
-
* `--no-normalize-
|
|
233
|
+
* `--hash-algo blake2b|sha1|sha256`
|
|
234
|
+
* `--no-normalize-hits`
|
|
235
235
|
|
|
236
236
|
---
|
|
237
237
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gitlab-harvester
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.9
|
|
4
4
|
Summary: Build a GitLab instance project index and search repositories for sensitive keywords (API-only, no cloning).
|
|
5
5
|
Author-email: Cur1iosity <cur1iosity@protonmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -132,7 +132,7 @@ pipx install gitlab-harvester
|
|
|
132
132
|
Run:
|
|
133
133
|
|
|
134
134
|
```bash
|
|
135
|
-
gitlab-harvester --
|
|
135
|
+
gitlab-harvester --help
|
|
136
136
|
```
|
|
137
137
|
|
|
138
138
|
---
|
|
@@ -264,8 +264,8 @@ gitlab-harvester -m dedup \
|
|
|
264
264
|
Options:
|
|
265
265
|
|
|
266
266
|
* `--sqlite-path file.sqlite`
|
|
267
|
-
* `--
|
|
268
|
-
* `--no-normalize-
|
|
267
|
+
* `--hash-algo blake2b|sha1|sha256`
|
|
268
|
+
* `--no-normalize-hits`
|
|
269
269
|
|
|
270
270
|
---
|
|
271
271
|
|
|
@@ -72,6 +72,7 @@ class CliArgs:
|
|
|
72
72
|
url: str | None
|
|
73
73
|
token: str | None
|
|
74
74
|
proxy: str | None
|
|
75
|
+
timeout: int
|
|
75
76
|
|
|
76
77
|
log_file: str | None
|
|
77
78
|
log_level: str
|
|
@@ -127,10 +128,11 @@ class CliParser:
|
|
|
127
128
|
)
|
|
128
129
|
parser.add_argument("-t", "--token", default=None, help="GitLab token (required for search/dump-index).",)
|
|
129
130
|
parser.add_argument("-p", "--proxy", default=None, help="HTTP(S) proxy URL for GitLab API traffic.",)
|
|
131
|
+
parser.add_argument("--timeout", default=60, type=int, help="Timeout in seconds for GitLab API request.",)
|
|
130
132
|
|
|
131
133
|
# Logging
|
|
132
134
|
parser.add_argument("--log-file", default=None, help="Write logs to a file (in addition to stderr).")
|
|
133
|
-
parser.add_argument("--log-level", choices=("ERROR", "WARN", "INFO"), default="
|
|
135
|
+
parser.add_argument("--log-level", choices=("ERROR", "WARN", "INFO"), default="WARN", help="Console log level.")
|
|
134
136
|
parser.add_argument("--debug", action="store_true", help="Enable debug logging (overrides --log-level).")
|
|
135
137
|
|
|
136
138
|
# Mode
|
|
@@ -223,6 +225,7 @@ class CliParser:
|
|
|
223
225
|
url=ns.url,
|
|
224
226
|
token=ns.token,
|
|
225
227
|
proxy=ns.proxy,
|
|
228
|
+
timeout=ns.timeout,
|
|
226
229
|
log_file=ns.log_file,
|
|
227
230
|
log_level=ns.log_level,
|
|
228
231
|
debug=ns.debug,
|
|
@@ -20,6 +20,7 @@ from tqdm import tqdm
|
|
|
20
20
|
from glh.exceptions import GitlabHarvesterError
|
|
21
21
|
from glh.planner import ScanOptions, branches_to_scan, should_scan_project
|
|
22
22
|
from glh.utils import logging_utils, terminal_utils
|
|
23
|
+
from glh.utils.terminal_utils import human_int
|
|
23
24
|
|
|
24
25
|
|
|
25
26
|
def client_required(func: Callable) -> Callable:
|
|
@@ -65,6 +66,7 @@ class GitlabHarvester:
|
|
|
65
66
|
token: str | None = None,
|
|
66
67
|
search_terms: list | None = None,
|
|
67
68
|
proxy: str | None = None,
|
|
69
|
+
timeout: int = 60,
|
|
68
70
|
*,
|
|
69
71
|
log_level: int | str = logging.WARNING,
|
|
70
72
|
log_file: str | None = None,
|
|
@@ -81,7 +83,7 @@ class GitlabHarvester:
|
|
|
81
83
|
)
|
|
82
84
|
|
|
83
85
|
url = url or host
|
|
84
|
-
self._gl: Gitlab | None = self._get_gl_client(url=url, token=token, proxy=proxy) if url and token else None
|
|
86
|
+
self._gl: Gitlab | None = self._get_gl_client(url=url, token=token, proxy=proxy, timeout=timeout) if url and token else None
|
|
85
87
|
|
|
86
88
|
@staticmethod
|
|
87
89
|
def _emit_record(fp: IO, record: dict[str, Any], indent: int | None = None) -> None:
|
|
@@ -239,7 +241,7 @@ class GitlabHarvester:
|
|
|
239
241
|
raise
|
|
240
242
|
|
|
241
243
|
# Fallback: retry once with ssl verification disabled
|
|
242
|
-
self.logger.
|
|
244
|
+
self.logger.info("TLS failed for %s; retrying with ssl_verify=False", url)
|
|
243
245
|
kwargs["ssl_verify"] = False
|
|
244
246
|
_maybe_disable_warnings()
|
|
245
247
|
|
|
@@ -849,7 +851,7 @@ class GitlabHarvester:
|
|
|
849
851
|
terminal_utils.set_desc(pbar, raw_desc, lay)
|
|
850
852
|
|
|
851
853
|
if ext_pbar is not None:
|
|
852
|
-
terminal_utils.set_desc(ext_pbar, f"
|
|
854
|
+
terminal_utils.set_desc(ext_pbar, f"'{term}' • hits: {human_int(hit_counter)}", lay)
|
|
853
855
|
|
|
854
856
|
if not should_scan_project(pr, options):
|
|
855
857
|
pbar.update(1)
|
|
@@ -895,6 +897,7 @@ class GitlabHarvester:
|
|
|
895
897
|
Scan a single project for a term across the selected branches.
|
|
896
898
|
"""
|
|
897
899
|
res: list[dict[str, Any]] = []
|
|
900
|
+
orig_desc = ''
|
|
898
901
|
|
|
899
902
|
project_obj: Project = self._gl.projects.get(project["id"], lazy=True)
|
|
900
903
|
project_url: str = project["web_url"]
|
|
@@ -906,10 +909,13 @@ class GitlabHarvester:
|
|
|
906
909
|
br_len: int = len(branches)
|
|
907
910
|
lay = terminal_utils.layout()
|
|
908
911
|
|
|
912
|
+
if ext_pbar is not None:
|
|
913
|
+
orig_desc = ext_pbar.desc
|
|
914
|
+
|
|
909
915
|
for num, b in enumerate(branches):
|
|
910
916
|
if ext_pbar is not None:
|
|
911
|
-
|
|
912
|
-
terminal_utils.
|
|
917
|
+
branch_info = b if br_len == 1 else f"{b} [{num + 1}/{br_len}]"
|
|
918
|
+
terminal_utils.set_desc(ext_pbar, f"{orig_desc} • {branch_info}", lay)
|
|
913
919
|
|
|
914
920
|
try:
|
|
915
921
|
s_res, hit_counter = self.scan_branch(
|
|
@@ -30,7 +30,9 @@ def calc_tqdm_widths(*, cols: int) -> tuple[int, int]:
|
|
|
30
30
|
Compute (desc_width, postfix_width) budgets for tqdm.
|
|
31
31
|
Goal: keep progress bar wide and stable.
|
|
32
32
|
"""
|
|
33
|
-
desc_w = max(18, min(42, int(cols * 0.28))) # ~28% of terminal
|
|
33
|
+
# desc_w = max(18, min(42, int(cols * 0.28))) # ~28% of terminal
|
|
34
|
+
# desc_w = max(24, min(80, int(cols * 0.5)))
|
|
35
|
+
desc_w = max(30, min(120, int(cols * 0.7)))
|
|
34
36
|
post_w = max(12, min(28, int(cols * 0.18))) # ~18% of terminal
|
|
35
37
|
return desc_w, post_w
|
|
36
38
|
|
|
@@ -71,9 +73,9 @@ def bar_format_stable(lay: TqdmLayout) -> str:
|
|
|
71
73
|
"""
|
|
72
74
|
return (
|
|
73
75
|
f"{{desc:<{lay.desc_w}}} "
|
|
74
|
-
f"[{{elapsed}}<{{remaining}}]"
|
|
75
76
|
f"{{percentage:3.0f}}% "
|
|
76
77
|
f"{{bar}} "
|
|
78
|
+
f"[{{elapsed}}<{{remaining}}]"
|
|
77
79
|
f"[{{n_fmt}}/{{total_fmt}}] "
|
|
78
80
|
f"{{postfix}}"
|
|
79
81
|
)
|
|
@@ -142,3 +144,30 @@ def set_desc(pbar: tqdm, text: str, lay: TqdmLayout) -> None:
|
|
|
142
144
|
def set_postfix(pbar: tqdm, text: str, lay: TqdmLayout) -> None:
|
|
143
145
|
"""Set shortened postfix respecting computed layout width."""
|
|
144
146
|
pbar.set_postfix_str(shorten(text, lay.post_w), refresh=True)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def human_int(n: int) -> str:
|
|
150
|
+
"""
|
|
151
|
+
Format integer into compact human-readable form.
|
|
152
|
+
|
|
153
|
+
Examples:
|
|
154
|
+
532 -> "532"
|
|
155
|
+
1200 -> "1.2K"
|
|
156
|
+
532000 -> "532K"
|
|
157
|
+
1200000 -> "1.2M"
|
|
158
|
+
3400000000 -> "3.4B"
|
|
159
|
+
"""
|
|
160
|
+
if n < 1000:
|
|
161
|
+
return str(n)
|
|
162
|
+
|
|
163
|
+
for unit, div in (
|
|
164
|
+
("K", 1_000),
|
|
165
|
+
("M", 1_000_000),
|
|
166
|
+
("B", 1_000_000_000),
|
|
167
|
+
("T", 1_000_000_000_000),
|
|
168
|
+
):
|
|
169
|
+
if n < div * 1000:
|
|
170
|
+
val = n / div
|
|
171
|
+
return f"{val:.1f}{unit}" if val < 10 else f"{val:.0f}{unit}"
|
|
172
|
+
|
|
173
|
+
return f"{n:.0e}"
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "gitlab-harvester"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.9"
|
|
8
8
|
description = "Build a GitLab instance project index and search repositories for sensitive keywords (API-only, no cloning)."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
File without changes
|
|
File without changes
|
{gitlab_harvester-0.2.7 → gitlab_harvester-0.2.9}/gitlab_harvester.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
{gitlab_harvester-0.2.7 → gitlab_harvester-0.2.9}/gitlab_harvester.egg-info/entry_points.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|