gitlab-harvester 0.2.8__tar.gz → 0.2.9__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. {gitlab_harvester-0.2.8 → gitlab_harvester-0.2.9}/PKG-INFO +1 -1
  2. {gitlab_harvester-0.2.8 → gitlab_harvester-0.2.9}/gitlab_harvester.egg-info/PKG-INFO +1 -1
  3. {gitlab_harvester-0.2.8 → gitlab_harvester-0.2.9}/glh/cli.py +4 -1
  4. {gitlab_harvester-0.2.8 → gitlab_harvester-0.2.9}/glh/harvester.py +11 -5
  5. {gitlab_harvester-0.2.8 → gitlab_harvester-0.2.9}/glh/modes.py +1 -0
  6. {gitlab_harvester-0.2.8 → gitlab_harvester-0.2.9}/glh/utils/terminal_utils.py +31 -2
  7. {gitlab_harvester-0.2.8 → gitlab_harvester-0.2.9}/pyproject.toml +1 -1
  8. {gitlab_harvester-0.2.8 → gitlab_harvester-0.2.9}/LICENSE +0 -0
  9. {gitlab_harvester-0.2.8 → gitlab_harvester-0.2.9}/README.md +0 -0
  10. {gitlab_harvester-0.2.8 → gitlab_harvester-0.2.9}/gitlab_harvester.egg-info/SOURCES.txt +0 -0
  11. {gitlab_harvester-0.2.8 → gitlab_harvester-0.2.9}/gitlab_harvester.egg-info/dependency_links.txt +0 -0
  12. {gitlab_harvester-0.2.8 → gitlab_harvester-0.2.9}/gitlab_harvester.egg-info/entry_points.txt +0 -0
  13. {gitlab_harvester-0.2.8 → gitlab_harvester-0.2.9}/gitlab_harvester.egg-info/requires.txt +0 -0
  14. {gitlab_harvester-0.2.8 → gitlab_harvester-0.2.9}/gitlab_harvester.egg-info/top_level.txt +0 -0
  15. {gitlab_harvester-0.2.8 → gitlab_harvester-0.2.9}/gitlab_harvester.py +0 -0
  16. {gitlab_harvester-0.2.8 → gitlab_harvester-0.2.9}/glh/__init__.py +0 -0
  17. {gitlab_harvester-0.2.8 → gitlab_harvester-0.2.9}/glh/exceptions.py +0 -0
  18. {gitlab_harvester-0.2.8 → gitlab_harvester-0.2.9}/glh/planner.py +0 -0
  19. {gitlab_harvester-0.2.8 → gitlab_harvester-0.2.9}/glh/postprocess/__init__.py +0 -0
  20. {gitlab_harvester-0.2.8 → gitlab_harvester-0.2.9}/glh/postprocess/convert.py +0 -0
  21. {gitlab_harvester-0.2.8 → gitlab_harvester-0.2.9}/glh/postprocess/dedup.py +0 -0
  22. {gitlab_harvester-0.2.8 → gitlab_harvester-0.2.9}/glh/session.py +0 -0
  23. {gitlab_harvester-0.2.8 → gitlab_harvester-0.2.9}/glh/utils/__init__.py +0 -0
  24. {gitlab_harvester-0.2.8 → gitlab_harvester-0.2.9}/glh/utils/logging_utils.py +0 -0
  25. {gitlab_harvester-0.2.8 → gitlab_harvester-0.2.9}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: gitlab-harvester
3
- Version: 0.2.8
3
+ Version: 0.2.9
4
4
  Summary: Build a GitLab instance project index and search repositories for sensitive keywords (API-only, no cloning).
5
5
  Author-email: Cur1iosity <cur1iosity@protonmail.com>
6
6
  License: MIT
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: gitlab-harvester
3
- Version: 0.2.8
3
+ Version: 0.2.9
4
4
  Summary: Build a GitLab instance project index and search repositories for sensitive keywords (API-only, no cloning).
5
5
  Author-email: Cur1iosity <cur1iosity@protonmail.com>
6
6
  License: MIT
@@ -72,6 +72,7 @@ class CliArgs:
72
72
  url: str | None
73
73
  token: str | None
74
74
  proxy: str | None
75
+ timeout: int
75
76
 
76
77
  log_file: str | None
77
78
  log_level: str
@@ -127,10 +128,11 @@ class CliParser:
127
128
  )
128
129
  parser.add_argument("-t", "--token", default=None, help="GitLab token (required for search/dump-index).",)
129
130
  parser.add_argument("-p", "--proxy", default=None, help="HTTP(S) proxy URL for GitLab API traffic.",)
131
+ parser.add_argument("--timeout", default=60, type=int, help="Timeout in seconds for GitLab API request.",)
130
132
 
131
133
  # Logging
132
134
  parser.add_argument("--log-file", default=None, help="Write logs to a file (in addition to stderr).")
133
- parser.add_argument("--log-level", choices=("ERROR", "WARN", "INFO"), default="INFO", help="Console log level.")
135
+ parser.add_argument("--log-level", choices=("ERROR", "WARN", "INFO"), default="WARN", help="Console log level.")
134
136
  parser.add_argument("--debug", action="store_true", help="Enable debug logging (overrides --log-level).")
135
137
 
136
138
  # Mode
@@ -223,6 +225,7 @@ class CliParser:
223
225
  url=ns.url,
224
226
  token=ns.token,
225
227
  proxy=ns.proxy,
228
+ timeout=ns.timeout,
226
229
  log_file=ns.log_file,
227
230
  log_level=ns.log_level,
228
231
  debug=ns.debug,
@@ -20,6 +20,7 @@ from tqdm import tqdm
20
20
  from glh.exceptions import GitlabHarvesterError
21
21
  from glh.planner import ScanOptions, branches_to_scan, should_scan_project
22
22
  from glh.utils import logging_utils, terminal_utils
23
+ from glh.utils.terminal_utils import human_int
23
24
 
24
25
 
25
26
  def client_required(func: Callable) -> Callable:
@@ -65,6 +66,7 @@ class GitlabHarvester:
65
66
  token: str | None = None,
66
67
  search_terms: list | None = None,
67
68
  proxy: str | None = None,
69
+ timeout: int = 60,
68
70
  *,
69
71
  log_level: int | str = logging.WARNING,
70
72
  log_file: str | None = None,
@@ -81,7 +83,7 @@ class GitlabHarvester:
81
83
  )
82
84
 
83
85
  url = url or host
84
- self._gl: Gitlab | None = self._get_gl_client(url=url, token=token, proxy=proxy) if url and token else None
86
+ self._gl: Gitlab | None = self._get_gl_client(url=url, token=token, proxy=proxy, timeout=timeout) if url and token else None
85
87
 
86
88
  @staticmethod
87
89
  def _emit_record(fp: IO, record: dict[str, Any], indent: int | None = None) -> None:
@@ -239,7 +241,7 @@ class GitlabHarvester:
239
241
  raise
240
242
 
241
243
  # Fallback: retry once with ssl verification disabled
242
- self.logger.warning("TLS failed for %s; retrying with ssl_verify=False", url)
244
+ self.logger.info("TLS failed for %s; retrying with ssl_verify=False", url)
243
245
  kwargs["ssl_verify"] = False
244
246
  _maybe_disable_warnings()
245
247
 
@@ -849,7 +851,7 @@ class GitlabHarvester:
849
851
  terminal_utils.set_desc(pbar, raw_desc, lay)
850
852
 
851
853
  if ext_pbar is not None:
852
- terminal_utils.set_desc(ext_pbar, f"Searching term: '{term}' | Hit Counter: {hit_counter}", lay)
854
+ terminal_utils.set_desc(ext_pbar, f"'{term}' • hits: {human_int(hit_counter)}", lay)
853
855
 
854
856
  if not should_scan_project(pr, options):
855
857
  pbar.update(1)
@@ -895,6 +897,7 @@ class GitlabHarvester:
895
897
  Scan a single project for a term across the selected branches.
896
898
  """
897
899
  res: list[dict[str, Any]] = []
900
+ orig_desc = ''
898
901
 
899
902
  project_obj: Project = self._gl.projects.get(project["id"], lazy=True)
900
903
  project_url: str = project["web_url"]
@@ -906,10 +909,13 @@ class GitlabHarvester:
906
909
  br_len: int = len(branches)
907
910
  lay = terminal_utils.layout()
908
911
 
912
+ if ext_pbar is not None:
913
+ orig_desc = ext_pbar.desc
914
+
909
915
  for num, b in enumerate(branches):
910
916
  if ext_pbar is not None:
911
- postfix = b if br_len == 1 else f"{b} [{num + 1}/{br_len}]"
912
- terminal_utils.set_postfix(ext_pbar, postfix, lay)
917
+ branch_info = b if br_len == 1 else f"{b} [{num + 1}/{br_len}]"
918
+ terminal_utils.set_desc(ext_pbar, f"{orig_desc} • {branch_info}", lay)
913
919
 
914
920
  try:
915
921
  s_res, hit_counter = self.scan_branch(
@@ -97,6 +97,7 @@ def _mk_harvester(args: CliArgs) -> GitlabHarvester:
97
97
  kwargs: dict[str, Any] = {
98
98
  "url": url,
99
99
  "token": token,
100
+ "timeout": args.timeout,
100
101
  "log_level": args.log_level,
101
102
  "log_file": args.log_file,
102
103
  }
@@ -30,7 +30,9 @@ def calc_tqdm_widths(*, cols: int) -> tuple[int, int]:
30
30
  Compute (desc_width, postfix_width) budgets for tqdm.
31
31
  Goal: keep progress bar wide and stable.
32
32
  """
33
- desc_w = max(18, min(42, int(cols * 0.28))) # ~28% of terminal
33
+ # desc_w = max(18, min(42, int(cols * 0.28))) # ~28% of terminal
34
+ # desc_w = max(24, min(80, int(cols * 0.5)))
35
+ desc_w = max(30, min(120, int(cols * 0.7)))
34
36
  post_w = max(12, min(28, int(cols * 0.18))) # ~18% of terminal
35
37
  return desc_w, post_w
36
38
 
@@ -71,9 +73,9 @@ def bar_format_stable(lay: TqdmLayout) -> str:
71
73
  """
72
74
  return (
73
75
  f"{{desc:<{lay.desc_w}}} "
74
- f"[{{elapsed}}<{{remaining}}]"
75
76
  f"{{percentage:3.0f}}% "
76
77
  f"{{bar}} "
78
+ f"[{{elapsed}}<{{remaining}}]"
77
79
  f"[{{n_fmt}}/{{total_fmt}}] "
78
80
  f"{{postfix}}"
79
81
  )
@@ -142,3 +144,30 @@ def set_desc(pbar: tqdm, text: str, lay: TqdmLayout) -> None:
142
144
  def set_postfix(pbar: tqdm, text: str, lay: TqdmLayout) -> None:
143
145
  """Set shortened postfix respecting computed layout width."""
144
146
  pbar.set_postfix_str(shorten(text, lay.post_w), refresh=True)
147
+
148
+
149
+ def human_int(n: int) -> str:
150
+ """
151
+ Format integer into compact human-readable form.
152
+
153
+ Examples:
154
+ 532 -> "532"
155
+ 1200 -> "1.2K"
156
+ 532000 -> "532K"
157
+ 1200000 -> "1.2M"
158
+ 3400000000 -> "3.4B"
159
+ """
160
+ if n < 1000:
161
+ return str(n)
162
+
163
+ for unit, div in (
164
+ ("K", 1_000),
165
+ ("M", 1_000_000),
166
+ ("B", 1_000_000_000),
167
+ ("T", 1_000_000_000_000),
168
+ ):
169
+ if n < div * 1000:
170
+ val = n / div
171
+ return f"{val:.1f}{unit}" if val < 10 else f"{val:.0f}{unit}"
172
+
173
+ return f"{n:.0e}"
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "gitlab-harvester"
7
- version = "0.2.8"
7
+ version = "0.2.9"
8
8
  description = "Build a GitLab instance project index and search repositories for sensitive keywords (API-only, no cloning)."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"