gitlab-harvester 1.2.2__tar.gz → 1.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/PKG-INFO +6 -3
  2. {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/README.md +5 -2
  3. {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/gitlab_harvester.egg-info/PKG-INFO +6 -3
  4. {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/gitlab_harvester.egg-info/SOURCES.txt +3 -0
  5. {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/cli.py +72 -2
  6. gitlab_harvester-1.3.0/glh/downloads.py +292 -0
  7. gitlab_harvester-1.3.0/glh/filefilter.py +138 -0
  8. {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/harvester.py +115 -23
  9. {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/modes.py +97 -72
  10. gitlab_harvester-1.3.0/glh/output.py +198 -0
  11. {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/planner.py +12 -0
  12. gitlab_harvester-1.3.0/glh/postprocess/split.py +79 -0
  13. {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/session.py +22 -0
  14. {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/pyproject.toml +1 -1
  15. gitlab_harvester-1.2.2/glh/postprocess/split.py +0 -123
  16. {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/LICENSE +0 -0
  17. {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/gitlab_harvester.egg-info/dependency_links.txt +0 -0
  18. {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/gitlab_harvester.egg-info/entry_points.txt +0 -0
  19. {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/gitlab_harvester.egg-info/requires.txt +0 -0
  20. {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/gitlab_harvester.egg-info/top_level.txt +0 -0
  21. {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/gitlab_harvester.py +0 -0
  22. {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/__init__.py +0 -0
  23. {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/config.py +0 -0
  24. {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/exceptions.py +0 -0
  25. {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/postprocess/__init__.py +0 -0
  26. {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/postprocess/convert.py +0 -0
  27. {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/postprocess/dedup.py +0 -0
  28. {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/postprocess/pipeline.py +0 -0
  29. {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/ratelimit.py +0 -0
  30. {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/tokens.py +0 -0
  31. {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/user_agents.py +0 -0
  32. {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/utils/__init__.py +0 -0
  33. {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/utils/logging_utils.py +0 -0
  34. {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/utils/terminal_utils.py +0 -0
  35. {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/workers.py +0 -0
  36. {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: gitlab-harvester
3
- Version: 1.2.2
3
+ Version: 1.3.0
4
4
  Summary: Build a GitLab instance project index and search repositories for sensitive keywords (API-only, no cloning).
5
5
  Author-email: Cur1iosity <cur1iosity@protonmail.com>
6
6
  License-Expression: MIT
@@ -90,8 +90,9 @@ gitlab-harvester -u https://gitlab.example.com -t "$GITLAB_TOKEN" -s password
90
90
  The first run builds a project index (`<host>_project_index.jsonl`), and later runs reuse it.
91
91
  Hits are written to a JSONL session file as they are found.
92
92
 
93
- Lists of terms, all branches and forks, fresh findings only, resuming, parallel requests
94
- and rate limits, deduplicated reports, config files:
93
+ Lists of terms, specific files only, saving the files with hits, all branches and forks,
94
+ fresh findings only, resuming, parallel requests and rate limits, deduplicated reports,
95
+ config files:
95
96
  **[docs/USE_CASES.md](docs/USE_CASES.md)**.
96
97
 
97
98
  ## Common options
@@ -102,6 +103,8 @@ and rate limits, deduplicated reports, config files:
102
103
  | `--forks include\|skip\|branch-diff\|all-branches` | how forks are handled |
103
104
  | `--time-after DATE`, `--time-before DATE` | limit the index, its projects and the files with hits to a time window |
104
105
  | `--started-from 2024-06-01` | search only indexed projects with activity since the date |
106
+ | `--filename '*.env'`, `--extension py` | search only files with this name (exact, or a mask with `*`) or extension |
107
+ | `--download-files`, `--download-max-size 10M` | save every file with hits to `<term>/files/` for offline analysis |
105
108
  | `-t "tok1;tok2"`, `--token-file FILE` | several tokens run in parallel, each with its own workers and rate-limit budget |
106
109
  | `-w, --workers N` | parallel API requests per token, for instances without rate limits |
107
110
  | `--rpm N`, `--index-rpm N`, `--search-rpm N` | requests per minute per worker: all, non-search, search |
@@ -52,8 +52,9 @@ gitlab-harvester -u https://gitlab.example.com -t "$GITLAB_TOKEN" -s password
52
52
  The first run builds a project index (`<host>_project_index.jsonl`), and later runs reuse it.
53
53
  Hits are written to a JSONL session file as they are found.
54
54
 
55
- Lists of terms, all branches and forks, fresh findings only, resuming, parallel requests
56
- and rate limits, deduplicated reports, config files:
55
+ Lists of terms, specific files only, saving the files with hits, all branches and forks,
56
+ fresh findings only, resuming, parallel requests and rate limits, deduplicated reports,
57
+ config files:
57
58
  **[docs/USE_CASES.md](docs/USE_CASES.md)**.
58
59
 
59
60
  ## Common options
@@ -64,6 +65,8 @@ and rate limits, deduplicated reports, config files:
64
65
  | `--forks include\|skip\|branch-diff\|all-branches` | how forks are handled |
65
66
  | `--time-after DATE`, `--time-before DATE` | limit the index, its projects and the files with hits to a time window |
66
67
  | `--started-from 2024-06-01` | search only indexed projects with activity since the date |
68
+ | `--filename '*.env'`, `--extension py` | search only files with this name (exact, or a mask with `*`) or extension |
69
+ | `--download-files`, `--download-max-size 10M` | save every file with hits to `<term>/files/` for offline analysis |
67
70
  | `-t "tok1;tok2"`, `--token-file FILE` | several tokens run in parallel, each with its own workers and rate-limit budget |
68
71
  | `-w, --workers N` | parallel API requests per token, for instances without rate limits |
69
72
  | `--rpm N`, `--index-rpm N`, `--search-rpm N` | requests per minute per worker: all, non-search, search |
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: gitlab-harvester
3
- Version: 1.2.2
3
+ Version: 1.3.0
4
4
  Summary: Build a GitLab instance project index and search repositories for sensitive keywords (API-only, no cloning).
5
5
  Author-email: Cur1iosity <cur1iosity@protonmail.com>
6
6
  License-Expression: MIT
@@ -90,8 +90,9 @@ gitlab-harvester -u https://gitlab.example.com -t "$GITLAB_TOKEN" -s password
90
90
  The first run builds a project index (`<host>_project_index.jsonl`), and later runs reuse it.
91
91
  Hits are written to a JSONL session file as they are found.
92
92
 
93
- Lists of terms, all branches and forks, fresh findings only, resuming, parallel requests
94
- and rate limits, deduplicated reports, config files:
93
+ Lists of terms, specific files only, saving the files with hits, all branches and forks,
94
+ fresh findings only, resuming, parallel requests and rate limits, deduplicated reports,
95
+ config files:
95
96
  **[docs/USE_CASES.md](docs/USE_CASES.md)**.
96
97
 
97
98
  ## Common options
@@ -102,6 +103,8 @@ and rate limits, deduplicated reports, config files:
102
103
  | `--forks include\|skip\|branch-diff\|all-branches` | how forks are handled |
103
104
  | `--time-after DATE`, `--time-before DATE` | limit the index, its projects and the files with hits to a time window |
104
105
  | `--started-from 2024-06-01` | search only indexed projects with activity since the date |
106
+ | `--filename '*.env'`, `--extension py` | search only files with this name (exact, or a mask with `*`) or extension |
107
+ | `--download-files`, `--download-max-size 10M` | save every file with hits to `<term>/files/` for offline analysis |
105
108
  | `-t "tok1;tok2"`, `--token-file FILE` | several tokens run in parallel, each with its own workers and rate-limit budget |
106
109
  | `-w, --workers N` | parallel API requests per token, for instances without rate limits |
107
110
  | `--rpm N`, `--index-rpm N`, `--search-rpm N` | requests per minute per worker: all, non-search, search |
@@ -11,9 +11,12 @@ gitlab_harvester.egg-info/top_level.txt
11
11
  glh/__init__.py
12
12
  glh/cli.py
13
13
  glh/config.py
14
+ glh/downloads.py
14
15
  glh/exceptions.py
16
+ glh/filefilter.py
15
17
  glh/harvester.py
16
18
  glh/modes.py
19
+ glh/output.py
17
20
  glh/planner.py
18
21
  glh/ratelimit.py
19
22
  glh/session.py
@@ -17,7 +17,7 @@ from dataclasses import dataclass
17
17
  from datetime import datetime
18
18
  from importlib.metadata import version, PackageNotFoundError
19
19
 
20
- from glh import ratelimit, user_agents
20
+ from glh import downloads, filefilter, ratelimit, user_agents
21
21
  from glh.config import TERMS_DEST, ConfigError, load_config
22
22
  from glh.planner import parse_timestamp
23
23
  from glh.postprocess.pipeline import normalize_steps
@@ -134,6 +134,34 @@ def parse_user_agent(value: str) -> str:
134
134
  raise argparse.ArgumentTypeError(str(e)) from None
135
135
 
136
136
 
137
+ def parse_filename(value: str) -> str:
138
+ """Parse --filename: a file name or a mask with '*' (no spaces)."""
139
+ try:
140
+ return filefilter.normalize_filename(value)
141
+ except filefilter.FileFilterError as e:
142
+ raise argparse.ArgumentTypeError(str(e)) from None
143
+
144
+
145
+ def parse_extension(value: str) -> str:
146
+ """Parse --extension: 'py', '.py' and '*.py' all mean 'py'."""
147
+ try:
148
+ return filefilter.normalize_extension(value)
149
+ except filefilter.FileFilterError as e:
150
+ raise argparse.ArgumentTypeError(str(e)) from None
151
+
152
+
153
+ def parse_size(value: str) -> int:
154
+ """Parse --download-max-size: bytes or with k/M/G (500k, 10M, 1G); 0 = no limit."""
155
+ try:
156
+ return downloads.parse_size(value)
157
+ except ValueError:
158
+ raise argparse.ArgumentTypeError(
159
+ "Use bytes or a size with k, M or G, e.g. 500k, 10M or 1G (0 = no limit).") from None
160
+
161
+
162
+ DEFAULT_DOWNLOAD_MAX_SIZE = 10 * 1024 * 1024
163
+
164
+
137
165
  # (specific dest, group dest) for every request rate limit; resolved like TIME_BOUNDS.
138
166
  RATE_LIMITS: tuple[tuple[str, str], ...] = (
139
167
  ("index_rpm", "rpm"),
@@ -218,10 +246,14 @@ class CliArgs:
218
246
  search: str | None
219
247
  terms_file: str | None
220
248
  terms: tuple[str, ...]
249
+ filename: str | None
250
+ extension: str | None
221
251
  session: str | None
222
252
  session_file: str | None
223
253
  resume: bool
224
254
  output_separated: bool
255
+ download_files: bool
256
+ download_max_size: int
225
257
 
226
258
  input_file: str | None
227
259
  output_file: str | None
@@ -306,7 +338,7 @@ class CliParser:
306
338
  parser.add_argument(
307
339
  "--index-rpm", type=parse_rpm, default=None, metavar="N",
308
340
  help="Max API requests per minute per worker, except search requests: project and branch "
309
- "listing, file-change checks (default 0 = no limit).",
341
+ "listing, file-change checks, file downloads (default 0 = no limit).",
310
342
  )
311
343
  parser.add_argument(
312
344
  "--search-rpm", type=parse_rpm, default=None, metavar="N",
@@ -401,6 +433,24 @@ class CliParser:
401
433
  group.add_argument("-s", "--search", default=None, help="Single search term.")
402
434
  group.add_argument("-f", "--terms-file", default=None, help="Path to file with one term per line.")
403
435
 
436
+ ff = parser.add_argument_group(
437
+ "file filters",
438
+ "Search the terms only in some files (case-insensitive; together, a file must match both).\n"
439
+ "GitLab applies them on its side (filename:/extension: in the search string) and the\n"
440
+ "harvester re-checks every result. They narrow a search: a term (-s/-f) is still needed.",
441
+ )
442
+ ff.add_argument(
443
+ "--filename", type=parse_filename, default=None, metavar="PATTERN",
444
+ help="Only files with this name. PATTERN is an exact name, which may include the extension "
445
+ "(settings.py, .env, Dockerfile), or a mask where * matches any characters (*.env, "
446
+ "config*, docker-compose*.yml, *secret*). A PATTERN with / matches the end of the "
447
+ "path (config/*.yml).",
448
+ )
449
+ ff.add_argument(
450
+ "--extension", type=parse_extension, default=None, metavar="EXT",
451
+ help="Only files with this extension; the dot is optional: py, .py and *.py are the same.",
452
+ )
453
+
404
454
  sess = parser.add_mutually_exclusive_group()
405
455
  sess.add_argument("--session", default=None, help="Session name (creates <name>.jsonl in the current directory).")
406
456
  sess.add_argument("--session-file", default=None, help="Explicit session file path (JSONL).")
@@ -415,6 +465,22 @@ class CliParser:
415
465
  help="With several search terms, also write each term's hits to its own file "
416
466
  "<session>.<term>.jsonl next to the session (ignored for a single term).",
417
467
  )
468
+ parser.add_argument(
469
+ "--download-files",
470
+ action=argparse.BooleanOptionalAction,
471
+ default=False,
472
+ help="Download every file with hits into a 'files' directory next to the session "
473
+ "(results/files by default), one sub-directory per term: "
474
+ "files/<term>/<project>_<branch>_<timestamp>_<path>.",
475
+ )
476
+ parser.add_argument(
477
+ "--download-max-size",
478
+ type=parse_size,
479
+ default=DEFAULT_DOWNLOAD_MAX_SIZE,
480
+ metavar="SIZE",
481
+ help="With --download-files: skip files larger than SIZE: bytes or with k, M, G "
482
+ "(500k, 10M, 1G; 0 = no limit). Default: 10M.",
483
+ )
418
484
 
419
485
  # Postprocess
420
486
  parser.add_argument(
@@ -593,10 +659,14 @@ class CliParser:
593
659
  search=ns.search,
594
660
  terms_file=ns.terms_file,
595
661
  terms=tuple(ns.terms),
662
+ filename=ns.filename,
663
+ extension=ns.extension,
596
664
  session=ns.session,
597
665
  session_file=ns.session_file,
598
666
  resume=ns.resume,
599
667
  output_separated=ns.output_separated,
668
+ download_files=ns.download_files,
669
+ download_max_size=ns.download_max_size,
600
670
  input_file=ns.input_file,
601
671
  output_file=ns.output_file,
602
672
  post_process=tuple(ns.post_process),
@@ -0,0 +1,292 @@
1
+ # glh/downloads.py
2
+ """
3
+ Download the files with hits (``--download-files``).
4
+
5
+ Every file that produced a hit is saved once per (term, project, branch, path)
6
+ into the term's ``files`` directory, ``<dir>/<term>/files/`` (see
7
+ ``glh.output.OutputLayout``; ``results/<term>/files/`` with the default session
8
+ location), created on the first download. The name follows the other
9
+ GitlabHarvester outputs (``<host>_session_<timestamp>.jsonl``):
10
+
11
+ <project>_<branch>_<timestamp>_<term>_<path>
12
+
13
+ e.g. ``results/password/files/acme_backend_api-service_main_20261117093000_password_config_settings.py``.
14
+ ``/`` in the project and the file path become ``_``, so the name ends with the
15
+ original file name and extension. The timestamp is the run's start (UTC), the
16
+ same as in a default session name. Unsafe characters become ``_`` (safe on
17
+ Windows, macOS and Linux, a name can't escape its directory), long parts are
18
+ shortened (the file path keeps its end), and a name that is taken gets a
19
+ ``~2``, ``~3`` … suffix before the extension.
20
+
21
+ Files are streamed to ``<name>.part`` and renamed when complete, so memory stays
22
+ constant and an interrupted download never looks like a finished file.
23
+ ``--download-max-size`` skips larger files: by ``Content-Length`` before any
24
+ data is read, else as soon as the stream passes the limit.
25
+
26
+ The same file hit by another term is copied from the earlier download instead
27
+ of being requested again. That cache and the taken names grow with the number
28
+ of downloaded files (like the file-change cache), not with the instance size.
29
+
30
+ The downloader doesn't talk to GitLab itself: the engine passes a ``fetch``
31
+ callable that opens the streamed response, so this module only deals with
32
+ names, sizes and the file system.
33
+ """
34
+
35
+ from __future__ import annotations
36
+
37
+ import re
38
+ import shutil
39
+ import threading
40
+ from dataclasses import dataclass
41
+ from pathlib import Path
42
+ from typing import Any, Callable, Iterable, Protocol
43
+
44
+ # Like the term names (glh/output.py): everything else becomes '_'.
45
+ _UNSAFE = re.compile(r"[^A-Za-z0-9._-]+")
46
+ # Part lengths; the whole name stays well below the 255-char limit of most file systems.
47
+ MAX_PROJECT_LEN = 50
48
+ MAX_BRANCH_LEN = 30
49
+ MAX_TERM_LEN = 30
50
+ MAX_PATH_LEN = 70
51
+ CHUNK_SIZE = 64 * 1024
52
+ PART_SUFFIX = ".part"
53
+
54
+ _SIZE = re.compile(r"^\s*(\d+(?:\.\d+)?)\s*([kmgt]?)(?:i?b)?\s*$", re.IGNORECASE)
55
+ _UNITS = {"": 1, "k": 1024, "m": 1024 ** 2, "g": 1024 ** 3, "t": 1024 ** 4}
56
+
57
+
58
+ def parse_size(value: str) -> int:
59
+ """
60
+ Parse a size: bytes or with a binary unit k/M/G/T (optional B/iB, any case):
61
+ '1048576', '500k', '10M', '1.5G', '10MB', '10MiB'. 0 means no limit.
62
+
63
+ Raises:
64
+ ValueError: If the value is not a size.
65
+ """
66
+ m = _SIZE.match(value or "")
67
+ if not m:
68
+ raise ValueError(f"invalid size {value!r}")
69
+ return int(float(m.group(1)) * _UNITS[m.group(2).lower()])
70
+
71
+
72
+ def format_size(size: int) -> str:
73
+ """'512 B', '500 KiB', '10 MiB', '1.5 GiB'."""
74
+ value = float(size)
75
+ for unit in ("B", "KiB", "MiB", "GiB"):
76
+ if value < 1024 or unit == "GiB":
77
+ text = f"{value:.1f}".rstrip("0").rstrip(".")
78
+ return f"{text} {unit}"
79
+ value /= 1024
80
+ return f"{size} B" # pragma: no cover
81
+
82
+
83
+ def _safe(text: str, limit: int, *, keep_end: bool = False, last: bool = False) -> str:
84
+ """
85
+ Filesystem-safe name part, at most ``limit`` chars (from the end with keep_end).
86
+
87
+ Dots and '_' are trimmed from the ends, except a leading dot of the ``last``
88
+ part (the file path, so ``.env`` stays ``.env``): only the name's end must
89
+ not be a dot (Windows drops trailing dots).
90
+ """
91
+ safe = _UNSAFE.sub("_", str(text).strip())
92
+ safe = safe.lstrip("_").rstrip("._") if last else safe.strip("._")
93
+ if len(safe) > limit:
94
+ safe = safe[-limit:].lstrip("_" if last else "._") if keep_end else safe[:limit].rstrip("._")
95
+ return safe or "_"
96
+
97
+
98
+ def file_name(project: str, branch: str, stamp: str, term: str, path: str) -> str:
99
+ """Base name of a downloaded file (before de-duplication), see the module doc."""
100
+ return "_".join((
101
+ _safe(project.replace("/", "_"), MAX_PROJECT_LEN, keep_end=True),
102
+ _safe(branch, MAX_BRANCH_LEN),
103
+ _safe(stamp, 20),
104
+ _safe(term, MAX_TERM_LEN),
105
+ _safe(path.replace("/", "_"), MAX_PATH_LEN, keep_end=True, last=True),
106
+ ))
107
+
108
+
109
+ class FileTooLarge(Exception):
110
+ """The file is larger than --download-max-size."""
111
+
112
+
113
+ class Layout(Protocol):
114
+ """Where the files of a term go (glh.output.OutputLayout)."""
115
+
116
+ def files_dir(self, term: str) -> Path: ...
117
+
118
+ def describe_files(self) -> str: ...
119
+
120
+
121
+ class Response(Protocol):
122
+ """What a download needs from a streamed HTTP response (requests.Response)."""
123
+
124
+ headers: Any
125
+
126
+ def iter_content(self, chunk_size: int) -> Iterable[bytes]: ...
127
+
128
+ def close(self) -> None: ...
129
+
130
+
131
+ @dataclass(slots=True)
132
+ class DownloadStats:
133
+ """Counters of a run's downloads."""
134
+
135
+ saved: int = 0 # downloaded from GitLab
136
+ copied: int = 0 # same file for another term: copied locally
137
+ too_large: int = 0 # skipped by --download-max-size
138
+ failed: int = 0 # API or file system error
139
+ bytes: int = 0 # downloaded bytes (copies not counted)
140
+
141
+ @property
142
+ def files(self) -> int:
143
+ return self.saved + self.copied
144
+
145
+
146
+ class FileDownloader:
147
+ """
148
+ Saves files with hits into the term directories; thread-safe (search workers call it).
149
+
150
+ Args:
151
+ layout: Gives the directory of each term's files (``files_dir``),
152
+ created on the first download.
153
+ stamp: Run timestamp for the names (``%Y%m%d%H%M%S``, UTC).
154
+ max_size: Skip files larger than this many bytes; 0 = no limit.
155
+ """
156
+
157
+ def __init__(self, layout: Layout, stamp: str, *, max_size: int = 0) -> None:
158
+ self.layout = layout
159
+ self.stamp = stamp
160
+ self.max_size = max(0, int(max_size))
161
+ self.stats = DownloadStats()
162
+ self._lock = threading.Lock()
163
+ # Lower-case full paths (Windows/macOS file names are case-insensitive).
164
+ self._taken: set[str] = set()
165
+ self._saved: dict[tuple[Any, str, str], Path] = {} # (project id, branch, path) -> first download
166
+
167
+ def describe(self) -> str:
168
+ """'download files to results/<term>/files (max 10 MiB each)' for the search plan."""
169
+ limit = f"max {format_size(self.max_size)} each" if self.max_size else "no size limit"
170
+ return f"download files to {self.layout.describe_files()} ({limit})"
171
+
172
+ def summary(self) -> str:
173
+ """'3 files (12 KiB) downloaded to results/<term>/files, 1 skipped: larger than 10 MiB'."""
174
+ s = self.stats
175
+ text = (f"{s.files} file{'s' if s.files != 1 else ''} ({format_size(s.bytes)}) "
176
+ f"downloaded to {self.layout.describe_files()}")
177
+ if s.copied:
178
+ text += f" ({s.copied} copied locally: same file, another term)"
179
+ if s.too_large:
180
+ text += f", {s.too_large} skipped: larger than {format_size(self.max_size)}"
181
+ if s.failed:
182
+ text += f", {s.failed} FAILED (see the warnings)"
183
+ return text
184
+
185
+ @staticmethod
186
+ def _key(target: Path) -> str:
187
+ return str(target).lower()
188
+
189
+ def _reserve(self, folder: Path, base: str) -> Path:
190
+ """A free path for ``base`` in ``folder`` (lock held): adds ~2, ~3 … before the extension if taken."""
191
+ stem, dot, ext = base.rpartition(".")
192
+ if not stem or stem.endswith("_"): # no extension, or a dot file like ".env"
193
+ stem, dot, ext = base, "", ""
194
+ candidate, n = folder / base, 1
195
+ while self._key(candidate) in self._taken or candidate.exists():
196
+ n += 1
197
+ candidate = folder / f"{stem}~{n}{dot}{ext}"
198
+ self._taken.add(self._key(candidate))
199
+ return candidate
200
+
201
+ def _release(self, target: Path) -> None:
202
+ with self._lock:
203
+ self._taken.discard(self._key(target))
204
+
205
+ def failed(self) -> None:
206
+ """Count a download the caller couldn't make (API error)."""
207
+ with self._lock:
208
+ self.stats.failed += 1
209
+
210
+ def save(
211
+ self,
212
+ *,
213
+ project_id: Any,
214
+ project_path: str,
215
+ branch: str,
216
+ term: str,
217
+ path: str,
218
+ fetch: Callable[[], Response],
219
+ ) -> Path | None:
220
+ """
221
+ Save one file; ``fetch`` opens the streamed raw-file response.
222
+
223
+ Returns:
224
+ The local path, or None if the file is larger than ``max_size``.
225
+
226
+ Raises:
227
+ Whatever ``fetch`` or reading the stream raises (API/network
228
+ errors), and OSError for file system errors; nothing is left on
229
+ disk then and the caller should call ``failed()``.
230
+ """
231
+ key = (project_id, branch, path)
232
+ base = file_name(project_path, branch, self.stamp, term, path)
233
+ folder = self.layout.files_dir(term)
234
+ with self._lock:
235
+ earlier = self._saved.get(key)
236
+ target = self._reserve(folder, base)
237
+ folder.mkdir(parents=True, exist_ok=True)
238
+
239
+ if earlier is not None and earlier.is_file():
240
+ try:
241
+ shutil.copyfile(earlier, target)
242
+ except OSError:
243
+ self._release(target)
244
+ target.unlink(missing_ok=True)
245
+ raise
246
+ with self._lock:
247
+ self.stats.copied += 1
248
+ return target
249
+
250
+ part = target.with_name(target.name + PART_SUFFIX)
251
+ written = 0
252
+ try:
253
+ response = fetch()
254
+ try:
255
+ length = _content_length(response.headers)
256
+ if self.max_size and length is not None and length > self.max_size:
257
+ raise FileTooLarge(length)
258
+ with part.open("wb") as fp:
259
+ for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
260
+ if not chunk:
261
+ continue
262
+ written += len(chunk)
263
+ if self.max_size and written > self.max_size:
264
+ raise FileTooLarge(None)
265
+ fp.write(chunk)
266
+ finally:
267
+ response.close()
268
+ part.replace(target)
269
+ except FileTooLarge:
270
+ part.unlink(missing_ok=True)
271
+ self._release(target)
272
+ with self._lock:
273
+ self.stats.too_large += 1
274
+ return None
275
+ except BaseException:
276
+ part.unlink(missing_ok=True)
277
+ self._release(target)
278
+ raise
279
+
280
+ with self._lock:
281
+ self.stats.saved += 1
282
+ self.stats.bytes += written
283
+ self._saved.setdefault(key, target)
284
+ return target
285
+
286
+
287
+ def _content_length(headers: Any) -> int | None:
288
+ try:
289
+ value = headers.get("Content-Length") if headers is not None else None
290
+ return int(value) if value is not None else None
291
+ except (TypeError, ValueError):
292
+ return None
@@ -0,0 +1,138 @@
1
+ # glh/filefilter.py
2
+ """
3
+ File filters for the search: ``--filename`` and ``--extension``.
4
+
5
+ GitLab code search (``scope=blobs``, basic and advanced search) understands
6
+ filters written into the search string itself: ``password filename:*.env``
7
+ searches "password" only in files whose name matches ``*.env``. There is no
8
+ separate API parameter, so the harvester appends the filters to every term in
9
+ the request (``FileFilter.query``); the term stored in the session stays as
10
+ the user wrote it.
11
+
12
+ The filters are also re-checked on the client (``FileFilter.matches``) against
13
+ the ``path`` of every result, for two reasons:
14
+
15
+ - an instance or GitLab version that ignores a filter still yields only the
16
+ files asked for;
17
+ - GitLab's basic search is looser than the option promises: its ``filename:``
18
+ matches the *end of the whole path* (``filename:config.yml`` also returns
19
+ ``app/myconfig.yml``), while ``--filename`` matches the file name exactly.
20
+
21
+ Matching (both filters case-insensitive, ``*`` is the only wildcard and
22
+ matches any characters):
23
+
24
+ - ``--filename PATTERN``: the file name (basename), e.g. ``settings.py``,
25
+ ``*.env``, ``config*``, ``docker-compose*.yml``. A pattern with ``/``
26
+ matches the end of the path instead (``config/*.yml``).
27
+ - ``--extension EXT``: the file name ends with ``.EXT``. Leading ``*``/``.``
28
+ and trailing dots are dropped, so ``.py``, ``py`` and ``*.py`` are the same.
29
+ """
30
+
31
+ from __future__ import annotations
32
+
33
+ import re
34
+ from dataclasses import dataclass
35
+ from typing import Any
36
+
37
+
38
+ class FileFilterError(ValueError):
39
+ """An invalid --filename / --extension value."""
40
+
41
+
42
+ def _check_no_spaces(value: str, option: str) -> None:
43
+ # GitLab reads a filter value up to the next whitespace: a space would turn
44
+ # the rest of the value into a search term.
45
+ if any(ch.isspace() for ch in value):
46
+ raise FileFilterError(f"{option} must not contain spaces: {value!r}")
47
+
48
+
49
+ def normalize_filename(value: str) -> str:
50
+ """
51
+ Validate a --filename pattern: no spaces, not empty; a leading '/' is
52
+ dropped (GitLab paths have none).
53
+ """
54
+ v = value.strip().lstrip("/")
55
+ if not v or not v.strip("*"):
56
+ raise FileFilterError("--filename needs a file name or a mask, e.g. settings.py or *.env")
57
+ _check_no_spaces(v, "--filename")
58
+ return v
59
+
60
+
61
+ def normalize_extension(value: str) -> str:
62
+ """
63
+ Validate an --extension value and drop the extra dots: '.py', 'py', '..py',
64
+ '*.py' and 'py.' all become 'py'. Inner dots stay ('tar.gz').
65
+ """
66
+ v = value.strip().lstrip("*.").rstrip(".")
67
+ if not v:
68
+ raise FileFilterError("--extension needs an extension, e.g. py or .py")
69
+ _check_no_spaces(v, "--extension")
70
+ if "/" in v:
71
+ raise FileFilterError(f"--extension must not contain '/': {value!r}")
72
+ return v
73
+
74
+
75
+ def _wildcard(pattern: str) -> str:
76
+ """Regex for a pattern where only '*' is special."""
77
+ return ".*".join(re.escape(part) for part in pattern.split("*"))
78
+
79
+
80
+ @dataclass(frozen=True, slots=True)
81
+ class FileFilter:
82
+ """
83
+ The file filters of a search; empty (falsy) when neither is set.
84
+
85
+ Values are normalized on construction (``normalize_filename``,
86
+ ``normalize_extension``), so library callers get the same behavior as the
87
+ CLI (``extension=".py"`` means ``py``); invalid values raise FileFilterError.
88
+ """
89
+
90
+ filename: str | None = None
91
+ extension: str | None = None
92
+
93
+ def __post_init__(self) -> None:
94
+ # Frozen dataclass: normalize through object.__setattr__.
95
+ if self.filename is not None:
96
+ object.__setattr__(self, "filename", normalize_filename(self.filename))
97
+ if self.extension is not None:
98
+ object.__setattr__(self, "extension", normalize_extension(self.extension))
99
+
100
+ def __bool__(self) -> bool:
101
+ return bool(self.filename or self.extension)
102
+
103
+ def query(self, term: str) -> str:
104
+ """The search string sent to GitLab: the term plus the filters."""
105
+ parts = [term]
106
+ if self.filename:
107
+ parts.append(f"filename:{self.filename}")
108
+ if self.extension:
109
+ parts.append(f"extension:{self.extension}")
110
+ return " ".join(parts)
111
+
112
+ def matches(self, path: str) -> bool:
113
+ """True if the file at ``path`` (repository path, '/'-separated) passes the filters."""
114
+ name = path.rsplit("/", 1)[-1]
115
+ if self.filename:
116
+ if "/" in self.filename:
117
+ ok = re.fullmatch(f"(?:.*/)?{_wildcard(self.filename)}", path, re.IGNORECASE)
118
+ else:
119
+ ok = re.fullmatch(_wildcard(self.filename), name, re.IGNORECASE)
120
+ if not ok:
121
+ return False
122
+ if self.extension:
123
+ if not re.fullmatch(rf".*\.{_wildcard(self.extension)}", name, re.IGNORECASE):
124
+ return False
125
+ return True
126
+
127
+ def describe(self) -> str:
128
+ """'filename *.env, extension py' (for the search plan and the run journal)."""
129
+ parts = []
130
+ if self.filename:
131
+ parts.append(f"filename {self.filename}")
132
+ if self.extension:
133
+ parts.append(f"extension {self.extension}")
134
+ return ", ".join(parts)
135
+
136
+ def to_meta(self) -> dict[str, Any]:
137
+ """Session meta fields."""
138
+ return {"filename": self.filename, "extension": self.extension}