gitlab-harvester 1.2.2__tar.gz → 1.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/PKG-INFO +6 -3
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/README.md +5 -2
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/gitlab_harvester.egg-info/PKG-INFO +6 -3
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/gitlab_harvester.egg-info/SOURCES.txt +3 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/cli.py +72 -2
- gitlab_harvester-1.3.0/glh/downloads.py +292 -0
- gitlab_harvester-1.3.0/glh/filefilter.py +138 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/harvester.py +115 -23
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/modes.py +97 -72
- gitlab_harvester-1.3.0/glh/output.py +198 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/planner.py +12 -0
- gitlab_harvester-1.3.0/glh/postprocess/split.py +79 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/session.py +22 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/pyproject.toml +1 -1
- gitlab_harvester-1.2.2/glh/postprocess/split.py +0 -123
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/LICENSE +0 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/gitlab_harvester.egg-info/dependency_links.txt +0 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/gitlab_harvester.egg-info/entry_points.txt +0 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/gitlab_harvester.egg-info/requires.txt +0 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/gitlab_harvester.egg-info/top_level.txt +0 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/gitlab_harvester.py +0 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/__init__.py +0 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/config.py +0 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/exceptions.py +0 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/postprocess/__init__.py +0 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/postprocess/convert.py +0 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/postprocess/dedup.py +0 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/postprocess/pipeline.py +0 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/ratelimit.py +0 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/tokens.py +0 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/user_agents.py +0 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/utils/__init__.py +0 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/utils/logging_utils.py +0 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/utils/terminal_utils.py +0 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/glh/workers.py +0 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.3.0}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gitlab-harvester
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.3.0
|
|
4
4
|
Summary: Build a GitLab instance project index and search repositories for sensitive keywords (API-only, no cloning).
|
|
5
5
|
Author-email: Cur1iosity <cur1iosity@protonmail.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -90,8 +90,9 @@ gitlab-harvester -u https://gitlab.example.com -t "$GITLAB_TOKEN" -s password
|
|
|
90
90
|
The first run builds a project index (`<host>_project_index.jsonl`), and later runs reuse it.
|
|
91
91
|
Hits are written to a JSONL session file as they are found.
|
|
92
92
|
|
|
93
|
-
Lists of terms,
|
|
94
|
-
and rate limits, deduplicated reports,
|
|
93
|
+
Lists of terms, specific files only, saving the files with hits, all branches and forks,
|
|
94
|
+
fresh findings only, resuming, parallel requests and rate limits, deduplicated reports,
|
|
95
|
+
config files:
|
|
95
96
|
**[docs/USE_CASES.md](docs/USE_CASES.md)**.
|
|
96
97
|
|
|
97
98
|
## Common options
|
|
@@ -102,6 +103,8 @@ and rate limits, deduplicated reports, config files:
|
|
|
102
103
|
| `--forks include\|skip\|branch-diff\|all-branches` | how forks are handled |
|
|
103
104
|
| `--time-after DATE`, `--time-before DATE` | limit the index, its projects and the files with hits to a time window |
|
|
104
105
|
| `--started-from 2024-06-01` | search only indexed projects with activity since the date |
|
|
106
|
+
| `--filename '*.env'`, `--extension py` | search only files with this name (exact, or a mask with `*`) or extension |
|
|
107
|
+
| `--download-files`, `--download-max-size 10M` | save every file with hits to `<term>/files/` for offline analysis |
|
|
105
108
|
| `-t "tok1;tok2"`, `--token-file FILE` | several tokens run in parallel, each with its own workers and rate-limit budget |
|
|
106
109
|
| `-w, --workers N` | parallel API requests per token, for instances without rate limits |
|
|
107
110
|
| `--rpm N`, `--index-rpm N`, `--search-rpm N` | requests per minute per worker: all, non-search, search |
|
|
@@ -52,8 +52,9 @@ gitlab-harvester -u https://gitlab.example.com -t "$GITLAB_TOKEN" -s password
|
|
|
52
52
|
The first run builds a project index (`<host>_project_index.jsonl`), and later runs reuse it.
|
|
53
53
|
Hits are written to a JSONL session file as they are found.
|
|
54
54
|
|
|
55
|
-
Lists of terms,
|
|
56
|
-
and rate limits, deduplicated reports,
|
|
55
|
+
Lists of terms, specific files only, saving the files with hits, all branches and forks,
|
|
56
|
+
fresh findings only, resuming, parallel requests and rate limits, deduplicated reports,
|
|
57
|
+
config files:
|
|
57
58
|
**[docs/USE_CASES.md](docs/USE_CASES.md)**.
|
|
58
59
|
|
|
59
60
|
## Common options
|
|
@@ -64,6 +65,8 @@ and rate limits, deduplicated reports, config files:
|
|
|
64
65
|
| `--forks include\|skip\|branch-diff\|all-branches` | how forks are handled |
|
|
65
66
|
| `--time-after DATE`, `--time-before DATE` | limit the index, its projects and the files with hits to a time window |
|
|
66
67
|
| `--started-from 2024-06-01` | search only indexed projects with activity since the date |
|
|
68
|
+
| `--filename '*.env'`, `--extension py` | search only files with this name (exact, or a mask with `*`) or extension |
|
|
69
|
+
| `--download-files`, `--download-max-size 10M` | save every file with hits to `<term>/files/` for offline analysis |
|
|
67
70
|
| `-t "tok1;tok2"`, `--token-file FILE` | several tokens run in parallel, each with its own workers and rate-limit budget |
|
|
68
71
|
| `-w, --workers N` | parallel API requests per token, for instances without rate limits |
|
|
69
72
|
| `--rpm N`, `--index-rpm N`, `--search-rpm N` | requests per minute per worker: all, non-search, search |
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gitlab-harvester
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.3.0
|
|
4
4
|
Summary: Build a GitLab instance project index and search repositories for sensitive keywords (API-only, no cloning).
|
|
5
5
|
Author-email: Cur1iosity <cur1iosity@protonmail.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -90,8 +90,9 @@ gitlab-harvester -u https://gitlab.example.com -t "$GITLAB_TOKEN" -s password
|
|
|
90
90
|
The first run builds a project index (`<host>_project_index.jsonl`), and later runs reuse it.
|
|
91
91
|
Hits are written to a JSONL session file as they are found.
|
|
92
92
|
|
|
93
|
-
Lists of terms,
|
|
94
|
-
and rate limits, deduplicated reports,
|
|
93
|
+
Lists of terms, specific files only, saving the files with hits, all branches and forks,
|
|
94
|
+
fresh findings only, resuming, parallel requests and rate limits, deduplicated reports,
|
|
95
|
+
config files:
|
|
95
96
|
**[docs/USE_CASES.md](docs/USE_CASES.md)**.
|
|
96
97
|
|
|
97
98
|
## Common options
|
|
@@ -102,6 +103,8 @@ and rate limits, deduplicated reports, config files:
|
|
|
102
103
|
| `--forks include\|skip\|branch-diff\|all-branches` | how forks are handled |
|
|
103
104
|
| `--time-after DATE`, `--time-before DATE` | limit the index, its projects and the files with hits to a time window |
|
|
104
105
|
| `--started-from 2024-06-01` | search only indexed projects with activity since the date |
|
|
106
|
+
| `--filename '*.env'`, `--extension py` | search only files with this name (exact, or a mask with `*`) or extension |
|
|
107
|
+
| `--download-files`, `--download-max-size 10M` | save every file with hits to `<term>/files/` for offline analysis |
|
|
105
108
|
| `-t "tok1;tok2"`, `--token-file FILE` | several tokens run in parallel, each with its own workers and rate-limit budget |
|
|
106
109
|
| `-w, --workers N` | parallel API requests per token, for instances without rate limits |
|
|
107
110
|
| `--rpm N`, `--index-rpm N`, `--search-rpm N` | requests per minute per worker: all, non-search, search |
|
|
@@ -11,9 +11,12 @@ gitlab_harvester.egg-info/top_level.txt
|
|
|
11
11
|
glh/__init__.py
|
|
12
12
|
glh/cli.py
|
|
13
13
|
glh/config.py
|
|
14
|
+
glh/downloads.py
|
|
14
15
|
glh/exceptions.py
|
|
16
|
+
glh/filefilter.py
|
|
15
17
|
glh/harvester.py
|
|
16
18
|
glh/modes.py
|
|
19
|
+
glh/output.py
|
|
17
20
|
glh/planner.py
|
|
18
21
|
glh/ratelimit.py
|
|
19
22
|
glh/session.py
|
|
@@ -17,7 +17,7 @@ from dataclasses import dataclass
|
|
|
17
17
|
from datetime import datetime
|
|
18
18
|
from importlib.metadata import version, PackageNotFoundError
|
|
19
19
|
|
|
20
|
-
from glh import ratelimit, user_agents
|
|
20
|
+
from glh import downloads, filefilter, ratelimit, user_agents
|
|
21
21
|
from glh.config import TERMS_DEST, ConfigError, load_config
|
|
22
22
|
from glh.planner import parse_timestamp
|
|
23
23
|
from glh.postprocess.pipeline import normalize_steps
|
|
@@ -134,6 +134,34 @@ def parse_user_agent(value: str) -> str:
|
|
|
134
134
|
raise argparse.ArgumentTypeError(str(e)) from None
|
|
135
135
|
|
|
136
136
|
|
|
137
|
+
def parse_filename(value: str) -> str:
|
|
138
|
+
"""Parse --filename: a file name or a mask with '*' (no spaces)."""
|
|
139
|
+
try:
|
|
140
|
+
return filefilter.normalize_filename(value)
|
|
141
|
+
except filefilter.FileFilterError as e:
|
|
142
|
+
raise argparse.ArgumentTypeError(str(e)) from None
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def parse_extension(value: str) -> str:
|
|
146
|
+
"""Parse --extension: 'py', '.py' and '*.py' all mean 'py'."""
|
|
147
|
+
try:
|
|
148
|
+
return filefilter.normalize_extension(value)
|
|
149
|
+
except filefilter.FileFilterError as e:
|
|
150
|
+
raise argparse.ArgumentTypeError(str(e)) from None
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def parse_size(value: str) -> int:
|
|
154
|
+
"""Parse --download-max-size: bytes or with k/M/G (500k, 10M, 1G); 0 = no limit."""
|
|
155
|
+
try:
|
|
156
|
+
return downloads.parse_size(value)
|
|
157
|
+
except ValueError:
|
|
158
|
+
raise argparse.ArgumentTypeError(
|
|
159
|
+
"Use bytes or a size with k, M or G, e.g. 500k, 10M or 1G (0 = no limit).") from None
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
DEFAULT_DOWNLOAD_MAX_SIZE = 10 * 1024 * 1024
|
|
163
|
+
|
|
164
|
+
|
|
137
165
|
# (specific dest, group dest) for every request rate limit; resolved like TIME_BOUNDS.
|
|
138
166
|
RATE_LIMITS: tuple[tuple[str, str], ...] = (
|
|
139
167
|
("index_rpm", "rpm"),
|
|
@@ -218,10 +246,14 @@ class CliArgs:
|
|
|
218
246
|
search: str | None
|
|
219
247
|
terms_file: str | None
|
|
220
248
|
terms: tuple[str, ...]
|
|
249
|
+
filename: str | None
|
|
250
|
+
extension: str | None
|
|
221
251
|
session: str | None
|
|
222
252
|
session_file: str | None
|
|
223
253
|
resume: bool
|
|
224
254
|
output_separated: bool
|
|
255
|
+
download_files: bool
|
|
256
|
+
download_max_size: int
|
|
225
257
|
|
|
226
258
|
input_file: str | None
|
|
227
259
|
output_file: str | None
|
|
@@ -306,7 +338,7 @@ class CliParser:
|
|
|
306
338
|
parser.add_argument(
|
|
307
339
|
"--index-rpm", type=parse_rpm, default=None, metavar="N",
|
|
308
340
|
help="Max API requests per minute per worker, except search requests: project and branch "
|
|
309
|
-
"listing, file-change checks (default 0 = no limit).",
|
|
341
|
+
"listing, file-change checks, file downloads (default 0 = no limit).",
|
|
310
342
|
)
|
|
311
343
|
parser.add_argument(
|
|
312
344
|
"--search-rpm", type=parse_rpm, default=None, metavar="N",
|
|
@@ -401,6 +433,24 @@ class CliParser:
|
|
|
401
433
|
group.add_argument("-s", "--search", default=None, help="Single search term.")
|
|
402
434
|
group.add_argument("-f", "--terms-file", default=None, help="Path to file with one term per line.")
|
|
403
435
|
|
|
436
|
+
ff = parser.add_argument_group(
|
|
437
|
+
"file filters",
|
|
438
|
+
"Search the terms only in some files (case-insensitive; together, a file must match both).\n"
|
|
439
|
+
"GitLab applies them on its side (filename:/extension: in the search string) and the\n"
|
|
440
|
+
"harvester re-checks every result. They narrow a search: a term (-s/-f) is still needed.",
|
|
441
|
+
)
|
|
442
|
+
ff.add_argument(
|
|
443
|
+
"--filename", type=parse_filename, default=None, metavar="PATTERN",
|
|
444
|
+
help="Only files with this name. PATTERN is an exact name, which may include the extension "
|
|
445
|
+
"(settings.py, .env, Dockerfile), or a mask where * matches any characters (*.env, "
|
|
446
|
+
"config*, docker-compose*.yml, *secret*). A PATTERN with / matches the end of the "
|
|
447
|
+
"path (config/*.yml).",
|
|
448
|
+
)
|
|
449
|
+
ff.add_argument(
|
|
450
|
+
"--extension", type=parse_extension, default=None, metavar="EXT",
|
|
451
|
+
help="Only files with this extension; the dot is optional: py, .py and *.py are the same.",
|
|
452
|
+
)
|
|
453
|
+
|
|
404
454
|
sess = parser.add_mutually_exclusive_group()
|
|
405
455
|
sess.add_argument("--session", default=None, help="Session name (creates <name>.jsonl in the current directory).")
|
|
406
456
|
sess.add_argument("--session-file", default=None, help="Explicit session file path (JSONL).")
|
|
@@ -415,6 +465,22 @@ class CliParser:
|
|
|
415
465
|
help="With several search terms, also write each term's hits to its own file "
|
|
416
466
|
"<session>.<term>.jsonl next to the session (ignored for a single term).",
|
|
417
467
|
)
|
|
468
|
+
parser.add_argument(
|
|
469
|
+
"--download-files",
|
|
470
|
+
action=argparse.BooleanOptionalAction,
|
|
471
|
+
default=False,
|
|
472
|
+
help="Download every file with hits into a 'files' directory next to the session "
|
|
473
|
+
"(results/files by default), one sub-directory per term: "
|
|
474
|
+
"files/<term>/<project>_<branch>_<timestamp>_<path>.",
|
|
475
|
+
)
|
|
476
|
+
parser.add_argument(
|
|
477
|
+
"--download-max-size",
|
|
478
|
+
type=parse_size,
|
|
479
|
+
default=DEFAULT_DOWNLOAD_MAX_SIZE,
|
|
480
|
+
metavar="SIZE",
|
|
481
|
+
help="With --download-files: skip files larger than SIZE: bytes or with k, M, G "
|
|
482
|
+
"(500k, 10M, 1G; 0 = no limit). Default: 10M.",
|
|
483
|
+
)
|
|
418
484
|
|
|
419
485
|
# Postprocess
|
|
420
486
|
parser.add_argument(
|
|
@@ -593,10 +659,14 @@ class CliParser:
|
|
|
593
659
|
search=ns.search,
|
|
594
660
|
terms_file=ns.terms_file,
|
|
595
661
|
terms=tuple(ns.terms),
|
|
662
|
+
filename=ns.filename,
|
|
663
|
+
extension=ns.extension,
|
|
596
664
|
session=ns.session,
|
|
597
665
|
session_file=ns.session_file,
|
|
598
666
|
resume=ns.resume,
|
|
599
667
|
output_separated=ns.output_separated,
|
|
668
|
+
download_files=ns.download_files,
|
|
669
|
+
download_max_size=ns.download_max_size,
|
|
600
670
|
input_file=ns.input_file,
|
|
601
671
|
output_file=ns.output_file,
|
|
602
672
|
post_process=tuple(ns.post_process),
|
|
@@ -0,0 +1,292 @@
|
|
|
1
|
+
# glh/downloads.py
|
|
2
|
+
"""
|
|
3
|
+
Download the files with hits (``--download-files``).
|
|
4
|
+
|
|
5
|
+
Every file that produced a hit is saved once per (term, project, branch, path)
|
|
6
|
+
into the term's ``files`` directory, ``<dir>/<term>/files/`` (see
|
|
7
|
+
``glh.output.OutputLayout``; ``results/<term>/files/`` with the default session
|
|
8
|
+
location), created on the first download. The name follows the other
|
|
9
|
+
GitlabHarvester outputs (``<host>_session_<timestamp>.jsonl``):
|
|
10
|
+
|
|
11
|
+
<project>_<branch>_<timestamp>_<term>_<path>
|
|
12
|
+
|
|
13
|
+
e.g. ``results/password/files/acme_backend_api-service_main_20261117093000_password_config_settings.py``.
|
|
14
|
+
``/`` in the project and the file path become ``_``, so the name ends with the
|
|
15
|
+
original file name and extension. The timestamp is the run's start (UTC), the
|
|
16
|
+
same as in a default session name. Unsafe characters become ``_`` (safe on
|
|
17
|
+
Windows, macOS and Linux, a name can't escape its directory), long parts are
|
|
18
|
+
shortened (the file path keeps its end), and a name that is taken gets a
|
|
19
|
+
``~2``, ``~3`` … suffix before the extension.
|
|
20
|
+
|
|
21
|
+
Files are streamed to ``<name>.part`` and renamed when complete, so memory stays
|
|
22
|
+
constant and an interrupted download never looks like a finished file.
|
|
23
|
+
``--download-max-size`` skips larger files: by ``Content-Length`` before any
|
|
24
|
+
data is read, else as soon as the stream passes the limit.
|
|
25
|
+
|
|
26
|
+
The same file hit by another term is copied from the earlier download instead
|
|
27
|
+
of being requested again. That cache and the taken names grow with the number
|
|
28
|
+
of downloaded files (like the file-change cache), not with the instance size.
|
|
29
|
+
|
|
30
|
+
The downloader doesn't talk to GitLab itself: the engine passes a ``fetch``
|
|
31
|
+
callable that opens the streamed response, so this module only deals with
|
|
32
|
+
names, sizes and the file system.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
from __future__ import annotations
|
|
36
|
+
|
|
37
|
+
import re
|
|
38
|
+
import shutil
|
|
39
|
+
import threading
|
|
40
|
+
from dataclasses import dataclass
|
|
41
|
+
from pathlib import Path
|
|
42
|
+
from typing import Any, Callable, Iterable, Protocol
|
|
43
|
+
|
|
44
|
+
# Like the term names (glh/output.py): everything else becomes '_'.
|
|
45
|
+
_UNSAFE = re.compile(r"[^A-Za-z0-9._-]+")
|
|
46
|
+
# Part lengths; the whole name stays well below the 255-char limit of most file systems.
|
|
47
|
+
MAX_PROJECT_LEN = 50
|
|
48
|
+
MAX_BRANCH_LEN = 30
|
|
49
|
+
MAX_TERM_LEN = 30
|
|
50
|
+
MAX_PATH_LEN = 70
|
|
51
|
+
CHUNK_SIZE = 64 * 1024
|
|
52
|
+
PART_SUFFIX = ".part"
|
|
53
|
+
|
|
54
|
+
_SIZE = re.compile(r"^\s*(\d+(?:\.\d+)?)\s*([kmgt]?)(?:i?b)?\s*$", re.IGNORECASE)
|
|
55
|
+
_UNITS = {"": 1, "k": 1024, "m": 1024 ** 2, "g": 1024 ** 3, "t": 1024 ** 4}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def parse_size(value: str) -> int:
|
|
59
|
+
"""
|
|
60
|
+
Parse a size: bytes or with a binary unit k/M/G/T (optional B/iB, any case):
|
|
61
|
+
'1048576', '500k', '10M', '1.5G', '10MB', '10MiB'. 0 means no limit.
|
|
62
|
+
|
|
63
|
+
Raises:
|
|
64
|
+
ValueError: If the value is not a size.
|
|
65
|
+
"""
|
|
66
|
+
m = _SIZE.match(value or "")
|
|
67
|
+
if not m:
|
|
68
|
+
raise ValueError(f"invalid size {value!r}")
|
|
69
|
+
return int(float(m.group(1)) * _UNITS[m.group(2).lower()])
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def format_size(size: int) -> str:
|
|
73
|
+
"""'512 B', '500 KiB', '10 MiB', '1.5 GiB'."""
|
|
74
|
+
value = float(size)
|
|
75
|
+
for unit in ("B", "KiB", "MiB", "GiB"):
|
|
76
|
+
if value < 1024 or unit == "GiB":
|
|
77
|
+
text = f"{value:.1f}".rstrip("0").rstrip(".")
|
|
78
|
+
return f"{text} {unit}"
|
|
79
|
+
value /= 1024
|
|
80
|
+
return f"{size} B" # pragma: no cover
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _safe(text: str, limit: int, *, keep_end: bool = False, last: bool = False) -> str:
|
|
84
|
+
"""
|
|
85
|
+
Filesystem-safe name part, at most ``limit`` chars (from the end with keep_end).
|
|
86
|
+
|
|
87
|
+
Dots and '_' are trimmed from the ends, except a leading dot of the ``last``
|
|
88
|
+
part (the file path, so ``.env`` stays ``.env``): only the name's end must
|
|
89
|
+
not be a dot (Windows drops trailing dots).
|
|
90
|
+
"""
|
|
91
|
+
safe = _UNSAFE.sub("_", str(text).strip())
|
|
92
|
+
safe = safe.lstrip("_").rstrip("._") if last else safe.strip("._")
|
|
93
|
+
if len(safe) > limit:
|
|
94
|
+
safe = safe[-limit:].lstrip("_" if last else "._") if keep_end else safe[:limit].rstrip("._")
|
|
95
|
+
return safe or "_"
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def file_name(project: str, branch: str, stamp: str, term: str, path: str) -> str:
|
|
99
|
+
"""Base name of a downloaded file (before de-duplication), see the module doc."""
|
|
100
|
+
return "_".join((
|
|
101
|
+
_safe(project.replace("/", "_"), MAX_PROJECT_LEN, keep_end=True),
|
|
102
|
+
_safe(branch, MAX_BRANCH_LEN),
|
|
103
|
+
_safe(stamp, 20),
|
|
104
|
+
_safe(term, MAX_TERM_LEN),
|
|
105
|
+
_safe(path.replace("/", "_"), MAX_PATH_LEN, keep_end=True, last=True),
|
|
106
|
+
))
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
class FileTooLarge(Exception):
|
|
110
|
+
"""The file is larger than --download-max-size."""
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
class Layout(Protocol):
|
|
114
|
+
"""Where the files of a term go (glh.output.OutputLayout)."""
|
|
115
|
+
|
|
116
|
+
def files_dir(self, term: str) -> Path: ...
|
|
117
|
+
|
|
118
|
+
def describe_files(self) -> str: ...
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
class Response(Protocol):
|
|
122
|
+
"""What a download needs from a streamed HTTP response (requests.Response)."""
|
|
123
|
+
|
|
124
|
+
headers: Any
|
|
125
|
+
|
|
126
|
+
def iter_content(self, chunk_size: int) -> Iterable[bytes]: ...
|
|
127
|
+
|
|
128
|
+
def close(self) -> None: ...
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
@dataclass(slots=True)
|
|
132
|
+
class DownloadStats:
|
|
133
|
+
"""Counters of a run's downloads."""
|
|
134
|
+
|
|
135
|
+
saved: int = 0 # downloaded from GitLab
|
|
136
|
+
copied: int = 0 # same file for another term: copied locally
|
|
137
|
+
too_large: int = 0 # skipped by --download-max-size
|
|
138
|
+
failed: int = 0 # API or file system error
|
|
139
|
+
bytes: int = 0 # downloaded bytes (copies not counted)
|
|
140
|
+
|
|
141
|
+
@property
|
|
142
|
+
def files(self) -> int:
|
|
143
|
+
return self.saved + self.copied
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
class FileDownloader:
|
|
147
|
+
"""
|
|
148
|
+
Saves files with hits into the term directories; thread-safe (search workers call it).
|
|
149
|
+
|
|
150
|
+
Args:
|
|
151
|
+
layout: Gives the directory of each term's files (``files_dir``),
|
|
152
|
+
created on the first download.
|
|
153
|
+
stamp: Run timestamp for the names (``%Y%m%d%H%M%S``, UTC).
|
|
154
|
+
max_size: Skip files larger than this many bytes; 0 = no limit.
|
|
155
|
+
"""
|
|
156
|
+
|
|
157
|
+
def __init__(self, layout: Layout, stamp: str, *, max_size: int = 0) -> None:
|
|
158
|
+
self.layout = layout
|
|
159
|
+
self.stamp = stamp
|
|
160
|
+
self.max_size = max(0, int(max_size))
|
|
161
|
+
self.stats = DownloadStats()
|
|
162
|
+
self._lock = threading.Lock()
|
|
163
|
+
# Lower-case full paths (Windows/macOS file names are case-insensitive).
|
|
164
|
+
self._taken: set[str] = set()
|
|
165
|
+
self._saved: dict[tuple[Any, str, str], Path] = {} # (project id, branch, path) -> first download
|
|
166
|
+
|
|
167
|
+
def describe(self) -> str:
|
|
168
|
+
"""'download files to results/<term>/files (max 10 MiB each)' for the search plan."""
|
|
169
|
+
limit = f"max {format_size(self.max_size)} each" if self.max_size else "no size limit"
|
|
170
|
+
return f"download files to {self.layout.describe_files()} ({limit})"
|
|
171
|
+
|
|
172
|
+
def summary(self) -> str:
|
|
173
|
+
"""'3 files (12 KiB) downloaded to results/<term>/files, 1 skipped: larger than 10 MiB'."""
|
|
174
|
+
s = self.stats
|
|
175
|
+
text = (f"{s.files} file{'s' if s.files != 1 else ''} ({format_size(s.bytes)}) "
|
|
176
|
+
f"downloaded to {self.layout.describe_files()}")
|
|
177
|
+
if s.copied:
|
|
178
|
+
text += f" ({s.copied} copied locally: same file, another term)"
|
|
179
|
+
if s.too_large:
|
|
180
|
+
text += f", {s.too_large} skipped: larger than {format_size(self.max_size)}"
|
|
181
|
+
if s.failed:
|
|
182
|
+
text += f", {s.failed} FAILED (see the warnings)"
|
|
183
|
+
return text
|
|
184
|
+
|
|
185
|
+
@staticmethod
|
|
186
|
+
def _key(target: Path) -> str:
|
|
187
|
+
return str(target).lower()
|
|
188
|
+
|
|
189
|
+
def _reserve(self, folder: Path, base: str) -> Path:
|
|
190
|
+
"""A free path for ``base`` in ``folder`` (lock held): adds ~2, ~3 … before the extension if taken."""
|
|
191
|
+
stem, dot, ext = base.rpartition(".")
|
|
192
|
+
if not stem or stem.endswith("_"): # no extension, or a dot file like ".env"
|
|
193
|
+
stem, dot, ext = base, "", ""
|
|
194
|
+
candidate, n = folder / base, 1
|
|
195
|
+
while self._key(candidate) in self._taken or candidate.exists():
|
|
196
|
+
n += 1
|
|
197
|
+
candidate = folder / f"{stem}~{n}{dot}{ext}"
|
|
198
|
+
self._taken.add(self._key(candidate))
|
|
199
|
+
return candidate
|
|
200
|
+
|
|
201
|
+
def _release(self, target: Path) -> None:
|
|
202
|
+
with self._lock:
|
|
203
|
+
self._taken.discard(self._key(target))
|
|
204
|
+
|
|
205
|
+
def failed(self) -> None:
|
|
206
|
+
"""Count a download the caller couldn't make (API error)."""
|
|
207
|
+
with self._lock:
|
|
208
|
+
self.stats.failed += 1
|
|
209
|
+
|
|
210
|
+
def save(
|
|
211
|
+
self,
|
|
212
|
+
*,
|
|
213
|
+
project_id: Any,
|
|
214
|
+
project_path: str,
|
|
215
|
+
branch: str,
|
|
216
|
+
term: str,
|
|
217
|
+
path: str,
|
|
218
|
+
fetch: Callable[[], Response],
|
|
219
|
+
) -> Path | None:
|
|
220
|
+
"""
|
|
221
|
+
Save one file; ``fetch`` opens the streamed raw-file response.
|
|
222
|
+
|
|
223
|
+
Returns:
|
|
224
|
+
The local path, or None if the file is larger than ``max_size``.
|
|
225
|
+
|
|
226
|
+
Raises:
|
|
227
|
+
Whatever ``fetch`` or reading the stream raises (API/network
|
|
228
|
+
errors), and OSError for file system errors; nothing is left on
|
|
229
|
+
disk then and the caller should call ``failed()``.
|
|
230
|
+
"""
|
|
231
|
+
key = (project_id, branch, path)
|
|
232
|
+
base = file_name(project_path, branch, self.stamp, term, path)
|
|
233
|
+
folder = self.layout.files_dir(term)
|
|
234
|
+
with self._lock:
|
|
235
|
+
earlier = self._saved.get(key)
|
|
236
|
+
target = self._reserve(folder, base)
|
|
237
|
+
folder.mkdir(parents=True, exist_ok=True)
|
|
238
|
+
|
|
239
|
+
if earlier is not None and earlier.is_file():
|
|
240
|
+
try:
|
|
241
|
+
shutil.copyfile(earlier, target)
|
|
242
|
+
except OSError:
|
|
243
|
+
self._release(target)
|
|
244
|
+
target.unlink(missing_ok=True)
|
|
245
|
+
raise
|
|
246
|
+
with self._lock:
|
|
247
|
+
self.stats.copied += 1
|
|
248
|
+
return target
|
|
249
|
+
|
|
250
|
+
part = target.with_name(target.name + PART_SUFFIX)
|
|
251
|
+
written = 0
|
|
252
|
+
try:
|
|
253
|
+
response = fetch()
|
|
254
|
+
try:
|
|
255
|
+
length = _content_length(response.headers)
|
|
256
|
+
if self.max_size and length is not None and length > self.max_size:
|
|
257
|
+
raise FileTooLarge(length)
|
|
258
|
+
with part.open("wb") as fp:
|
|
259
|
+
for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
|
|
260
|
+
if not chunk:
|
|
261
|
+
continue
|
|
262
|
+
written += len(chunk)
|
|
263
|
+
if self.max_size and written > self.max_size:
|
|
264
|
+
raise FileTooLarge(None)
|
|
265
|
+
fp.write(chunk)
|
|
266
|
+
finally:
|
|
267
|
+
response.close()
|
|
268
|
+
part.replace(target)
|
|
269
|
+
except FileTooLarge:
|
|
270
|
+
part.unlink(missing_ok=True)
|
|
271
|
+
self._release(target)
|
|
272
|
+
with self._lock:
|
|
273
|
+
self.stats.too_large += 1
|
|
274
|
+
return None
|
|
275
|
+
except BaseException:
|
|
276
|
+
part.unlink(missing_ok=True)
|
|
277
|
+
self._release(target)
|
|
278
|
+
raise
|
|
279
|
+
|
|
280
|
+
with self._lock:
|
|
281
|
+
self.stats.saved += 1
|
|
282
|
+
self.stats.bytes += written
|
|
283
|
+
self._saved.setdefault(key, target)
|
|
284
|
+
return target
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def _content_length(headers: Any) -> int | None:
|
|
288
|
+
try:
|
|
289
|
+
value = headers.get("Content-Length") if headers is not None else None
|
|
290
|
+
return int(value) if value is not None else None
|
|
291
|
+
except (TypeError, ValueError):
|
|
292
|
+
return None
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
# glh/filefilter.py
|
|
2
|
+
"""
|
|
3
|
+
File filters for the search: ``--filename`` and ``--extension``.
|
|
4
|
+
|
|
5
|
+
GitLab code search (``scope=blobs``, basic and advanced search) understands
|
|
6
|
+
filters written into the search string itself: ``password filename:*.env``
|
|
7
|
+
searches "password" only in files whose name matches ``*.env``. There is no
|
|
8
|
+
separate API parameter, so the harvester appends the filters to every term in
|
|
9
|
+
the request (``FileFilter.query``); the term stored in the session stays as
|
|
10
|
+
the user wrote it.
|
|
11
|
+
|
|
12
|
+
The filters are also re-checked on the client (``FileFilter.matches``) against
|
|
13
|
+
the ``path`` of every result, for two reasons:
|
|
14
|
+
|
|
15
|
+
- an instance or GitLab version that ignores a filter still yields only the
|
|
16
|
+
files asked for;
|
|
17
|
+
- GitLab's basic search is looser than the option promises: its ``filename:``
|
|
18
|
+
matches the *end of the whole path* (``filename:config.yml`` also returns
|
|
19
|
+
``app/myconfig.yml``), while ``--filename`` matches the file name exactly.
|
|
20
|
+
|
|
21
|
+
Matching (both filters case-insensitive, ``*`` is the only wildcard and
|
|
22
|
+
matches any characters):
|
|
23
|
+
|
|
24
|
+
- ``--filename PATTERN``: the file name (basename), e.g. ``settings.py``,
|
|
25
|
+
``*.env``, ``config*``, ``docker-compose*.yml``. A pattern with ``/``
|
|
26
|
+
matches the end of the path instead (``config/*.yml``).
|
|
27
|
+
- ``--extension EXT``: the file name ends with ``.EXT``. Leading ``*``/``.``
|
|
28
|
+
and trailing dots are dropped, so ``.py``, ``py`` and ``*.py`` are the same.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
from __future__ import annotations
|
|
32
|
+
|
|
33
|
+
import re
|
|
34
|
+
from dataclasses import dataclass
|
|
35
|
+
from typing import Any
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class FileFilterError(ValueError):
|
|
39
|
+
"""An invalid --filename / --extension value."""
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _check_no_spaces(value: str, option: str) -> None:
|
|
43
|
+
# GitLab reads a filter value up to the next whitespace: a space would turn
|
|
44
|
+
# the rest of the value into a search term.
|
|
45
|
+
if any(ch.isspace() for ch in value):
|
|
46
|
+
raise FileFilterError(f"{option} must not contain spaces: {value!r}")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def normalize_filename(value: str) -> str:
|
|
50
|
+
"""
|
|
51
|
+
Validate a --filename pattern: no spaces, not empty; a leading '/' is
|
|
52
|
+
dropped (GitLab paths have none).
|
|
53
|
+
"""
|
|
54
|
+
v = value.strip().lstrip("/")
|
|
55
|
+
if not v or not v.strip("*"):
|
|
56
|
+
raise FileFilterError("--filename needs a file name or a mask, e.g. settings.py or *.env")
|
|
57
|
+
_check_no_spaces(v, "--filename")
|
|
58
|
+
return v
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def normalize_extension(value: str) -> str:
|
|
62
|
+
"""
|
|
63
|
+
Validate an --extension value and drop the extra dots: '.py', 'py', '..py',
|
|
64
|
+
'*.py' and 'py.' all become 'py'. Inner dots stay ('tar.gz').
|
|
65
|
+
"""
|
|
66
|
+
v = value.strip().lstrip("*.").rstrip(".")
|
|
67
|
+
if not v:
|
|
68
|
+
raise FileFilterError("--extension needs an extension, e.g. py or .py")
|
|
69
|
+
_check_no_spaces(v, "--extension")
|
|
70
|
+
if "/" in v:
|
|
71
|
+
raise FileFilterError(f"--extension must not contain '/': {value!r}")
|
|
72
|
+
return v
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _wildcard(pattern: str) -> str:
|
|
76
|
+
"""Regex for a pattern where only '*' is special."""
|
|
77
|
+
return ".*".join(re.escape(part) for part in pattern.split("*"))
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@dataclass(frozen=True, slots=True)
|
|
81
|
+
class FileFilter:
|
|
82
|
+
"""
|
|
83
|
+
The file filters of a search; empty (falsy) when neither is set.
|
|
84
|
+
|
|
85
|
+
Values are normalized on construction (``normalize_filename``,
|
|
86
|
+
``normalize_extension``), so library callers get the same behavior as the
|
|
87
|
+
CLI (``extension=".py"`` means ``py``); invalid values raise FileFilterError.
|
|
88
|
+
"""
|
|
89
|
+
|
|
90
|
+
filename: str | None = None
|
|
91
|
+
extension: str | None = None
|
|
92
|
+
|
|
93
|
+
def __post_init__(self) -> None:
|
|
94
|
+
# Frozen dataclass: normalize through object.__setattr__.
|
|
95
|
+
if self.filename is not None:
|
|
96
|
+
object.__setattr__(self, "filename", normalize_filename(self.filename))
|
|
97
|
+
if self.extension is not None:
|
|
98
|
+
object.__setattr__(self, "extension", normalize_extension(self.extension))
|
|
99
|
+
|
|
100
|
+
def __bool__(self) -> bool:
|
|
101
|
+
return bool(self.filename or self.extension)
|
|
102
|
+
|
|
103
|
+
def query(self, term: str) -> str:
|
|
104
|
+
"""The search string sent to GitLab: the term plus the filters."""
|
|
105
|
+
parts = [term]
|
|
106
|
+
if self.filename:
|
|
107
|
+
parts.append(f"filename:{self.filename}")
|
|
108
|
+
if self.extension:
|
|
109
|
+
parts.append(f"extension:{self.extension}")
|
|
110
|
+
return " ".join(parts)
|
|
111
|
+
|
|
112
|
+
def matches(self, path: str) -> bool:
|
|
113
|
+
"""True if the file at ``path`` (repository path, '/'-separated) passes the filters."""
|
|
114
|
+
name = path.rsplit("/", 1)[-1]
|
|
115
|
+
if self.filename:
|
|
116
|
+
if "/" in self.filename:
|
|
117
|
+
ok = re.fullmatch(f"(?:.*/)?{_wildcard(self.filename)}", path, re.IGNORECASE)
|
|
118
|
+
else:
|
|
119
|
+
ok = re.fullmatch(_wildcard(self.filename), name, re.IGNORECASE)
|
|
120
|
+
if not ok:
|
|
121
|
+
return False
|
|
122
|
+
if self.extension:
|
|
123
|
+
if not re.fullmatch(rf".*\.{_wildcard(self.extension)}", name, re.IGNORECASE):
|
|
124
|
+
return False
|
|
125
|
+
return True
|
|
126
|
+
|
|
127
|
+
def describe(self) -> str:
|
|
128
|
+
"""'filename *.env, extension py' (for the search plan and the run journal)."""
|
|
129
|
+
parts = []
|
|
130
|
+
if self.filename:
|
|
131
|
+
parts.append(f"filename {self.filename}")
|
|
132
|
+
if self.extension:
|
|
133
|
+
parts.append(f"extension {self.extension}")
|
|
134
|
+
return ", ".join(parts)
|
|
135
|
+
|
|
136
|
+
def to_meta(self) -> dict[str, Any]:
|
|
137
|
+
"""Session meta fields."""
|
|
138
|
+
return {"filename": self.filename, "extension": self.extension}
|