gitlab-harvester 1.2.2__tar.gz → 1.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.4.1}/PKG-INFO +17 -3
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.4.1}/README.md +16 -2
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.4.1}/gitlab_harvester.egg-info/PKG-INFO +17 -3
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.4.1}/gitlab_harvester.egg-info/SOURCES.txt +8 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.4.1}/gitlab_harvester.py +7 -1
- gitlab_harvester-1.4.1/glh/archive.py +150 -0
- gitlab_harvester-1.4.1/glh/autolimit.py +394 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.4.1}/glh/cli.py +195 -6
- gitlab_harvester-1.4.1/glh/downloads.py +310 -0
- gitlab_harvester-1.4.1/glh/filefilter.py +138 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.4.1}/glh/harvester.py +480 -45
- gitlab_harvester-1.4.1/glh/limits.py +805 -0
- gitlab_harvester-1.4.1/glh/modes.py +829 -0
- gitlab_harvester-1.4.1/glh/output.py +198 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.4.1}/glh/planner.py +43 -3
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.4.1}/glh/postprocess/convert.py +0 -3
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.4.1}/glh/postprocess/dedup.py +1 -6
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.4.1}/glh/postprocess/pipeline.py +5 -0
- gitlab_harvester-1.4.1/glh/postprocess/split.py +79 -0
- gitlab_harvester-1.4.1/glh/projectfilter.py +80 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.4.1}/glh/ratelimit.py +32 -6
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.4.1}/glh/session.py +22 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.4.1}/glh/tokens.py +49 -0
- gitlab_harvester-1.4.1/glh/utils/keys.py +134 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.4.1}/glh/utils/terminal_utils.py +259 -9
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.4.1}/pyproject.toml +1 -1
- gitlab_harvester-1.2.2/glh/modes.py +0 -507
- gitlab_harvester-1.2.2/glh/postprocess/split.py +0 -123
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.4.1}/LICENSE +0 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.4.1}/gitlab_harvester.egg-info/dependency_links.txt +0 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.4.1}/gitlab_harvester.egg-info/entry_points.txt +0 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.4.1}/gitlab_harvester.egg-info/requires.txt +0 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.4.1}/gitlab_harvester.egg-info/top_level.txt +0 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.4.1}/glh/__init__.py +0 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.4.1}/glh/config.py +0 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.4.1}/glh/exceptions.py +0 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.4.1}/glh/postprocess/__init__.py +0 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.4.1}/glh/user_agents.py +0 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.4.1}/glh/utils/__init__.py +0 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.4.1}/glh/utils/logging_utils.py +0 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.4.1}/glh/workers.py +0 -0
- {gitlab_harvester-1.2.2 → gitlab_harvester-1.4.1}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gitlab-harvester
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.4.1
|
|
4
4
|
Summary: Build a GitLab instance project index and search repositories for sensitive keywords (API-only, no cloning).
|
|
5
5
|
Author-email: Cur1iosity <cur1iosity@protonmail.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -90,10 +90,19 @@ gitlab-harvester -u https://gitlab.example.com -t "$GITLAB_TOKEN" -s password
|
|
|
90
90
|
The first run builds a project index (`<host>_project_index.jsonl`), and later runs reuse it.
|
|
91
91
|
Hits are written to a JSONL session file as they are found.
|
|
92
92
|
|
|
93
|
-
Lists of terms,
|
|
94
|
-
and rate limits, deduplicated reports,
|
|
93
|
+
Lists of terms, specific files only, saving the files with hits, all branches and forks,
|
|
94
|
+
fresh findings only, resuming, parallel requests and rate limits, deduplicated reports,
|
|
95
|
+
config files:
|
|
95
96
|
**[docs/USE_CASES.md](docs/USE_CASES.md)**.
|
|
96
97
|
|
|
98
|
+
<p align="center">
|
|
99
|
+
<a href="docs/USE_CASES.md">
|
|
100
|
+
<img src="docs/assets/use-cases.svg"
|
|
101
|
+
alt="Use cases at a glance: gitlab-harvester branches into scenarios from docs/USE_CASES.md (search terms, only some files, keep the files in one ZIP, every file by name without a term, only some projects, project inventory, branches and forks, fresh findings, speed and rate limits, several tokens, reports, config file), each with its main flags; shortcuts such as -b, --time-after and --rpm show what they expand to"
|
|
102
|
+
width="900">
|
|
103
|
+
</a>
|
|
104
|
+
</p>
|
|
105
|
+
|
|
97
106
|
## Common options
|
|
98
107
|
|
|
99
108
|
| Option | Purpose |
|
|
@@ -102,14 +111,19 @@ and rate limits, deduplicated reports, config files:
|
|
|
102
111
|
| `--forks include\|skip\|branch-diff\|all-branches` | how forks are handled |
|
|
103
112
|
| `--time-after DATE`, `--time-before DATE` | limit the index, its projects and the files with hits to a time window |
|
|
104
113
|
| `--started-from 2024-06-01` | search only indexed projects with activity since the date |
|
|
114
|
+
| `--project-filter k8s,infra/*` | search only the indexed projects whose path matches (substring or mask) |
|
|
115
|
+
| `--filename '*.env'`, `--extension py` | search only files with this name (exact, or a mask with `*`) or extension; without a term, collect every such file |
|
|
116
|
+
| `--download-files`, `--download-max-size 10M` | save every file with hits to `<term>/files/` for offline analysis |
|
|
105
117
|
| `-t "tok1;tok2"`, `--token-file FILE` | several tokens run in parallel, each with its own workers and rate-limit budget |
|
|
106
118
|
| `-w, --workers N` | parallel API requests per token, for instances without rate limits |
|
|
107
119
|
| `--rpm N`, `--index-rpm N`, `--search-rpm N` | requests per minute per worker: all, non-search, search |
|
|
120
|
+
| `-m find-limits`, `--auto-limits` | *experimental*: measure the API rate limits and get `--workers`/`--*-rpm` for them; during a search, propose lower limits after HTTP 429 |
|
|
108
121
|
| `--jitter 0.01-0.05` | random delay before every request (seconds, or `10ms-50ms`) |
|
|
109
122
|
| `-A STRING`, `--user-agent-file FILE` | fixed User-Agent, or a random one from the file for every request |
|
|
110
123
|
| `-m dump-index`, `--dump-projects` | build or rebuild the project index |
|
|
111
124
|
| `--session NAME`, `--resume` | name a session and continue it later |
|
|
112
125
|
| `--post-process dedup,convert`, `--final-only` | deduplicate and convert the session right after the search |
|
|
126
|
+
| `--to-zip` | pack the run's results (session, term files, downloaded files) into `<session>.zip` |
|
|
113
127
|
| `-c, --config FILE` | read options from an INI file ([config.example.ini](config.example.ini)) |
|
|
114
128
|
|
|
115
129
|
See [docs/usage.md](docs/usage.md) for every option, branch and fork strategies, and the output formats,
|
|
@@ -52,10 +52,19 @@ gitlab-harvester -u https://gitlab.example.com -t "$GITLAB_TOKEN" -s password
|
|
|
52
52
|
The first run builds a project index (`<host>_project_index.jsonl`), and later runs reuse it.
|
|
53
53
|
Hits are written to a JSONL session file as they are found.
|
|
54
54
|
|
|
55
|
-
Lists of terms,
|
|
56
|
-
and rate limits, deduplicated reports,
|
|
55
|
+
Lists of terms, specific files only, saving the files with hits, all branches and forks,
|
|
56
|
+
fresh findings only, resuming, parallel requests and rate limits, deduplicated reports,
|
|
57
|
+
config files:
|
|
57
58
|
**[docs/USE_CASES.md](docs/USE_CASES.md)**.
|
|
58
59
|
|
|
60
|
+
<p align="center">
|
|
61
|
+
<a href="docs/USE_CASES.md">
|
|
62
|
+
<img src="docs/assets/use-cases.svg"
|
|
63
|
+
alt="Use cases at a glance: gitlab-harvester branches into scenarios from docs/USE_CASES.md (search terms, only some files, keep the files in one ZIP, every file by name without a term, only some projects, project inventory, branches and forks, fresh findings, speed and rate limits, several tokens, reports, config file), each with its main flags; shortcuts such as -b, --time-after and --rpm show what they expand to"
|
|
64
|
+
width="900">
|
|
65
|
+
</a>
|
|
66
|
+
</p>
|
|
67
|
+
|
|
59
68
|
## Common options
|
|
60
69
|
|
|
61
70
|
| Option | Purpose |
|
|
@@ -64,14 +73,19 @@ and rate limits, deduplicated reports, config files:
|
|
|
64
73
|
| `--forks include\|skip\|branch-diff\|all-branches` | how forks are handled |
|
|
65
74
|
| `--time-after DATE`, `--time-before DATE` | limit the index, its projects and the files with hits to a time window |
|
|
66
75
|
| `--started-from 2024-06-01` | search only indexed projects with activity since the date |
|
|
76
|
+
| `--project-filter k8s,infra/*` | search only the indexed projects whose path matches (substring or mask) |
|
|
77
|
+
| `--filename '*.env'`, `--extension py` | search only files with this name (exact, or a mask with `*`) or extension; without a term, collect every such file |
|
|
78
|
+
| `--download-files`, `--download-max-size 10M` | save every file with hits to `<term>/files/` for offline analysis |
|
|
67
79
|
| `-t "tok1;tok2"`, `--token-file FILE` | several tokens run in parallel, each with its own workers and rate-limit budget |
|
|
68
80
|
| `-w, --workers N` | parallel API requests per token, for instances without rate limits |
|
|
69
81
|
| `--rpm N`, `--index-rpm N`, `--search-rpm N` | requests per minute per worker: all, non-search, search |
|
|
82
|
+
| `-m find-limits`, `--auto-limits` | *experimental*: measure the API rate limits and get `--workers`/`--*-rpm` for them; during a search, propose lower limits after HTTP 429 |
|
|
70
83
|
| `--jitter 0.01-0.05` | random delay before every request (seconds, or `10ms-50ms`) |
|
|
71
84
|
| `-A STRING`, `--user-agent-file FILE` | fixed User-Agent, or a random one from the file for every request |
|
|
72
85
|
| `-m dump-index`, `--dump-projects` | build or rebuild the project index |
|
|
73
86
|
| `--session NAME`, `--resume` | name a session and continue it later |
|
|
74
87
|
| `--post-process dedup,convert`, `--final-only` | deduplicate and convert the session right after the search |
|
|
88
|
+
| `--to-zip` | pack the run's results (session, term files, downloaded files) into `<session>.zip` |
|
|
75
89
|
| `-c, --config FILE` | read options from an INI file ([config.example.ini](config.example.ini)) |
|
|
76
90
|
|
|
77
91
|
See [docs/usage.md](docs/usage.md) for every option, branch and fork strategies, and the output formats,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gitlab-harvester
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.4.1
|
|
4
4
|
Summary: Build a GitLab instance project index and search repositories for sensitive keywords (API-only, no cloning).
|
|
5
5
|
Author-email: Cur1iosity <cur1iosity@protonmail.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -90,10 +90,19 @@ gitlab-harvester -u https://gitlab.example.com -t "$GITLAB_TOKEN" -s password
|
|
|
90
90
|
The first run builds a project index (`<host>_project_index.jsonl`), and later runs reuse it.
|
|
91
91
|
Hits are written to a JSONL session file as they are found.
|
|
92
92
|
|
|
93
|
-
Lists of terms,
|
|
94
|
-
and rate limits, deduplicated reports,
|
|
93
|
+
Lists of terms, specific files only, saving the files with hits, all branches and forks,
|
|
94
|
+
fresh findings only, resuming, parallel requests and rate limits, deduplicated reports,
|
|
95
|
+
config files:
|
|
95
96
|
**[docs/USE_CASES.md](docs/USE_CASES.md)**.
|
|
96
97
|
|
|
98
|
+
<p align="center">
|
|
99
|
+
<a href="docs/USE_CASES.md">
|
|
100
|
+
<img src="docs/assets/use-cases.svg"
|
|
101
|
+
alt="Use cases at a glance: gitlab-harvester branches into scenarios from docs/USE_CASES.md (search terms, only some files, keep the files in one ZIP, every file by name without a term, only some projects, project inventory, branches and forks, fresh findings, speed and rate limits, several tokens, reports, config file), each with its main flags; shortcuts such as -b, --time-after and --rpm show what they expand to"
|
|
102
|
+
width="900">
|
|
103
|
+
</a>
|
|
104
|
+
</p>
|
|
105
|
+
|
|
97
106
|
## Common options
|
|
98
107
|
|
|
99
108
|
| Option | Purpose |
|
|
@@ -102,14 +111,19 @@ and rate limits, deduplicated reports, config files:
|
|
|
102
111
|
| `--forks include\|skip\|branch-diff\|all-branches` | how forks are handled |
|
|
103
112
|
| `--time-after DATE`, `--time-before DATE` | limit the index, its projects and the files with hits to a time window |
|
|
104
113
|
| `--started-from 2024-06-01` | search only indexed projects with activity since the date |
|
|
114
|
+
| `--project-filter k8s,infra/*` | search only the indexed projects whose path matches (substring or mask) |
|
|
115
|
+
| `--filename '*.env'`, `--extension py` | search only files with this name (exact, or a mask with `*`) or extension; without a term, collect every such file |
|
|
116
|
+
| `--download-files`, `--download-max-size 10M` | save every file with hits to `<term>/files/` for offline analysis |
|
|
105
117
|
| `-t "tok1;tok2"`, `--token-file FILE` | several tokens run in parallel, each with its own workers and rate-limit budget |
|
|
106
118
|
| `-w, --workers N` | parallel API requests per token, for instances without rate limits |
|
|
107
119
|
| `--rpm N`, `--index-rpm N`, `--search-rpm N` | requests per minute per worker: all, non-search, search |
|
|
120
|
+
| `-m find-limits`, `--auto-limits` | *experimental*: measure the API rate limits and get `--workers`/`--*-rpm` for them; during a search, propose lower limits after HTTP 429 |
|
|
108
121
|
| `--jitter 0.01-0.05` | random delay before every request (seconds, or `10ms-50ms`) |
|
|
109
122
|
| `-A STRING`, `--user-agent-file FILE` | fixed User-Agent, or a random one from the file for every request |
|
|
110
123
|
| `-m dump-index`, `--dump-projects` | build or rebuild the project index |
|
|
111
124
|
| `--session NAME`, `--resume` | name a session and continue it later |
|
|
112
125
|
| `--post-process dedup,convert`, `--final-only` | deduplicate and convert the session right after the search |
|
|
126
|
+
| `--to-zip` | pack the run's results (session, term files, downloaded files) into `<session>.zip` |
|
|
113
127
|
| `-c, --config FILE` | read options from an INI file ([config.example.ini](config.example.ini)) |
|
|
114
128
|
|
|
115
129
|
See [docs/usage.md](docs/usage.md) for every option, branch and fork strategies, and the output formats,
|
|
@@ -9,12 +9,19 @@ gitlab_harvester.egg-info/entry_points.txt
|
|
|
9
9
|
gitlab_harvester.egg-info/requires.txt
|
|
10
10
|
gitlab_harvester.egg-info/top_level.txt
|
|
11
11
|
glh/__init__.py
|
|
12
|
+
glh/archive.py
|
|
13
|
+
glh/autolimit.py
|
|
12
14
|
glh/cli.py
|
|
13
15
|
glh/config.py
|
|
16
|
+
glh/downloads.py
|
|
14
17
|
glh/exceptions.py
|
|
18
|
+
glh/filefilter.py
|
|
15
19
|
glh/harvester.py
|
|
20
|
+
glh/limits.py
|
|
16
21
|
glh/modes.py
|
|
22
|
+
glh/output.py
|
|
17
23
|
glh/planner.py
|
|
24
|
+
glh/projectfilter.py
|
|
18
25
|
glh/ratelimit.py
|
|
19
26
|
glh/session.py
|
|
20
27
|
glh/tokens.py
|
|
@@ -26,5 +33,6 @@ glh/postprocess/dedup.py
|
|
|
26
33
|
glh/postprocess/pipeline.py
|
|
27
34
|
glh/postprocess/split.py
|
|
28
35
|
glh/utils/__init__.py
|
|
36
|
+
glh/utils/keys.py
|
|
29
37
|
glh/utils/logging_utils.py
|
|
30
38
|
glh/utils/terminal_utils.py
|
|
@@ -2,11 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
+
import shlex
|
|
5
6
|
import signal
|
|
6
7
|
import sys
|
|
7
8
|
|
|
8
9
|
from glh.cli import CliParser, get_banner
|
|
9
10
|
from glh.modes import run_mode
|
|
11
|
+
from glh.tokens import mask_argv
|
|
12
|
+
from glh.utils.terminal_utils import COMMAND_STYLE, style
|
|
10
13
|
|
|
11
14
|
__all__ = ["main"]
|
|
12
15
|
|
|
@@ -45,10 +48,13 @@ def main(argv: list[str] | None = None) -> None:
|
|
|
45
48
|
|
|
46
49
|
if not any(a in {"-h", "--help", "-V", "--version"} for a in argv):
|
|
47
50
|
print(get_banner(sys.stderr), file=sys.stderr)
|
|
51
|
+
# The command as typed (secrets masked), before anything the run prints.
|
|
52
|
+
command = shlex.join(["gitlab-harvester", *mask_argv(argv)])
|
|
53
|
+
print(style(f"$ {command}", COMMAND_STYLE, sys.stderr), file=sys.stderr)
|
|
48
54
|
|
|
49
55
|
try:
|
|
50
56
|
args = CliParser.parse(argv)
|
|
51
|
-
run_mode(args)
|
|
57
|
+
run_mode(args, argv=argv)
|
|
52
58
|
except KeyboardInterrupt:
|
|
53
59
|
_handle_exit(signal.SIGINT, None)
|
|
54
60
|
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
# glh/archive.py
|
|
2
|
+
"""
|
|
3
|
+
Pack the results of a search run into one ZIP archive (``--to-zip``).
|
|
4
|
+
|
|
5
|
+
The archive sits next to the session, ``<dir>/<session>.zip``, and keeps the
|
|
6
|
+
layout of ``glh.output.OutputLayout``: every member is named by its path
|
|
7
|
+
relative to ``<dir>``::
|
|
8
|
+
|
|
9
|
+
<session>.jsonl the session (unless --final-only removed it)
|
|
10
|
+
<session>.deduped.jsonl, ... --post-process outputs left on disk
|
|
11
|
+
<term>/<session>.jsonl --output-separated
|
|
12
|
+
<term>/files/<downloaded file> --download-files
|
|
13
|
+
|
|
14
|
+
Only files of this session go in, never everything in ``<dir>`` (the default
|
|
15
|
+
``results/`` is shared by all sessions): the downloaded files are the ones the
|
|
16
|
+
session's hits point to (their ``"file"`` key), so files downloaded before a
|
|
17
|
+
``--resume`` are included too. A file outside ``<dir>`` goes under
|
|
18
|
+
``external/``.
|
|
19
|
+
|
|
20
|
+
Members are compressed one at a time in chunks (constant memory, ZIP64 for big
|
|
21
|
+
archives), into ``<session>.zip.part`` renamed when complete, so an interrupted
|
|
22
|
+
run never leaves a half-written archive under the final name. An existing
|
|
23
|
+
archive of the same session is replaced. The original files stay where they are.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import json
|
|
29
|
+
import os
|
|
30
|
+
import zipfile
|
|
31
|
+
from dataclasses import dataclass, field
|
|
32
|
+
from pathlib import Path
|
|
33
|
+
from typing import Iterable, Iterator
|
|
34
|
+
|
|
35
|
+
from glh.downloads import format_size
|
|
36
|
+
|
|
37
|
+
ZIP_SUFFIX = ".zip"
|
|
38
|
+
PART_SUFFIX = ".part"
|
|
39
|
+
EXTERNAL_DIR = "external"
|
|
40
|
+
COMPRESSION = zipfile.ZIP_DEFLATED
|
|
41
|
+
COMPRESS_LEVEL = 6
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def archive_path(session_path: Path | str) -> Path:
|
|
45
|
+
"""``<dir>/<session>.zip`` for ``<dir>/<session>.jsonl``."""
|
|
46
|
+
session_path = Path(session_path)
|
|
47
|
+
return session_path.with_name(session_path.stem + ZIP_SUFFIX)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def downloaded_files(session_path: Path | str) -> Iterator[str]:
|
|
51
|
+
"""
|
|
52
|
+
The local paths in the ``"file"`` keys of the session's hit records, in
|
|
53
|
+
order (repeated when several hits share a file); the session is streamed.
|
|
54
|
+
"""
|
|
55
|
+
try:
|
|
56
|
+
fp = Path(session_path).open(encoding="utf-8")
|
|
57
|
+
except OSError:
|
|
58
|
+
return
|
|
59
|
+
with fp:
|
|
60
|
+
for line in fp:
|
|
61
|
+
line = line.strip()
|
|
62
|
+
if not line or '"file"' not in line:
|
|
63
|
+
continue
|
|
64
|
+
try:
|
|
65
|
+
record = json.loads(line)
|
|
66
|
+
except json.JSONDecodeError:
|
|
67
|
+
continue
|
|
68
|
+
result = record.get("result") if isinstance(record, dict) and "term" in record else None
|
|
69
|
+
if not isinstance(result, dict):
|
|
70
|
+
continue
|
|
71
|
+
for branch in result.get("project_search_result") or ():
|
|
72
|
+
for hit in (branch.get("branch_search_result") or ()) if isinstance(branch, dict) else ():
|
|
73
|
+
if isinstance(hit, dict) and isinstance(hit.get("file"), str) and hit["file"]:
|
|
74
|
+
yield hit["file"]
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@dataclass(slots=True)
|
|
78
|
+
class ZipResult:
|
|
79
|
+
"""What went into an archive."""
|
|
80
|
+
|
|
81
|
+
path: Path
|
|
82
|
+
files: int = 0
|
|
83
|
+
size: int = 0 # bytes before compression
|
|
84
|
+
compressed: int = 0 # size of the archive
|
|
85
|
+
missing: list[str] = field(default_factory=list) # paths that were gone
|
|
86
|
+
|
|
87
|
+
def describe(self) -> str:
|
|
88
|
+
"""'results/x.zip (12 files, 3.4 MiB -> 1.1 MiB)'."""
|
|
89
|
+
text = (f"{self.path} ({self.files} file{'s' if self.files != 1 else ''}, "
|
|
90
|
+
f"{format_size(self.size)} -> {format_size(self.compressed)})")
|
|
91
|
+
if self.missing:
|
|
92
|
+
text += f"; {len(self.missing)} missing, not archived"
|
|
93
|
+
return text
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _member_name(path: Path, root: Path) -> str:
|
|
97
|
+
"""The path relative to ``root`` (POSIX separators); outside it: ``external/<name>``."""
|
|
98
|
+
try:
|
|
99
|
+
return path.resolve().relative_to(root.resolve()).as_posix()
|
|
100
|
+
except ValueError:
|
|
101
|
+
return f"{EXTERNAL_DIR}/{path.name}"
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def zip_results(zip_path: Path | str, root: Path | str, paths: Iterable[Path | str]) -> ZipResult:
|
|
105
|
+
"""
|
|
106
|
+
Write ``paths`` into the ZIP ``zip_path`` under their paths relative to ``root``.
|
|
107
|
+
|
|
108
|
+
Duplicates (the same member name) are archived once; paths that don't
|
|
109
|
+
exist (any more) are listed in ``missing``. The archive itself is never
|
|
110
|
+
a member.
|
|
111
|
+
|
|
112
|
+
Raises:
|
|
113
|
+
OSError: if the archive can't be written; nothing is left behind then.
|
|
114
|
+
"""
|
|
115
|
+
zip_path, root = Path(zip_path), Path(root)
|
|
116
|
+
part = zip_path.with_name(zip_path.name + PART_SUFFIX)
|
|
117
|
+
result = ZipResult(path=zip_path)
|
|
118
|
+
names: set[str] = set() # member names, lower-case (unpacked on case-insensitive file systems too)
|
|
119
|
+
# Files already archived (several hits share a file), and the archive itself.
|
|
120
|
+
done = {os.path.normcase(str(p.resolve())) for p in (zip_path, part)}
|
|
121
|
+
zip_path.parent.mkdir(parents=True, exist_ok=True)
|
|
122
|
+
try:
|
|
123
|
+
with zipfile.ZipFile(part, "w", compression=COMPRESSION, compresslevel=COMPRESS_LEVEL,
|
|
124
|
+
allowZip64=True) as zf:
|
|
125
|
+
for raw in paths:
|
|
126
|
+
path = Path(raw)
|
|
127
|
+
key = os.path.normcase(str(path.resolve()))
|
|
128
|
+
if key in done:
|
|
129
|
+
continue
|
|
130
|
+
if not path.is_file():
|
|
131
|
+
if str(raw) not in result.missing:
|
|
132
|
+
result.missing.append(str(raw))
|
|
133
|
+
continue
|
|
134
|
+
done.add(key)
|
|
135
|
+
name = base = _member_name(path, root)
|
|
136
|
+
n = 1
|
|
137
|
+
while name.lower() in names: # two different files, one name (external/)
|
|
138
|
+
n += 1
|
|
139
|
+
stem, dot, ext = base.rpartition(".")
|
|
140
|
+
name = f"{stem}~{n}.{ext}" if dot and "/" not in ext else f"{base}~{n}"
|
|
141
|
+
names.add(name.lower())
|
|
142
|
+
zf.write(path, name)
|
|
143
|
+
result.files += 1
|
|
144
|
+
result.size += path.stat().st_size
|
|
145
|
+
part.replace(zip_path)
|
|
146
|
+
except BaseException:
|
|
147
|
+
part.unlink(missing_ok=True)
|
|
148
|
+
raise
|
|
149
|
+
result.compressed = zip_path.stat().st_size
|
|
150
|
+
return result
|