git-finalizer 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- git_finalizer/__init__.py +8 -0
- git_finalizer/delivery/__init__.py +3 -0
- git_finalizer/delivery/cli.py +166 -0
- git_finalizer/delivery/content_scan.py +524 -0
- git_finalizer/delivery/engine.py +2364 -0
- git_finalizer/delivery/integration_publish.py +547 -0
- git_finalizer/delivery/parser.py +552 -0
- git_finalizer/delivery/patch_check.py +165 -0
- git_finalizer/delivery/policy.py +311 -0
- git_finalizer/delivery/repo_bootstrap.py +588 -0
- git_finalizer/delivery/repo_state.py +612 -0
- git_finalizer/delivery/retirement_plan.py +740 -0
- git_finalizer/delivery/runtime.py +553 -0
- git_finalizer/delivery/scanners.py +329 -0
- git_finalizer/delivery/snapshot_verify.py +425 -0
- git_finalizer/delivery/source_review.py +374 -0
- git_finalizer/delivery/trailers.py +164 -0
- git_finalizer/delivery/validation.py +1095 -0
- git_finalizer/errors.py +105 -0
- git_finalizer/git_io.py +281 -0
- git_finalizer/json_io.py +55 -0
- git_finalizer/toolchain/__init__.py +3 -0
- git_finalizer/toolchain/tool_skill_sync.py +3044 -0
- git_finalizer/toolchain/tool_temp_dir.py +1057 -0
- git_finalizer/toolchain/workflow_health.py +504 -0
- git_finalizer-2.0.0.dist-info/METADATA +383 -0
- git_finalizer-2.0.0.dist-info/RECORD +30 -0
- git_finalizer-2.0.0.dist-info/WHEEL +4 -0
- git_finalizer-2.0.0.dist-info/entry_points.txt +5 -0
- git_finalizer-2.0.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,329 @@
|
|
|
1
|
+
"""Pluggable external scanner SARIF 2.1.0 orchestration (`gitleaks`, `trufflehog`, `osv`, `sarif-cmd`)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
from pathlib import Path, PurePosixPath
|
|
8
|
+
import shutil
|
|
9
|
+
import subprocess
|
|
10
|
+
import tempfile
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from git_finalizer.delivery.policy import DeliveryPolicy
|
|
14
|
+
from git_finalizer.errors import ErrorCode, FinalizerError
|
|
15
|
+
|
|
16
|
+
SUPPORTED_SCANNERS = frozenset({"gitleaks", "trufflehog", "osv", "sarif-cmd"})
|
|
17
|
+
MAX_SARIF_BYTES = 16 * 1024 * 1024
|
|
18
|
+
BLOCKING_LEVELS = frozenset({"error", "warning", "none", ""})
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _resolve_scanner_argv(
|
|
22
|
+
scanner: str,
|
|
23
|
+
target_dir: Path,
|
|
24
|
+
report_path: Path,
|
|
25
|
+
policy: DeliveryPolicy,
|
|
26
|
+
sarif_cmd: str | None = None,
|
|
27
|
+
) -> list[str]:
|
|
28
|
+
if scanner == "gitleaks":
|
|
29
|
+
exe = shutil.which("gitleaks")
|
|
30
|
+
if not exe:
|
|
31
|
+
raise FinalizerError(
|
|
32
|
+
ErrorCode.EXTERNAL_SCANNER_FAILED,
|
|
33
|
+
"已请求 --scanner gitleaks,但 PATH 中未找到 gitleaks",
|
|
34
|
+
final_phase="local_validation",
|
|
35
|
+
)
|
|
36
|
+
return [
|
|
37
|
+
exe,
|
|
38
|
+
"dir",
|
|
39
|
+
str(target_dir),
|
|
40
|
+
"--no-banner",
|
|
41
|
+
"--exit-code",
|
|
42
|
+
"0",
|
|
43
|
+
"--report-format",
|
|
44
|
+
"sarif",
|
|
45
|
+
"--report-path",
|
|
46
|
+
str(report_path),
|
|
47
|
+
]
|
|
48
|
+
if scanner == "trufflehog":
|
|
49
|
+
exe = shutil.which("trufflehog")
|
|
50
|
+
if not exe:
|
|
51
|
+
raise FinalizerError(
|
|
52
|
+
ErrorCode.EXTERNAL_SCANNER_FAILED,
|
|
53
|
+
"已请求 --scanner trufflehog,但 PATH 中未找到 trufflehog",
|
|
54
|
+
final_phase="local_validation",
|
|
55
|
+
)
|
|
56
|
+
return [exe, "filesystem", str(target_dir), "--no-update", "--json"]
|
|
57
|
+
if scanner == "osv":
|
|
58
|
+
exe = shutil.which("osv-scanner") or shutil.which("osv")
|
|
59
|
+
if not exe:
|
|
60
|
+
raise FinalizerError(
|
|
61
|
+
ErrorCode.EXTERNAL_SCANNER_FAILED,
|
|
62
|
+
"已请求 --scanner osv,但 PATH 中未找到 osv-scanner",
|
|
63
|
+
final_phase="local_validation",
|
|
64
|
+
)
|
|
65
|
+
return [
|
|
66
|
+
exe,
|
|
67
|
+
"scan",
|
|
68
|
+
"source",
|
|
69
|
+
"-r",
|
|
70
|
+
str(target_dir),
|
|
71
|
+
"--format",
|
|
72
|
+
"sarif",
|
|
73
|
+
"--output",
|
|
74
|
+
str(report_path),
|
|
75
|
+
]
|
|
76
|
+
if scanner == "sarif-cmd":
|
|
77
|
+
if sarif_cmd:
|
|
78
|
+
resolved = shutil.which(sarif_cmd) or sarif_cmd
|
|
79
|
+
exe_path = Path(resolved)
|
|
80
|
+
if not exe_path.is_file() or not os.access(exe_path, os.X_OK):
|
|
81
|
+
raise FinalizerError(
|
|
82
|
+
ErrorCode.EXTERNAL_SCANNER_FAILED,
|
|
83
|
+
f"--sarif-cmd 可执行文件不可用: {sarif_cmd}",
|
|
84
|
+
final_phase="cli",
|
|
85
|
+
)
|
|
86
|
+
return [str(exe_path), str(target_dir), str(report_path)]
|
|
87
|
+
if policy.sarif_cmd_argv:
|
|
88
|
+
first = shutil.which(policy.sarif_cmd_argv[0]) or policy.sarif_cmd_argv[0]
|
|
89
|
+
if not Path(first).is_file() or not os.access(first, os.X_OK):
|
|
90
|
+
raise FinalizerError(
|
|
91
|
+
ErrorCode.EXTERNAL_SCANNER_FAILED,
|
|
92
|
+
"policy [scanners.sarif_cmd] 可执行文件不可用",
|
|
93
|
+
final_phase="cli",
|
|
94
|
+
)
|
|
95
|
+
return [
|
|
96
|
+
str(first),
|
|
97
|
+
*[
|
|
98
|
+
arg.replace("{target_dir}", str(target_dir)).replace(
|
|
99
|
+
"{report_path}", str(report_path)
|
|
100
|
+
)
|
|
101
|
+
for arg in policy.sarif_cmd_argv[1:]
|
|
102
|
+
],
|
|
103
|
+
]
|
|
104
|
+
raise FinalizerError(
|
|
105
|
+
ErrorCode.EXTERNAL_SCANNER_FAILED,
|
|
106
|
+
"使用 --scanner sarif-cmd 时必须提供 --sarif-cmd 或 .agents/delivery-policy.toml [scanners.sarif_cmd]",
|
|
107
|
+
final_phase="cli",
|
|
108
|
+
)
|
|
109
|
+
raise FinalizerError(
|
|
110
|
+
ErrorCode.CLI_INVALID_ARGS,
|
|
111
|
+
f"不支持的 --scanner 值: {scanner}",
|
|
112
|
+
final_phase="cli",
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def parse_sarif_findings(raw_bytes: bytes, scanner: str) -> list[dict[str, Any]]:
|
|
117
|
+
"""Parse SARIF 2.1.0 JSON without retaining raw secret snippets."""
|
|
118
|
+
if len(raw_bytes) > MAX_SARIF_BYTES:
|
|
119
|
+
raise FinalizerError(
|
|
120
|
+
ErrorCode.EXTERNAL_SCANNER_FAILED,
|
|
121
|
+
f"外部扫描器 {scanner} 输出超过 SARIF 体积上限",
|
|
122
|
+
status="failed",
|
|
123
|
+
final_phase="local_validation",
|
|
124
|
+
)
|
|
125
|
+
try:
|
|
126
|
+
document = json.loads(raw_bytes.decode("utf-8"))
|
|
127
|
+
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
128
|
+
raise FinalizerError(
|
|
129
|
+
ErrorCode.EXTERNAL_SCANNER_FAILED,
|
|
130
|
+
f"外部扫描器 {scanner} 未返回合法 SARIF JSON: {exc}",
|
|
131
|
+
status="failed",
|
|
132
|
+
final_phase="local_validation",
|
|
133
|
+
) from exc
|
|
134
|
+
if not isinstance(document, dict) or str(document.get("version", "")) != "2.1.0":
|
|
135
|
+
raise FinalizerError(
|
|
136
|
+
ErrorCode.EXTERNAL_SCANNER_FAILED,
|
|
137
|
+
f"外部扫描器 {scanner} 仅支持 SARIF 2.1.0 输出",
|
|
138
|
+
status="failed",
|
|
139
|
+
final_phase="local_validation",
|
|
140
|
+
)
|
|
141
|
+
runs = document.get("runs")
|
|
142
|
+
if not isinstance(runs, list):
|
|
143
|
+
raise FinalizerError(
|
|
144
|
+
ErrorCode.EXTERNAL_SCANNER_FAILED,
|
|
145
|
+
f"外部扫描器 {scanner} 缺少 SARIF runs 列表",
|
|
146
|
+
status="failed",
|
|
147
|
+
final_phase="local_validation",
|
|
148
|
+
)
|
|
149
|
+
findings: list[dict[str, Any]] = []
|
|
150
|
+
for run in runs:
|
|
151
|
+
if not isinstance(run, dict):
|
|
152
|
+
continue
|
|
153
|
+
results = run.get("results")
|
|
154
|
+
if not isinstance(results, list):
|
|
155
|
+
continue
|
|
156
|
+
for item in results:
|
|
157
|
+
if not isinstance(item, dict):
|
|
158
|
+
continue
|
|
159
|
+
suppressions = item.get("suppressions")
|
|
160
|
+
if isinstance(suppressions, list) and suppressions:
|
|
161
|
+
continue
|
|
162
|
+
kind = str(item.get("kind", "fail"))
|
|
163
|
+
if kind in {"pass", "notApplicable", "informational"}:
|
|
164
|
+
continue
|
|
165
|
+
level = str(item.get("level", "error")).lower()
|
|
166
|
+
if level not in {"error", "warning"}:
|
|
167
|
+
continue
|
|
168
|
+
rule_id = str(item.get("ruleId") or "sarif-finding")
|
|
169
|
+
path_str = _extract_sarif_path(item)
|
|
170
|
+
line_no = _extract_sarif_line(item)
|
|
171
|
+
findings.append(
|
|
172
|
+
{
|
|
173
|
+
"scanner": scanner,
|
|
174
|
+
"rule_id": rule_id[:128],
|
|
175
|
+
"level": level,
|
|
176
|
+
"path": path_str,
|
|
177
|
+
"line": line_no,
|
|
178
|
+
}
|
|
179
|
+
)
|
|
180
|
+
return findings
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def _extract_sarif_path(item: dict[str, Any]) -> str | None:
|
|
184
|
+
locations = item.get("locations")
|
|
185
|
+
if not isinstance(locations, list) or not locations:
|
|
186
|
+
return None
|
|
187
|
+
phys = locations[0].get("physicalLocation") if isinstance(locations[0], dict) else None
|
|
188
|
+
artifact = phys.get("artifactLocation") if isinstance(phys, dict) else None
|
|
189
|
+
uri = artifact.get("uri") if isinstance(artifact, dict) else None
|
|
190
|
+
return str(uri) if isinstance(uri, str) and uri else None
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _extract_sarif_line(item: dict[str, Any]) -> int | None:
|
|
194
|
+
locations = item.get("locations")
|
|
195
|
+
if not isinstance(locations, list) or not locations:
|
|
196
|
+
return None
|
|
197
|
+
phys = locations[0].get("physicalLocation") if isinstance(locations[0], dict) else None
|
|
198
|
+
region = phys.get("region") if isinstance(phys, dict) else None
|
|
199
|
+
line = region.get("startLine") if isinstance(region, dict) else None
|
|
200
|
+
return int(line) if isinstance(line, int) and line > 0 else None
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _parse_trufflehog_ndjson(raw_bytes: bytes) -> list[dict[str, Any]]:
|
|
204
|
+
findings: list[dict[str, Any]] = []
|
|
205
|
+
for raw_line in raw_bytes.decode("utf-8", errors="replace").splitlines():
|
|
206
|
+
line = raw_line.strip()
|
|
207
|
+
if not line:
|
|
208
|
+
continue
|
|
209
|
+
try:
|
|
210
|
+
entry = json.loads(line)
|
|
211
|
+
except json.JSONDecodeError as exc:
|
|
212
|
+
raise FinalizerError(
|
|
213
|
+
ErrorCode.EXTERNAL_SCANNER_FAILED,
|
|
214
|
+
f"trufflehog JSON 输出解析失败: {exc}",
|
|
215
|
+
status="failed",
|
|
216
|
+
final_phase="local_validation",
|
|
217
|
+
) from exc
|
|
218
|
+
if not isinstance(entry, dict) or "DetectorName" not in entry:
|
|
219
|
+
continue
|
|
220
|
+
meta = entry.get("SourceMetadata", {}).get("Data", {}).get("Filesystem", {})
|
|
221
|
+
findings.append(
|
|
222
|
+
{
|
|
223
|
+
"scanner": "trufflehog",
|
|
224
|
+
"rule_id": str(entry.get("DetectorName") or "trufflehog"),
|
|
225
|
+
"level": "error",
|
|
226
|
+
"path": str(meta.get("file")) if meta.get("file") else None,
|
|
227
|
+
"line": int(meta["line"]) if isinstance(meta.get("line"), int) else None,
|
|
228
|
+
}
|
|
229
|
+
)
|
|
230
|
+
return findings
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def run_external_scanners(
|
|
234
|
+
candidates: dict[str, bytes],
|
|
235
|
+
scanners: list[str],
|
|
236
|
+
policy: DeliveryPolicy,
|
|
237
|
+
*,
|
|
238
|
+
sarif_cmd: str | None = None,
|
|
239
|
+
timeout: float = 60.0,
|
|
240
|
+
) -> list[dict[str, Any]]:
|
|
241
|
+
"""Materialize validated candidates into a fresh temp tree and run configured SARIF scanners."""
|
|
242
|
+
if not scanners or not candidates:
|
|
243
|
+
return []
|
|
244
|
+
reports: list[dict[str, Any]] = []
|
|
245
|
+
with tempfile.TemporaryDirectory(prefix="gf-ext-scan-") as temporary:
|
|
246
|
+
root = Path(temporary)
|
|
247
|
+
target_dir = root / "tree"
|
|
248
|
+
target_dir.mkdir(mode=0o700)
|
|
249
|
+
for rel, data in candidates.items():
|
|
250
|
+
posix = PurePosixPath(rel)
|
|
251
|
+
if posix.is_absolute() or ".." in posix.parts:
|
|
252
|
+
raise FinalizerError(
|
|
253
|
+
ErrorCode.SCOPE_INVALID,
|
|
254
|
+
f"外部扫描器拒绝非法路径: {rel}",
|
|
255
|
+
final_phase="local_validation",
|
|
256
|
+
)
|
|
257
|
+
dest = target_dir / rel
|
|
258
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
259
|
+
dest.write_bytes(data)
|
|
260
|
+
|
|
261
|
+
all_findings: list[dict[str, Any]] = []
|
|
262
|
+
for idx, scanner in enumerate(scanners):
|
|
263
|
+
if scanner not in SUPPORTED_SCANNERS:
|
|
264
|
+
raise FinalizerError(
|
|
265
|
+
ErrorCode.CLI_INVALID_ARGS,
|
|
266
|
+
f"不支持的外部扫描器: {scanner}",
|
|
267
|
+
final_phase="cli",
|
|
268
|
+
)
|
|
269
|
+
report_path = root / f"report-{idx}.sarif.json"
|
|
270
|
+
argv = _resolve_scanner_argv(scanner, target_dir, report_path, policy, sarif_cmd=sarif_cmd)
|
|
271
|
+
try:
|
|
272
|
+
completed = subprocess.run(
|
|
273
|
+
argv,
|
|
274
|
+
capture_output=True,
|
|
275
|
+
timeout=timeout,
|
|
276
|
+
env={**os.environ, "LC_ALL": "C"},
|
|
277
|
+
close_fds=True,
|
|
278
|
+
check=False,
|
|
279
|
+
)
|
|
280
|
+
except (OSError, subprocess.SubprocessError) as exc:
|
|
281
|
+
raise FinalizerError(
|
|
282
|
+
ErrorCode.EXTERNAL_SCANNER_FAILED,
|
|
283
|
+
f"外部扫描器 {scanner} 执行失败: {exc}",
|
|
284
|
+
status="failed",
|
|
285
|
+
final_phase="local_validation",
|
|
286
|
+
) from exc
|
|
287
|
+
|
|
288
|
+
if scanner == "trufflehog":
|
|
289
|
+
if completed.returncode not in (0, 183):
|
|
290
|
+
raise FinalizerError(
|
|
291
|
+
ErrorCode.EXTERNAL_SCANNER_FAILED,
|
|
292
|
+
f"trufflehog 返回失败退出码 {completed.returncode}",
|
|
293
|
+
status="failed",
|
|
294
|
+
final_phase="local_validation",
|
|
295
|
+
)
|
|
296
|
+
findings = _parse_trufflehog_ndjson(completed.stdout)
|
|
297
|
+
else:
|
|
298
|
+
if completed.returncode != 0 and not (
|
|
299
|
+
scanner == "osv" and completed.returncode == 1 and report_path.is_file()
|
|
300
|
+
) and not (
|
|
301
|
+
scanner == "sarif-cmd" and report_path.is_file()
|
|
302
|
+
):
|
|
303
|
+
raise FinalizerError(
|
|
304
|
+
ErrorCode.EXTERNAL_SCANNER_FAILED,
|
|
305
|
+
f"外部扫描器 {scanner} 返回失败退出码 {completed.returncode}",
|
|
306
|
+
status="failed",
|
|
307
|
+
final_phase="local_validation",
|
|
308
|
+
)
|
|
309
|
+
raw_sarif = report_path.read_bytes() if report_path.is_file() else completed.stdout
|
|
310
|
+
findings = parse_sarif_findings(raw_sarif, scanner)
|
|
311
|
+
|
|
312
|
+
all_findings.extend(findings)
|
|
313
|
+
reports.append({"scanner": scanner, "findings_count": len(findings)})
|
|
314
|
+
|
|
315
|
+
if all_findings:
|
|
316
|
+
first = all_findings[0]
|
|
317
|
+
summary_msg = f"{first['scanner']}({first['rule_id']})"
|
|
318
|
+
if first.get("path"):
|
|
319
|
+
summary_msg += f" at {first['path']}"
|
|
320
|
+
raise FinalizerError(
|
|
321
|
+
ErrorCode.EXTERNAL_SCANNER_BLOCKED,
|
|
322
|
+
f"外部扫描器发现安全风险 ({len(all_findings)} 项): {summary_msg}",
|
|
323
|
+
final_phase="local_validation",
|
|
324
|
+
diagnostics={
|
|
325
|
+
"schema_version": 1,
|
|
326
|
+
"external_scanners": all_findings[:10],
|
|
327
|
+
},
|
|
328
|
+
)
|
|
329
|
+
return reports
|