git-finalizer 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- git_finalizer/__init__.py +8 -0
- git_finalizer/delivery/__init__.py +3 -0
- git_finalizer/delivery/cli.py +166 -0
- git_finalizer/delivery/content_scan.py +524 -0
- git_finalizer/delivery/engine.py +2364 -0
- git_finalizer/delivery/integration_publish.py +547 -0
- git_finalizer/delivery/parser.py +552 -0
- git_finalizer/delivery/patch_check.py +165 -0
- git_finalizer/delivery/policy.py +311 -0
- git_finalizer/delivery/repo_bootstrap.py +588 -0
- git_finalizer/delivery/repo_state.py +612 -0
- git_finalizer/delivery/retirement_plan.py +740 -0
- git_finalizer/delivery/runtime.py +553 -0
- git_finalizer/delivery/scanners.py +329 -0
- git_finalizer/delivery/snapshot_verify.py +425 -0
- git_finalizer/delivery/source_review.py +374 -0
- git_finalizer/delivery/trailers.py +164 -0
- git_finalizer/delivery/validation.py +1095 -0
- git_finalizer/errors.py +105 -0
- git_finalizer/git_io.py +281 -0
- git_finalizer/json_io.py +55 -0
- git_finalizer/toolchain/__init__.py +3 -0
- git_finalizer/toolchain/tool_skill_sync.py +3044 -0
- git_finalizer/toolchain/tool_temp_dir.py +1057 -0
- git_finalizer/toolchain/workflow_health.py +504 -0
- git_finalizer-2.0.0.dist-info/METADATA +383 -0
- git_finalizer-2.0.0.dist-info/RECORD +30 -0
- git_finalizer-2.0.0.dist-info/WHEEL +4 -0
- git_finalizer-2.0.0.dist-info/entry_points.txt +5 -0
- git_finalizer-2.0.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,524 @@
|
|
|
1
|
+
"""Sensitive-content, token, private-key, and unified binary scanning for Git Finalizer."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import ast
|
|
6
|
+
from collections import Counter
|
|
7
|
+
import json
|
|
8
|
+
from pathlib import Path, PurePosixPath
|
|
9
|
+
import re
|
|
10
|
+
import tempfile
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from git_finalizer.errors import ErrorCode, FinalizerError
|
|
14
|
+
from git_finalizer.git_io import clean_git_env, is_binary_bytes, run_git
|
|
15
|
+
from git_finalizer.json_io import canonical_json, sha256_bytes
|
|
16
|
+
|
|
17
|
+
MAX_SCAN_BYTES = 64 * 1024 * 1024
|
|
18
|
+
MAX_LITERAL_BYTES = 1024 * 1024
|
|
19
|
+
LITERAL = re.compile(rb'"(?:[^"\\]|\\.)*"')
|
|
20
|
+
DIRECTORY_ENV_KEY = re.compile(r"[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)*_CREDENTIALS")
|
|
21
|
+
SYSTEMD_DIRECTORY = re.compile(r"/run/credentials/[A-Za-z0-9][A-Za-z0-9_.@-]*\.service")
|
|
22
|
+
PRIVATE_KEY_END = re.compile(rb"-----END [^\r\n]*-----")
|
|
23
|
+
VERDICT_ORDER = ("PRIVATE", "KNOWN", "SSH", "ASSIGNMENT")
|
|
24
|
+
|
|
25
|
+
# Case-sensitive token signatures avoid uppercase false positives like SK-2026-... or TASK-...
|
|
26
|
+
KNOWN_TOKEN_PATTERNS: tuple[re.Pattern[bytes], ...] = (
|
|
27
|
+
re.compile(rb"(?<![A-Za-z0-9])(?:gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,})(?![A-Za-z0-9_-])"),
|
|
28
|
+
re.compile(rb"\b(?:AKIA|ASIA)[A-Z0-9]{16}\b"),
|
|
29
|
+
re.compile(rb"(?<![A-Za-z0-9_])sk-(?:proj-|ant-[a-z0-9-]+-)?[A-Za-z0-9_-]{20,}"),
|
|
30
|
+
re.compile(rb"(?<![A-Za-z0-9_])(?:sk|rk)_(?:live|test)_[0-9A-Za-z]{20,}"),
|
|
31
|
+
re.compile(rb"(?<![A-Za-z0-9_])whsec_[0-9A-Za-z]{20,}"),
|
|
32
|
+
re.compile(rb"(?<![A-Za-z0-9_])xox[baprs]-[0-9A-Za-z-]{10,}"),
|
|
33
|
+
re.compile(rb"(?<![A-Za-z0-9_])AIza[0-9A-Za-z_-]{20,}"),
|
|
34
|
+
re.compile(rb"(?<![A-Za-z0-9_])glpat-[A-Za-z0-9_-]{20,}"),
|
|
35
|
+
re.compile(rb"(?<![A-Za-z0-9_])pypi-[A-Za-z0-9_-]{32,}"),
|
|
36
|
+
re.compile(rb"(?<![A-Za-z0-9_])hf_[A-Za-z0-9]{20,}"),
|
|
37
|
+
re.compile(rb"\beyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b"),
|
|
38
|
+
re.compile(
|
|
39
|
+
rb"\b(?:postgres|postgresql|mysql|mongodb(?:\+srv)?|redis|rediss|amqp|amqps)://"
|
|
40
|
+
rb"[^:\s/@\"'<>`]+:[^@\s/\"'<>`]{4,}@[A-Za-z0-9._:-]+",
|
|
41
|
+
re.IGNORECASE,
|
|
42
|
+
),
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
HEADER_PATTERNS: tuple[tuple[str, re.Pattern[bytes]], ...] = (
|
|
46
|
+
("PRIVATE", re.compile(rb"-----BEGIN ([A-Z0-9]+[\x09-\x0d ]+)*PRIVATE KEY-----", re.I | re.M)),
|
|
47
|
+
(
|
|
48
|
+
"SSH",
|
|
49
|
+
re.compile(
|
|
50
|
+
rb"(^|[\x09-\x0d ])(?:ssh-(?:rsa|dss|ecdsa|ed25519)|ecdsa-sha2-nistp(?:256|384|521)|sk-ssh-ed25519@openssh\.com)[\x09-\x0d ]+[A-Za-z0-9+/=]{24,}",
|
|
51
|
+
re.I | re.M,
|
|
52
|
+
),
|
|
53
|
+
),
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
_KEY_ALT = (
|
|
57
|
+
rb"api[_-]?key|secret(?:[_-]?key)?|token|password|passwd|credentials?|"
|
|
58
|
+
rb"private[_-]?key|client[_-]?secret|access[_-]?token|auth[_-]?token|"
|
|
59
|
+
rb"refresh[_-]?token|signing[_-]?key|db[_-]?password"
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
QUOTED_ASSIGNMENT = re.compile(
|
|
63
|
+
rb"(?<![A-Za-z0-9_])['\"]?(?:[A-Za-z0-9_]*_)?(?:" + _KEY_ALT + rb")(?:_[A-Za-z0-9]+)*['\"]?"
|
|
64
|
+
rb"[\x09-\x0d ]*[:=][\x09-\x0d ]*(?:[bfrBuFrR]{1,2})?(?P<q>['\"`])(?P<val>(?:\\.|(?!(?P=q))[^\r\n]){12,})(?P=q)",
|
|
65
|
+
re.I,
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
UNQUOTED_ASSIGNMENT = re.compile(
|
|
69
|
+
rb"(?<![A-Za-z0-9_])(?:[A-Za-z0-9_]*_)?(?:" + _KEY_ALT + rb")(?:_[A-Za-z0-9]+)*"
|
|
70
|
+
rb"[ \t]*(?:=(?![=>])|:[ \t]+)(?P<val>[^\s'\"`#;,{}()\[\]]{12,})",
|
|
71
|
+
re.I,
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
VERDICT_DETECTOR = {
|
|
75
|
+
"PRIVATE": "private-key-header-v1",
|
|
76
|
+
"SSH": "ssh-key",
|
|
77
|
+
"KNOWN": "known-token-v1",
|
|
78
|
+
"ASSIGNMENT": "credential-assignment-v1",
|
|
79
|
+
"ERROR": "content-scan-error",
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def mask_systemd_directory_references(raw: bytes) -> bytes:
|
|
84
|
+
"""Classify narrow static Python credential directory references, without reading them."""
|
|
85
|
+
if b"/run/credentials/" not in raw:
|
|
86
|
+
return raw
|
|
87
|
+
try:
|
|
88
|
+
tree = ast.parse(raw.decode("utf-8"))
|
|
89
|
+
except (SyntaxError, UnicodeError, ValueError, RecursionError):
|
|
90
|
+
return raw
|
|
91
|
+
# A bare JSON-like expression is not a Python environment declaration.
|
|
92
|
+
declarations = (
|
|
93
|
+
ast.Assign,
|
|
94
|
+
ast.AnnAssign,
|
|
95
|
+
ast.FunctionDef,
|
|
96
|
+
ast.AsyncFunctionDef,
|
|
97
|
+
ast.ClassDef,
|
|
98
|
+
ast.Import,
|
|
99
|
+
ast.ImportFrom,
|
|
100
|
+
)
|
|
101
|
+
if not any(isinstance(node, declarations) for node in tree.body):
|
|
102
|
+
return raw
|
|
103
|
+
offsets = [0]
|
|
104
|
+
for line in raw.splitlines(keepends=True):
|
|
105
|
+
offsets.append(offsets[-1] + len(line))
|
|
106
|
+
|
|
107
|
+
def literal_span(node: ast.AST, text: str) -> tuple[int, int] | None:
|
|
108
|
+
if node.lineno != node.end_lineno or node.lineno is None or node.col_offset is None or node.end_col_offset is None:
|
|
109
|
+
return None
|
|
110
|
+
start = offsets[node.lineno - 1] + node.col_offset
|
|
111
|
+
end = offsets[node.end_lineno - 1] + node.end_col_offset
|
|
112
|
+
value = text.encode("ascii")
|
|
113
|
+
if raw[start:end] not in (b'"' + value + b'"', b"'" + value + b"'"):
|
|
114
|
+
return None
|
|
115
|
+
return start, end
|
|
116
|
+
|
|
117
|
+
def directory_span(node: ast.AST) -> tuple[int, int] | None:
|
|
118
|
+
if (
|
|
119
|
+
isinstance(node, ast.Constant)
|
|
120
|
+
and isinstance(node.value, str)
|
|
121
|
+
and SYSTEMD_DIRECTORY.fullmatch(node.value)
|
|
122
|
+
and len(node.value.removeprefix("/run/credentials/")) <= 255
|
|
123
|
+
):
|
|
124
|
+
return literal_span(node, node.value)
|
|
125
|
+
return None
|
|
126
|
+
|
|
127
|
+
spans: list[tuple[int, int]] = []
|
|
128
|
+
for mapping in ast.walk(tree):
|
|
129
|
+
if not isinstance(mapping, ast.Dict):
|
|
130
|
+
continue
|
|
131
|
+
if any(not isinstance(key, ast.Constant) or not isinstance(key.value, str) for key in mapping.keys):
|
|
132
|
+
continue
|
|
133
|
+
keys = [key.value for key in mapping.keys] # type: ignore[union-attr]
|
|
134
|
+
if len(keys) != len(set(keys)):
|
|
135
|
+
continue
|
|
136
|
+
for key, value in zip(mapping.keys, mapping.values, strict=True):
|
|
137
|
+
assert isinstance(key, ast.Constant) and isinstance(key.value, str)
|
|
138
|
+
if not (DIRECTORY_ENV_KEY.fullmatch(key.value) or key.value == "container_credentials"):
|
|
139
|
+
continue
|
|
140
|
+
span = directory_span(value)
|
|
141
|
+
if literal_span(key, key.value) and span:
|
|
142
|
+
spans.append(span)
|
|
143
|
+
for declaration in tree.body:
|
|
144
|
+
if (
|
|
145
|
+
isinstance(declaration, ast.Assign)
|
|
146
|
+
and len(declaration.targets) == 1
|
|
147
|
+
and isinstance(declaration.targets[0], ast.Name)
|
|
148
|
+
and declaration.targets[0].id == "CREDENTIALS"
|
|
149
|
+
):
|
|
150
|
+
span = directory_span(declaration.value)
|
|
151
|
+
if span:
|
|
152
|
+
spans.append(span)
|
|
153
|
+
for start, end in sorted(spans, reverse=True):
|
|
154
|
+
# Preserve byte positions and never mask adjacent comments or literals.
|
|
155
|
+
raw = raw[:start] + b'""' + b" " * (end - start - 2) + raw[end:]
|
|
156
|
+
return raw
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
|
160
|
+
result: dict[str, Any] = {}
|
|
161
|
+
for key, value in pairs:
|
|
162
|
+
if key in result:
|
|
163
|
+
raise ValueError("duplicate JSON key")
|
|
164
|
+
result[key] = value
|
|
165
|
+
return result
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _invalid_constant(_value: str) -> Any:
|
|
169
|
+
raise ValueError("non-JSON constant")
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _json_strings(value: Any):
|
|
173
|
+
if isinstance(value, str):
|
|
174
|
+
yield value.encode("utf-8")
|
|
175
|
+
elif isinstance(value, dict):
|
|
176
|
+
for key, item in value.items():
|
|
177
|
+
yield str(key).encode("utf-8")
|
|
178
|
+
yield from _json_strings(item)
|
|
179
|
+
elif isinstance(value, list):
|
|
180
|
+
for item in value:
|
|
181
|
+
yield from _json_strings(item)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _decoded_assignments(value: Any):
|
|
185
|
+
if isinstance(value, dict):
|
|
186
|
+
for key, item in value.items():
|
|
187
|
+
if isinstance(item, str) and len(item.encode("utf-8")) >= 12 and QUOTED_ASSIGNMENT.search(
|
|
188
|
+
str(key).encode("utf-8") + b'="AAAAAAAAAAAA"'
|
|
189
|
+
):
|
|
190
|
+
yield str(key).encode("utf-8") + b"=" + item.encode("utf-8")
|
|
191
|
+
yield from _decoded_assignments(item)
|
|
192
|
+
elif isinstance(value, list):
|
|
193
|
+
for item in value:
|
|
194
|
+
yield from _decoded_assignments(item)
|
|
195
|
+
elif isinstance(value, str):
|
|
196
|
+
encoded = value.encode("utf-8")
|
|
197
|
+
yield from (match.group() for match in QUOTED_ASSIGNMENT.finditer(encoded))
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _match_identity(verdict: str, source: bytes, match: re.Match[bytes]) -> bytes:
|
|
201
|
+
if verdict == "PRIVATE":
|
|
202
|
+
end = PRIVATE_KEY_END.search(source, match.end())
|
|
203
|
+
if end is None:
|
|
204
|
+
line_end = source.find(b"\n", match.end())
|
|
205
|
+
return source[match.start() : line_end if line_end >= 0 else len(source)]
|
|
206
|
+
return source[match.start() : end.end()]
|
|
207
|
+
return match.group()
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def _finding_identities(raw: bytes) -> list[tuple[str, bytes]]:
|
|
211
|
+
"""Return every `(verdict, identity_bytes)` match for exact baseline subtraction."""
|
|
212
|
+
if len(raw) > MAX_SCAN_BYTES:
|
|
213
|
+
raise ValueError(f"content exceeds scan byte budget ({MAX_SCAN_BYTES} bytes)")
|
|
214
|
+
if not raw or is_binary_bytes(raw):
|
|
215
|
+
return []
|
|
216
|
+
|
|
217
|
+
found: list[tuple[str, bytes]] = []
|
|
218
|
+
document: Any = None
|
|
219
|
+
valid_json = False
|
|
220
|
+
try:
|
|
221
|
+
document = json.loads(
|
|
222
|
+
raw.decode("utf-8"),
|
|
223
|
+
object_pairs_hook=_unique_object,
|
|
224
|
+
parse_constant=_invalid_constant,
|
|
225
|
+
)
|
|
226
|
+
decoded = list(_json_strings(document))
|
|
227
|
+
valid_json = True
|
|
228
|
+
except (ValueError, UnicodeError):
|
|
229
|
+
decoded = []
|
|
230
|
+
for match in LITERAL.finditer(raw):
|
|
231
|
+
if len(match.group()) > MAX_LITERAL_BYTES:
|
|
232
|
+
continue
|
|
233
|
+
try:
|
|
234
|
+
decoded.append(json.loads(match.group()).encode("utf-8"))
|
|
235
|
+
except (ValueError, UnicodeError):
|
|
236
|
+
continue
|
|
237
|
+
|
|
238
|
+
for value in (raw, *decoded):
|
|
239
|
+
for verdict, pattern in HEADER_PATTERNS:
|
|
240
|
+
found.extend((verdict, _match_identity(verdict, value, match)) for match in pattern.finditer(value))
|
|
241
|
+
for pattern in KNOWN_TOKEN_PATTERNS:
|
|
242
|
+
found.extend(("KNOWN", _match_identity("KNOWN", value, match)) for match in pattern.finditer(value))
|
|
243
|
+
|
|
244
|
+
masked = raw if valid_json else mask_systemd_directory_references(raw)
|
|
245
|
+
|
|
246
|
+
assignments = [match.group() for match in QUOTED_ASSIGNMENT.finditer(masked)]
|
|
247
|
+
assignments.extend(match.group() for match in UNQUOTED_ASSIGNMENT.finditer(masked))
|
|
248
|
+
if valid_json:
|
|
249
|
+
assignments.extend(_decoded_assignments(document))
|
|
250
|
+
else:
|
|
251
|
+
def decode_match(m: re.Match[bytes]) -> bytes:
|
|
252
|
+
if len(m.group()) > MAX_LITERAL_BYTES:
|
|
253
|
+
return m.group()
|
|
254
|
+
try:
|
|
255
|
+
return b'"' + json.loads(m.group()).encode("utf-8") + b'"'
|
|
256
|
+
except (ValueError, UnicodeError):
|
|
257
|
+
return m.group()
|
|
258
|
+
|
|
259
|
+
decoded_text = LITERAL.sub(decode_match, masked)
|
|
260
|
+
assignments.extend(match.group() for match in QUOTED_ASSIGNMENT.finditer(decoded_text))
|
|
261
|
+
assignments.extend(match.group() for match in UNQUOTED_ASSIGNMENT.finditer(decoded_text))
|
|
262
|
+
for value in decoded:
|
|
263
|
+
assignments.extend(match.group() for match in QUOTED_ASSIGNMENT.finditer(value))
|
|
264
|
+
assignments.extend(match.group() for match in UNQUOTED_ASSIGNMENT.finditer(value))
|
|
265
|
+
found.extend(("ASSIGNMENT", identity) for identity in assignments)
|
|
266
|
+
return found
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def findings(raw: bytes) -> dict[str, int]:
|
|
270
|
+
"""Count security findings by verdict for callers inspecting category totals."""
|
|
271
|
+
counts: dict[str, int] = {}
|
|
272
|
+
for verdict, _identity in _finding_identities(raw):
|
|
273
|
+
counts[verdict] = counts.get(verdict, 0) + 1
|
|
274
|
+
return counts
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def scan(raw: bytes, *, baseline: bytes | None = None) -> str:
|
|
278
|
+
"""Return highest-severity finding verdict (`CLEAR`, `PRIVATE`, `SSH`, `KNOWN`, `ASSIGNMENT`)."""
|
|
279
|
+
remaining = Counter(_finding_identities(raw))
|
|
280
|
+
if baseline is not None:
|
|
281
|
+
remaining -= Counter(_finding_identities(baseline))
|
|
282
|
+
verdicts = {verdict for (verdict, _id), count in remaining.items() if count > 0}
|
|
283
|
+
return next((verdict for verdict in VERDICT_ORDER if verdict in verdicts), "CLEAR")
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def scan_diagnostic(
|
|
287
|
+
raw: bytes,
|
|
288
|
+
*,
|
|
289
|
+
path: str,
|
|
290
|
+
stage: str,
|
|
291
|
+
blob_oid: str | None = None,
|
|
292
|
+
baseline: bytes | None = None,
|
|
293
|
+
) -> dict[str, Any] | None:
|
|
294
|
+
"""Scan bytes against optional baseline and return structured redacted refusal diagnostic if blocked."""
|
|
295
|
+
try:
|
|
296
|
+
verdict = scan(raw, baseline=baseline)
|
|
297
|
+
except ValueError:
|
|
298
|
+
verdict = "ERROR"
|
|
299
|
+
if verdict == "CLEAR":
|
|
300
|
+
return None
|
|
301
|
+
if verdict not in VERDICT_DETECTOR:
|
|
302
|
+
verdict = "ERROR"
|
|
303
|
+
return {
|
|
304
|
+
"path": path,
|
|
305
|
+
"stage": stage,
|
|
306
|
+
"verdict": verdict,
|
|
307
|
+
"detector": VERDICT_DETECTOR[verdict],
|
|
308
|
+
"byte_length": len(raw),
|
|
309
|
+
"binary": is_binary_bytes(raw),
|
|
310
|
+
"sha256": sha256_bytes(raw),
|
|
311
|
+
"blob_oid": blob_oid,
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def hash_candidate_clean(
|
|
316
|
+
repo: Path,
|
|
317
|
+
relative_path: str,
|
|
318
|
+
file_path: Path,
|
|
319
|
+
*,
|
|
320
|
+
env: dict[str, str] | None = None,
|
|
321
|
+
) -> tuple[str, bytes]:
|
|
322
|
+
"""Hash candidate worktree file through Git clean filter inside an isolated temporary object dir."""
|
|
323
|
+
posix = PurePosixPath(relative_path)
|
|
324
|
+
if posix.is_absolute() or ".." in posix.parts:
|
|
325
|
+
raise FinalizerError(ErrorCode.SCOPE_INVALID, f"非法路径: {relative_path}")
|
|
326
|
+
with tempfile.TemporaryDirectory(prefix="gf-scan-cand-") as temporary:
|
|
327
|
+
obj_dir = Path(temporary) / "objects"
|
|
328
|
+
obj_dir.mkdir(mode=0o700)
|
|
329
|
+
scan_env = clean_git_env(env)
|
|
330
|
+
scan_env["GIT_OBJECT_DIRECTORY"] = str(obj_dir)
|
|
331
|
+
hashed = run_git(
|
|
332
|
+
repo,
|
|
333
|
+
"hash-object",
|
|
334
|
+
"-w",
|
|
335
|
+
"--path",
|
|
336
|
+
relative_path,
|
|
337
|
+
"--",
|
|
338
|
+
str(file_path),
|
|
339
|
+
env=scan_env,
|
|
340
|
+
)
|
|
341
|
+
if hashed.returncode != 0:
|
|
342
|
+
raise FinalizerError(
|
|
343
|
+
ErrorCode.CONTENT_SCAN_BLOCKED,
|
|
344
|
+
f"候选文件 Git clean-filter 处理失败: {relative_path}",
|
|
345
|
+
final_phase="local_validation",
|
|
346
|
+
)
|
|
347
|
+
oid = hashed.stdout.decode("ascii").strip()
|
|
348
|
+
size_res = run_git(repo, "cat-file", "-s", oid, env=scan_env)
|
|
349
|
+
if size_res.returncode != 0:
|
|
350
|
+
raise FinalizerError(
|
|
351
|
+
ErrorCode.CONTENT_SCAN_BLOCKED,
|
|
352
|
+
f"无法读取候选对象大小: {relative_path}",
|
|
353
|
+
final_phase="local_validation",
|
|
354
|
+
)
|
|
355
|
+
size = int(size_res.stdout.decode("ascii").strip())
|
|
356
|
+
if size > MAX_SCAN_BYTES:
|
|
357
|
+
raise ValueError(f"candidate exceeds scan byte budget ({size} > {MAX_SCAN_BYTES})")
|
|
358
|
+
blob = run_git(repo, "cat-file", "blob", oid, env=scan_env)
|
|
359
|
+
if blob.returncode != 0:
|
|
360
|
+
raise FinalizerError(
|
|
361
|
+
ErrorCode.CONTENT_SCAN_BLOCKED,
|
|
362
|
+
f"无法读取候选对象内容: {relative_path}",
|
|
363
|
+
final_phase="local_validation",
|
|
364
|
+
)
|
|
365
|
+
return oid, blob.stdout
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def scan_changelog_header_preservation(
|
|
369
|
+
raw: bytes,
|
|
370
|
+
baseline_raw: bytes,
|
|
371
|
+
) -> bool:
|
|
372
|
+
"""Allow CHANGELOG.md ONLY when inline-backtick header examples match baseline byte-for-byte."""
|
|
373
|
+
if baseline_raw == raw:
|
|
374
|
+
return True
|
|
375
|
+
baseline_text = baseline_raw.decode("utf-8", errors="strict")
|
|
376
|
+
current_text = raw.decode("utf-8", errors="strict")
|
|
377
|
+
header_re = re.compile(r"`-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----`", re.IGNORECASE)
|
|
378
|
+
end_re = re.compile(r"-----END [A-Z0-9 ]*PRIVATE KEY-----", re.IGNORECASE)
|
|
379
|
+
if end_re.search(current_text):
|
|
380
|
+
return False
|
|
381
|
+
base_matches = list(header_re.finditer(baseline_text))
|
|
382
|
+
curr_matches = list(header_re.finditer(current_text))
|
|
383
|
+
if not base_matches or len(base_matches) != len(curr_matches):
|
|
384
|
+
return False
|
|
385
|
+
earliest_base = base_matches[0].start()
|
|
386
|
+
earliest_curr = curr_matches[0].start()
|
|
387
|
+
# Every character from the earliest baseline inline-backtick header to EOF must remain byte-for-byte identical.
|
|
388
|
+
if current_text[earliest_curr:] != baseline_text[earliest_base:]:
|
|
389
|
+
return False
|
|
390
|
+
# Prefix before earliest historical marker must have no private headers/secrets.
|
|
391
|
+
prefix_bytes = current_text[:earliest_curr].encode("utf-8")
|
|
392
|
+
return scan(prefix_bytes) == "CLEAR"
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def scan_published_prose_reasons(
|
|
396
|
+
raw: bytes,
|
|
397
|
+
baseline_raw: bytes,
|
|
398
|
+
) -> bool:
|
|
399
|
+
"""Allow `*_REASONS` module-level dict literals matching baseline raw key/value pairs."""
|
|
400
|
+
if len(raw) > 1024 * 1024 or len(baseline_raw) > 1024 * 1024:
|
|
401
|
+
return False
|
|
402
|
+
try:
|
|
403
|
+
base_text = baseline_raw.decode("utf-8", errors="strict")
|
|
404
|
+
curr_text = raw.decode("utf-8", errors="strict")
|
|
405
|
+
base_tree = ast.parse(base_text)
|
|
406
|
+
curr_tree = ast.parse(curr_text)
|
|
407
|
+
except (UnicodeDecodeError, SyntaxError):
|
|
408
|
+
return False
|
|
409
|
+
|
|
410
|
+
_PROSE_PAIR_RE = re.compile(r"[A-Za-z]+(?:[ -]+[A-Za-z]+){2,}")
|
|
411
|
+
_REASONS_NAME_RE = re.compile(r"[A-Z][A-Z0-9_]*_REASONS")
|
|
412
|
+
|
|
413
|
+
def reason_pairs(tree: ast.Module) -> set[tuple[str, str]]:
|
|
414
|
+
pairs: set[tuple[str, str]] = set()
|
|
415
|
+
for stmt in tree.body:
|
|
416
|
+
if (
|
|
417
|
+
isinstance(stmt, ast.Assign)
|
|
418
|
+
and len(stmt.targets) == 1
|
|
419
|
+
and isinstance(stmt.targets[0], ast.Name)
|
|
420
|
+
and _REASONS_NAME_RE.fullmatch(stmt.targets[0].id)
|
|
421
|
+
and isinstance(stmt.value, ast.Dict)
|
|
422
|
+
):
|
|
423
|
+
for key_node, val_node in zip(stmt.value.keys, stmt.value.values, strict=True):
|
|
424
|
+
if (
|
|
425
|
+
isinstance(key_node, ast.Constant)
|
|
426
|
+
and isinstance(val_node, ast.Constant)
|
|
427
|
+
and isinstance(key_node.value, str)
|
|
428
|
+
and isinstance(val_node.value, str)
|
|
429
|
+
and _PROSE_PAIR_RE.fullmatch(key_node.value)
|
|
430
|
+
and _PROSE_PAIR_RE.fullmatch(val_node.value)
|
|
431
|
+
):
|
|
432
|
+
pairs.add((key_node.value, val_node.value))
|
|
433
|
+
return pairs
|
|
434
|
+
|
|
435
|
+
base_pairs = reason_pairs(base_tree)
|
|
436
|
+
curr_pairs = reason_pairs(curr_tree)
|
|
437
|
+
if not base_pairs or not curr_pairs <= base_pairs:
|
|
438
|
+
return False
|
|
439
|
+
|
|
440
|
+
# Build sanitized copy of curr_text where matched published reason dict entries are blanked.
|
|
441
|
+
lines = curr_text.splitlines(keepends=True)
|
|
442
|
+
offsets = [0]
|
|
443
|
+
for line in lines:
|
|
444
|
+
offsets.append(offsets[-1] + len(line))
|
|
445
|
+
|
|
446
|
+
def char_span(node: ast.AST) -> tuple[int, int] | None:
|
|
447
|
+
if node.lineno is None or node.end_lineno is None or node.col_offset is None or node.end_col_offset is None:
|
|
448
|
+
return None
|
|
449
|
+
return offsets[node.lineno - 1] + node.col_offset, offsets[node.end_lineno - 1] + node.end_col_offset
|
|
450
|
+
|
|
451
|
+
spans_to_mask: list[tuple[int, int]] = []
|
|
452
|
+
seen_pairs: dict[tuple[str, str], int] = {}
|
|
453
|
+
base_counts: dict[tuple[str, str], int] = {}
|
|
454
|
+
for stmt in base_tree.body:
|
|
455
|
+
if (
|
|
456
|
+
isinstance(stmt, ast.Assign)
|
|
457
|
+
and len(stmt.targets) == 1
|
|
458
|
+
and isinstance(stmt.targets[0], ast.Name)
|
|
459
|
+
and _REASONS_NAME_RE.fullmatch(stmt.targets[0].id)
|
|
460
|
+
and isinstance(stmt.value, ast.Dict)
|
|
461
|
+
):
|
|
462
|
+
for k, v in zip(stmt.value.keys, stmt.value.values, strict=True):
|
|
463
|
+
if (
|
|
464
|
+
isinstance(k, ast.Constant)
|
|
465
|
+
and isinstance(v, ast.Constant)
|
|
466
|
+
and isinstance(k.value, str)
|
|
467
|
+
and isinstance(v.value, str)
|
|
468
|
+
and _PROSE_PAIR_RE.fullmatch(k.value)
|
|
469
|
+
and _PROSE_PAIR_RE.fullmatch(v.value)
|
|
470
|
+
):
|
|
471
|
+
pair = (k.value, v.value)
|
|
472
|
+
base_counts[pair] = base_counts.get(pair, 0) + 1
|
|
473
|
+
|
|
474
|
+
for stmt in curr_tree.body:
|
|
475
|
+
if (
|
|
476
|
+
isinstance(stmt, ast.Assign)
|
|
477
|
+
and len(stmt.targets) == 1
|
|
478
|
+
and isinstance(stmt.targets[0], ast.Name)
|
|
479
|
+
and _REASONS_NAME_RE.fullmatch(stmt.targets[0].id)
|
|
480
|
+
and isinstance(stmt.value, ast.Dict)
|
|
481
|
+
):
|
|
482
|
+
for k, v in zip(stmt.value.keys, stmt.value.values, strict=True):
|
|
483
|
+
if not (
|
|
484
|
+
isinstance(k, ast.Constant)
|
|
485
|
+
and isinstance(v, ast.Constant)
|
|
486
|
+
and isinstance(k.value, str)
|
|
487
|
+
and isinstance(v.value, str)
|
|
488
|
+
and _PROSE_PAIR_RE.fullmatch(k.value)
|
|
489
|
+
and _PROSE_PAIR_RE.fullmatch(v.value)
|
|
490
|
+
):
|
|
491
|
+
continue
|
|
492
|
+
pair = (k.value, v.value)
|
|
493
|
+
seen_pairs[pair] = seen_pairs.get(pair, 0) + 1
|
|
494
|
+
if seen_pairs[pair] > base_counts.get(pair, 0):
|
|
495
|
+
return False
|
|
496
|
+
s_k = char_span(k)
|
|
497
|
+
s_v = char_span(v)
|
|
498
|
+
if not s_k or not s_v:
|
|
499
|
+
return False
|
|
500
|
+
between = curr_text[s_k[1] : s_v[0]]
|
|
501
|
+
if "#" in between or any(c not in " \t\r\n:" for c in between):
|
|
502
|
+
return False
|
|
503
|
+
spans_to_mask.append((s_k[0], s_v[1]))
|
|
504
|
+
|
|
505
|
+
if not spans_to_mask:
|
|
506
|
+
return False
|
|
507
|
+
|
|
508
|
+
buf = list(curr_text)
|
|
509
|
+
for start, end in spans_to_mask:
|
|
510
|
+
for idx in range(start, end):
|
|
511
|
+
if buf[idx] not in ("\n", "\r"):
|
|
512
|
+
buf[idx] = " "
|
|
513
|
+
masked = "".join(buf).encode("utf-8")
|
|
514
|
+
return scan(masked) == "CLEAR"
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
def canonical_report_line(diagnostic: dict[str, Any] | None) -> str:
|
|
518
|
+
if diagnostic is None:
|
|
519
|
+
return "CLEAR"
|
|
520
|
+
return f"{diagnostic['verdict']}"
|
|
521
|
+
|
|
522
|
+
|
|
523
|
+
def format_content_scan_summary(diagnostic: dict[str, Any] | None) -> bytes:
|
|
524
|
+
return canonical_json(diagnostic)
|