pre-commit-bigfix 2.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pre_commit_bigfix/__init__.py +3 -0
- pre_commit_bigfix/bes_actionscript_lint_schclass.py +937 -0
- pre_commit_bigfix/bes_actionscript_validate_prefetch.py +1258 -0
- pre_commit_bigfix/bes_actionscript_validate_script.py +2871 -0
- pre_commit_bigfix/bes_common.py +271 -0
- pre_commit_bigfix/bes_conventions_check.py +3484 -0
- pre_commit_bigfix/bes_relevance_convert_group.py +407 -0
- pre_commit_bigfix/bes_relevance_lint.py +402 -0
- pre_commit_bigfix/bes_schema_validate.py +84 -0
- pre_commit_bigfix/bes_siterepo_validate_layout.py +1203 -0
- pre_commit_bigfix/run_all.py +305 -0
- pre_commit_bigfix/schclass.py +284 -0
- pre_commit_bigfix/schclass_data/ExpandedActionScript.schclass +412 -0
- pre_commit_bigfix/schclass_data/bigfix_overrides.schclass +43 -0
- pre_commit_bigfix/schclass_tokenizer.py +321 -0
- pre_commit_bigfix-2.0.1.dist-info/METADATA +821 -0
- pre_commit_bigfix-2.0.1.dist-info/RECORD +21 -0
- pre_commit_bigfix-2.0.1.dist-info/WHEEL +5 -0
- pre_commit_bigfix-2.0.1.dist-info/entry_points.txt +10 -0
- pre_commit_bigfix-2.0.1.dist-info/licenses/LICENSE +21 -0
- pre_commit_bigfix-2.0.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1258 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Pre-commit hook: validate BigFix prefetch lines in ActionScript.
|
|
3
|
+
|
|
4
|
+
Finds every prefetch line in every <ActionScript> body of a BES file (and, for
|
|
5
|
+
non-.bes/.ojo paths, in the whole file read as one raw ActionScript body) and
|
|
6
|
+
hands it to `validate_prefetch()` from the
|
|
7
|
+
[bigfix_prefetch](https://pypi.org/project/bigfix-prefetch/) package, which is
|
|
8
|
+
the reference implementation of what a valid prefetch is. Both spellings are
|
|
9
|
+
covered:
|
|
10
|
+
|
|
11
|
+
prefetch <name> sha1:<40> size:<n> <url> sha256:<64>
|
|
12
|
+
add prefetch item name=<name> sha1=<40> size=<n> url=<url> sha256=<64>
|
|
13
|
+
|
|
14
|
+
SCOPE: this hook only answers "is this prefetch line internally valid" --
|
|
15
|
+
size present and > 0, hash lengths right, sha256 present, sha1 present when the
|
|
16
|
+
line is a prefetch statement without a sha256. It is deliberately offline:
|
|
17
|
+
nothing here downloads the URL or checks that the hashes match the real file.
|
|
18
|
+
The line's overall shape and its http-vs-https scheme are W206/W207 in
|
|
19
|
+
bes-conventions-check, and its lexical validity is E300 in
|
|
20
|
+
bes-actionscript-lint-schclass -- three altitudes on the same line, all
|
|
21
|
+
intentional.
|
|
22
|
+
|
|
23
|
+
Checks:
|
|
24
|
+
E400 a prefetch line is invalid; the reason is the message
|
|
25
|
+
`validate_prefetch()` reported (bad size, wrong hash length, an
|
|
26
|
+
unparsable line, ...)
|
|
27
|
+
E401 a prefetch line has no sha256. Upstream treats sha256 as optional
|
|
28
|
+
unless asked; this hook treats it as mandatory, because in 2026 it
|
|
29
|
+
is what enhanced security requires. It is its own code so a repo
|
|
30
|
+
that still wants it optional can `--disable E401`.
|
|
31
|
+
E402 a prefetch downloads the retired unzip-5.52.exe from the BigFix
|
|
32
|
+
redist folder; unzip-6.0.exe is the current one. Auto-fixable.
|
|
33
|
+
E403 a prefetch line holds text its syntax does not define: a statement
|
|
34
|
+
allows only `prefetch <name>`, the `sha1:`/`sha256:`/`size:` fields
|
|
35
|
+
and the URL; a block item only `name=`/`sha1=`/`sha256=`/`size=`/
|
|
36
|
+
`url=`. The reference parser silently ignores the rest, so e.g. a
|
|
37
|
+
template's `vs_SSMS.exe; filename*=UTF-8''vs_SSMS.exe` validates
|
|
38
|
+
while the file is saved as `vs_SSMS.exe;`. Two shapes are
|
|
39
|
+
auto-fixable offline, having one meaning each: `sha2:`/`sha2=` in
|
|
40
|
+
front of a 64-hex value becomes `sha256`, and tokens directly
|
|
41
|
+
after the URL (an unencoded space in it) are joined with `%20`;
|
|
42
|
+
stray text anywhere else is left for a human
|
|
43
|
+
E404 an auto-fix pass (E403, E402, E401 or W407) was not written,
|
|
44
|
+
because it would make a schema-valid file fail BES.xsd validation;
|
|
45
|
+
the other passes are still written (--disable E404 writes it
|
|
46
|
+
anyway)
|
|
47
|
+
W400 the file is not parseable BES XML; skipped (advisory --
|
|
48
|
+
bes-schema-validate is the authority on file validity)
|
|
49
|
+
W402 a prefetch block item has no sha1; technically valid, but unusual
|
|
50
|
+
W403 an `add nohash prefetch item` line: hashless by definition, so it is
|
|
51
|
+
reported rather than validated -- the download cannot be verified
|
|
52
|
+
W404 --auto-fix-network was asked to add a sha256 and could not: the URL
|
|
53
|
+
would not download, or what came back did not match the size and
|
|
54
|
+
sha1 already on the line. The E401 stands; this says why the fix did
|
|
55
|
+
not land.
|
|
56
|
+
W405 a prefetch statement has no sha1. Current BigFix clients accept a
|
|
57
|
+
statement with sha256 alone, so this is valid but unusual --
|
|
58
|
+
previously this hook (following upstream) treated it as mandatory
|
|
59
|
+
and failed the line as E400 with a misleading "could not be parsed"
|
|
60
|
+
message; it is now its own advisory code, same footing as W402's
|
|
61
|
+
block-item case.
|
|
62
|
+
W406 a prefetch's declared name and its URL's file end in different
|
|
63
|
+
extensions (`<sha1>.msi` fetched from `ccsetup531.exe`). The
|
|
64
|
+
extension decides how the file is run or unpacked, so a mismatch
|
|
65
|
+
is usually a copy-paste slip. An extension is a trailing `.` + 1-4
|
|
66
|
+
characters with a letter in them; a name or URL without one, and
|
|
67
|
+
a server-side script URL (`download.php`), are not compared
|
|
68
|
+
W407 --auto-fix-network tried to switch an http prefetch URL to https
|
|
69
|
+
(W207 in bes-conventions-check) and did not: the https URL was
|
|
70
|
+
unreachable, redirected off https, reported a Content-Length other
|
|
71
|
+
than the line's size (so nothing was downloaded), or served a file
|
|
72
|
+
whose size/sha1/sha256 differ. The line stays on http.
|
|
73
|
+
|
|
74
|
+
E-codes are real issues and fail the hook. W-codes are advisory and do NOT
|
|
75
|
+
fail the hook unless --strict is given.
|
|
76
|
+
|
|
77
|
+
AUTO-FIXES. They sit behind two separate flags, because some touch the
|
|
78
|
+
network:
|
|
79
|
+
|
|
80
|
+
--auto-fix (E402, E403), on by default. An E403 `sha2:` for `sha256:`, or a
|
|
81
|
+
space in the URL, is rewritten as described under E403 above. The retired unzip-5.52.exe prefetch is
|
|
82
|
+
rewritten in place to the current unzip-6.0.exe one, in whichever spelling the
|
|
83
|
+
line already used -- the replacement is built by `prefetch_from_dictionary()`
|
|
84
|
+
from bigfix_prefetch, so its shape is the reference implementation's, not this
|
|
85
|
+
hook's idea of it. Purely offline: the current file's size and hashes are
|
|
86
|
+
constants here. The original download's *name* is kept (the rest of the
|
|
87
|
+
ActionScript refers to the file by that name, so renaming it here would break
|
|
88
|
+
the script); everything else -- url, size, sha1, sha256 -- becomes the current
|
|
89
|
+
file's.
|
|
90
|
+
|
|
91
|
+
--auto-fix-network (E401), OFF by default. A prefetch with no sha256 gets one,
|
|
92
|
+
which means downloading the file to hash it: there is no other way to learn a
|
|
93
|
+
sha256. The download is handed to `add_sha256_prefetch()` from bigfix_prefetch,
|
|
94
|
+
which streams the file, checks it against the size and sha1 already on the
|
|
95
|
+
line, and re-emits the prefetch with sha256 added; a download that does not
|
|
96
|
+
match, or does not happen, is W404 and the line is left alone. Because this
|
|
97
|
+
reaches out to whatever URLs the content names, it is opt-in, never the
|
|
98
|
+
default, and each URL is fetched at most once per run.
|
|
99
|
+
|
|
100
|
+
The same flag also switches an `http://` prefetch URL to `https://` (the fix
|
|
101
|
+
for bes-conventions-check's W207), but only once the https copy is proven to
|
|
102
|
+
be the same file: a HEAD request must report a Content-Length equal to the
|
|
103
|
+
line's size -- checked BEFORE anything is downloaded, so a mismatch costs no
|
|
104
|
+
download -- and must not redirect off https; then the file is downloaded
|
|
105
|
+
(capped one byte past that size) and its size and every hash on the line
|
|
106
|
+
(sha1 and/or sha256) must match. Only the URL's scheme changes. Anything else
|
|
107
|
+
is W407 and the line stays on http. A line with no literal size, no hash, a
|
|
108
|
+
`{...}` in it, or `add nohash prefetch item` is not attempted. `--disable
|
|
109
|
+
W407` turns this part off.
|
|
110
|
+
|
|
111
|
+
An auto-fixed file fails the hook so the change is reviewed and re-staged.
|
|
112
|
+
Nothing else is fixable: E400's right size and hashes are properties of the
|
|
113
|
+
real file, and a hook has no way to know which file was meant.
|
|
114
|
+
|
|
115
|
+
Lines whose values are not knowable until the agent runs are skipped: a line
|
|
116
|
+
containing a `{...}` relevance substitution (a dynamic prefetch) has no fixed
|
|
117
|
+
size or hash to check, and `//` comment lines are not commands. The raw
|
|
118
|
+
content of a `createfile until <MARKER>` block is file text, not ActionScript,
|
|
119
|
+
so a prefetch-looking line inside one is skipped too.
|
|
120
|
+
|
|
121
|
+
Only <ActionScript> elements with MIMEType application/x-Fixlet-Windows-Shell
|
|
122
|
+
(or no MIMEType, which defaults to it) are BigFix ActionScript; other bodies
|
|
123
|
+
are other languages and are skipped silently. Bodies are extracted with lxml,
|
|
124
|
+
so entities are decoded and adjacent CDATA sections merged, with lxml's
|
|
125
|
+
sourceline mapping issues back to file line numbers.
|
|
126
|
+
|
|
127
|
+
Usage:
|
|
128
|
+
bes_actionscript_validate_prefetch.py [--strict] [--disable E401]
|
|
129
|
+
[--auto-fix=yes|no]
|
|
130
|
+
[--auto-fix-network=yes|no]
|
|
131
|
+
[file ...]
|
|
132
|
+
|
|
133
|
+
With no file arguments, all *.bes files in the current folder and below are
|
|
134
|
+
checked.
|
|
135
|
+
|
|
136
|
+
A file whose prefetches knowingly do not meet these rules opts out of every
|
|
137
|
+
check here with `prefetch-ok` anywhere in it, e.g.
|
|
138
|
+
|
|
139
|
+
<!-- prefetch-ok -->
|
|
140
|
+
|
|
141
|
+
which is the same marker that opts out of the prefetch-shape warning (W206) in
|
|
142
|
+
bes-conventions-check: one judgement, one marker. The longer
|
|
143
|
+
|
|
144
|
+
<!-- pre-commit-skip: bes-actionscript-validate-prefetch -->
|
|
145
|
+
|
|
146
|
+
also skips the file, and matches the other hooks' spelling. To turn a check
|
|
147
|
+
off across a whole repo instead of file by file, disable its code in the hook
|
|
148
|
+
args -- `--disable E401` is the way back to sha256-optional.
|
|
149
|
+
|
|
150
|
+
Files that look like mustache templates (containing `{{ ... }}`) are skipped
|
|
151
|
+
silently: they are not real content until rendered.
|
|
152
|
+
|
|
153
|
+
Exit codes:
|
|
154
|
+
0 no E-code issues and nothing auto-fixed (and, without --strict,
|
|
155
|
+
regardless of warnings)
|
|
156
|
+
1 an E-code issue was found, a file was auto-fixed, or a warning was found
|
|
157
|
+
while --strict is set
|
|
158
|
+
"""
|
|
159
|
+
|
|
160
|
+
import argparse
|
|
161
|
+
import contextlib
|
|
162
|
+
import hashlib
|
|
163
|
+
import io
|
|
164
|
+
import os
|
|
165
|
+
import re
|
|
166
|
+
import socket
|
|
167
|
+
import sys
|
|
168
|
+
import urllib.request
|
|
169
|
+
import warnings
|
|
170
|
+
from urllib.parse import urlsplit
|
|
171
|
+
|
|
172
|
+
from bigfix_prefetch.prefetch import add_sha256_prefetch
|
|
173
|
+
from bigfix_prefetch.prefetch_from_dictionary import prefetch_from_dictionary
|
|
174
|
+
from bigfix_prefetch.prefetch_parse import parse_prefetch
|
|
175
|
+
from bigfix_prefetch.prefetch_validate import validate_prefetch
|
|
176
|
+
from lxml import etree
|
|
177
|
+
|
|
178
|
+
if __package__ in (None, ""): # run directly as a script, not as a module
|
|
179
|
+
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
180
|
+
|
|
181
|
+
# `_mask_heredocs` is shared with the sibling ActionScript hook on purpose:
|
|
182
|
+
# both need to know that the lines inside a `createfile until` block are file
|
|
183
|
+
# content rather than commands, and one implementation of that rule is enough.
|
|
184
|
+
from pre_commit_bigfix import bes_common
|
|
185
|
+
from pre_commit_bigfix.bes_actionscript_lint_schclass import _mask_heredocs
|
|
186
|
+
from pre_commit_bigfix.bes_common import MUSTACHE_RE, encode, read_source
|
|
187
|
+
|
|
188
|
+
SKIP_MARKER = "pre-commit-skip: bes-actionscript-validate-prefetch"
|
|
189
|
+
|
|
190
|
+
# The one opt-out marker (matched anywhere in the file text): a file whose
|
|
191
|
+
# prefetches are knowingly not up to standard opts out of the whole hook, not
|
|
192
|
+
# one code at a time. A repo that wants a check off everywhere disables the
|
|
193
|
+
# code instead, with --disable. NOTE: this is the same marker that opts out of
|
|
194
|
+
# W206 in bes-conventions-check, so a file carrying it opts out of both -- the
|
|
195
|
+
# two are the same judgement ("these prefetch lines are intentional"), and one
|
|
196
|
+
# marker is what was asked for.
|
|
197
|
+
PREFETCH_MARKER = "prefetch-ok"
|
|
198
|
+
|
|
199
|
+
KNOWN_CODES = frozenset(
|
|
200
|
+
[
|
|
201
|
+
"E400",
|
|
202
|
+
"E401",
|
|
203
|
+
"E402",
|
|
204
|
+
"E403",
|
|
205
|
+
"E404",
|
|
206
|
+
"W400",
|
|
207
|
+
"W402",
|
|
208
|
+
"W403",
|
|
209
|
+
"W404",
|
|
210
|
+
"W405",
|
|
211
|
+
"W406",
|
|
212
|
+
"W407",
|
|
213
|
+
]
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
# W406: a trailing `.` + 1-4 characters, at least one a letter, is taken as a
|
|
217
|
+
# file extension -- `.7z` and `.gz` are, a version like `/1.2` is not
|
|
218
|
+
EXTENSION_RE = re.compile(r"\.((?=[0-9]*[A-Za-z])[A-Za-z0-9]{1,4})$")
|
|
219
|
+
# server-side script endpoints: the URL's extension says nothing about the
|
|
220
|
+
# file served (`download.php?id=...`)
|
|
221
|
+
SCRIPT_EXTENSIONS = frozenset(
|
|
222
|
+
["php", "asp", "aspx", "ashx", "cgi", "jsp", "pl", "py", "do", "cfm"]
|
|
223
|
+
)
|
|
224
|
+
# spellings of the same file type
|
|
225
|
+
EXTENSION_ALIASES = {
|
|
226
|
+
"tgz": "gz",
|
|
227
|
+
"jpeg": "jpg",
|
|
228
|
+
"tiff": "tif",
|
|
229
|
+
"htm": "html",
|
|
230
|
+
"yaml": "yml",
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
# Seconds any one --auto-fix-network download may stall for. bigfix_prefetch
|
|
234
|
+
# calls urlopen() without a timeout, which can hang a commit indefinitely, so
|
|
235
|
+
# the socket default is set around the call and put back afterwards.
|
|
236
|
+
NETWORK_TIMEOUT = 60
|
|
237
|
+
|
|
238
|
+
# The retired redist download (E402). Matched on either scheme: what is out of
|
|
239
|
+
# date is the file, not the URL's scheme (the http-vs-https judgement is W207
|
|
240
|
+
# in bes-conventions-check), and BigFix content has shipped it both ways.
|
|
241
|
+
OUTDATED_UNZIP_RE = re.compile(
|
|
242
|
+
r"https?://software\.bigfix\.com/download/redist/unzip-5\.52\.exe",
|
|
243
|
+
re.IGNORECASE,
|
|
244
|
+
)
|
|
245
|
+
|
|
246
|
+
# The current unzip in the same folder, and the E402 auto-fix's replacement
|
|
247
|
+
# values. `file_name` is only the fallback: the fix keeps whatever name the
|
|
248
|
+
# original prefetch used, since the rest of the ActionScript refers to the
|
|
249
|
+
# downloaded file by that name.
|
|
250
|
+
CURRENT_UNZIP = {
|
|
251
|
+
"file_name": "unzip.exe",
|
|
252
|
+
"file_sha1": "84debf12767785cd9b43811022407de7413beb6f",
|
|
253
|
+
"file_size": "204800",
|
|
254
|
+
"download_url": "http://software.bigfix.com/download/redist/unzip-6.0.exe",
|
|
255
|
+
"file_sha256": ("2122557d350fd1c59fb0ef32125330bde673e9331eb9371b454c2ad2d82091ac"),
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
BES_EXTENSIONS = (".bes", ".ojo")
|
|
259
|
+
|
|
260
|
+
# the one MIMEType that IS BigFix ActionScript; a missing MIMEType defaults to
|
|
261
|
+
# it, every other value is some other language and holds no prefetch lines.
|
|
262
|
+
ACTIONSCRIPT_MIMETYPE = "application/x-Fixlet-Windows-Shell"
|
|
263
|
+
|
|
264
|
+
# `add nohash prefetch item` is the documented hashless form; it is matched
|
|
265
|
+
# before the hashed form so it is reported (W403), not validated.
|
|
266
|
+
NOHASH_PREFETCH = "add nohash prefetch item"
|
|
267
|
+
BLOCK_PREFETCH = "add prefetch item"
|
|
268
|
+
STATEMENT_PREFETCH = "prefetch "
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def find_prefetch_lines(body):
|
|
272
|
+
"""Yield (lineno, line, is_nohash) for each prefetch line in `body`.
|
|
273
|
+
|
|
274
|
+
Line numbers are local to the body, 1-based. Comment lines, lines holding
|
|
275
|
+
a `{...}` relevance substitution, and the raw content of a
|
|
276
|
+
`createfile until` block are not yielded -- see the module docstring.
|
|
277
|
+
"""
|
|
278
|
+
body = body.replace("\r\n", "\n").replace("\r", "\n")
|
|
279
|
+
masked, _issues = _mask_heredocs(body.split("\n"))
|
|
280
|
+
for lineno, line in enumerate(masked, start=1):
|
|
281
|
+
stripped = line.strip()
|
|
282
|
+
lowered = stripped.lower()
|
|
283
|
+
if not stripped or lowered.startswith("//"):
|
|
284
|
+
continue
|
|
285
|
+
is_nohash = lowered.startswith(NOHASH_PREFETCH)
|
|
286
|
+
if not is_nohash and not lowered.startswith(
|
|
287
|
+
(BLOCK_PREFETCH, STATEMENT_PREFETCH)
|
|
288
|
+
):
|
|
289
|
+
continue
|
|
290
|
+
if "{" in stripped:
|
|
291
|
+
continue # a dynamic prefetch: no fixed size or hash to check
|
|
292
|
+
yield lineno, stripped, is_nohash
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def is_outdated_unzip(line):
|
|
296
|
+
"""Say whether `line` downloads the retired redist unzip-5.52.exe."""
|
|
297
|
+
return bool(OUTDATED_UNZIP_RE.search(line))
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def current_unzip_prefetch(line):
|
|
301
|
+
"""Return the current unzip prefetch, spelled the way `line` is spelled.
|
|
302
|
+
|
|
303
|
+
The replacement string itself comes from `prefetch_from_dictionary()` in
|
|
304
|
+
bigfix_prefetch, so a block item gets the block spelling (`name=`/`sha1=`)
|
|
305
|
+
and a statement gets the statement one (`sha1:`/`size:`) exactly as the
|
|
306
|
+
reference implementation writes them. The original download's name is
|
|
307
|
+
carried over -- the surrounding ActionScript refers to the file by name, so
|
|
308
|
+
renaming it here would break the script -- and the fallback if the line is
|
|
309
|
+
too broken to parse a name out of is the canonical `unzip.exe`.
|
|
310
|
+
"""
|
|
311
|
+
lowered = line.strip().lower()
|
|
312
|
+
prefetch_type = "block" if lowered.startswith(BLOCK_PREFETCH) else "statement"
|
|
313
|
+
values = dict(CURRENT_UNZIP)
|
|
314
|
+
with warnings.catch_warnings():
|
|
315
|
+
# parse_prefetch warns about the very things E400/E401/W402 report;
|
|
316
|
+
# they are reported from validate_prefetch_line(), not from here.
|
|
317
|
+
warnings.simplefilter("ignore")
|
|
318
|
+
try:
|
|
319
|
+
parsed = parse_prefetch(line)
|
|
320
|
+
except (AttributeError, TypeError, ValueError):
|
|
321
|
+
parsed = {}
|
|
322
|
+
if parsed.get("file_name"):
|
|
323
|
+
values["file_name"] = parsed["file_name"]
|
|
324
|
+
return prefetch_from_dictionary(values, prefetch_type)
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def has_sha256(line):
|
|
328
|
+
"""Say whether `line` already carries a sha256 (of any length: E400 owns that)."""
|
|
329
|
+
return "sha256" in line.lower()
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
# Fed to validate_prefetch() only, to check the rest of a sha1-less statement
|
|
333
|
+
# through the reference implementation; never parsed for its value and never
|
|
334
|
+
# written to a file. See statement_missing_sha1()/_with_placeholder_sha1().
|
|
335
|
+
PLACEHOLDER_SHA1 = "0" * 40
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def statement_missing_sha1(line):
|
|
339
|
+
"""Say whether `line` is a prefetch *statement* with no sha1.
|
|
340
|
+
|
|
341
|
+
A prefetch block item with no sha1 is W402's business (upstream parses it
|
|
342
|
+
fine, sha1 just being optional there); a statement is different -- upstream
|
|
343
|
+
`parse_prefetch()` has no `try` around its `sha1:` regex for a statement,
|
|
344
|
+
so a sha1-less one raises `AttributeError` instead of reporting anything
|
|
345
|
+
useful. This is the pre-check that catches that case before it reaches
|
|
346
|
+
`validate_prefetch()`.
|
|
347
|
+
"""
|
|
348
|
+
lowered = line.strip().lower()
|
|
349
|
+
return lowered.startswith(STATEMENT_PREFETCH) and " sha1:" not in lowered
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def _with_placeholder_sha1(line):
|
|
353
|
+
"""Return `line` with a placeholder sha1 spliced in after the size field.
|
|
354
|
+
|
|
355
|
+
Lets a sha1-less statement be checked for every *other* defect (bad size,
|
|
356
|
+
wrong sha256 length, ...) through the real `validate_prefetch()` rather
|
|
357
|
+
than re-implementing its statement regexes here. The placeholder is never
|
|
358
|
+
parsed for its value, never reported in a message (messages are trimmed
|
|
359
|
+
to their first line by `_first_line()`, which drops the raw prefetch
|
|
360
|
+
upstream appends), and never written back to the file.
|
|
361
|
+
"""
|
|
362
|
+
return re.sub(r" size:", f" sha1:{PLACEHOLDER_SHA1} size:", line, count=1)
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def sha256_added_prefetch(line):
|
|
366
|
+
"""Download the prefetch's file and return the same line with sha256 added.
|
|
367
|
+
|
|
368
|
+
A thin wrapper over `add_sha256_prefetch()` from bigfix_prefetch (the
|
|
369
|
+
upstream examples/add_sha256_prefetch_string.py is exactly this call): it
|
|
370
|
+
streams the download, checks it against the size and sha1 the line already
|
|
371
|
+
claims, and re-emits the prefetch -- in the same spelling -- with sha256
|
|
372
|
+
added. Anything that goes wrong (no network, a 404, a download that does
|
|
373
|
+
not match the line) raises; the caller turns that into W404.
|
|
374
|
+
|
|
375
|
+
Three things are handled here that upstream does not:
|
|
376
|
+
* a socket timeout, so a hanging URL cannot hang a commit;
|
|
377
|
+
* upstream's progress printing and warnings, which are swallowed so they
|
|
378
|
+
cannot land in the middle of this hook's own output;
|
|
379
|
+
* the download's *name*, which upstream takes from the URL's basename --
|
|
380
|
+
the name the line already used is put back, since the rest of the
|
|
381
|
+
ActionScript refers to the downloaded file by that name.
|
|
382
|
+
"""
|
|
383
|
+
old_timeout = socket.getdefaulttimeout()
|
|
384
|
+
with warnings.catch_warnings(), contextlib.redirect_stdout(io.StringIO()):
|
|
385
|
+
warnings.simplefilter("ignore") # upstream warns about what E401 reports
|
|
386
|
+
socket.setdefaulttimeout(NETWORK_TIMEOUT)
|
|
387
|
+
try:
|
|
388
|
+
updated = add_sha256_prefetch(line)
|
|
389
|
+
finally:
|
|
390
|
+
socket.setdefaulttimeout(old_timeout)
|
|
391
|
+
if not updated or not has_sha256(updated):
|
|
392
|
+
raise ValueError("the regenerated prefetch still has no sha256")
|
|
393
|
+
parsed_old = parse_prefetch(line)
|
|
394
|
+
parsed_new = parse_prefetch(updated)
|
|
395
|
+
|
|
396
|
+
parsed_new["prefetch_type"] = parsed_old.get("prefetch_type", "statement")
|
|
397
|
+
if parsed_old.get("file_name"):
|
|
398
|
+
parsed_new["file_name"] = parsed_old["file_name"]
|
|
399
|
+
return prefetch_from_dictionary(parsed_new)
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def _first_line(message):
|
|
403
|
+
"""Return a warning message as one line (validate_prefetch appends the raw)."""
|
|
404
|
+
return str(message).strip().splitlines()[0].strip()
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def _is_missing(message, hash_name):
|
|
408
|
+
"""Say whether `message` is the "<hash_name> is missing" reason."""
|
|
409
|
+
lowered = message.lower()
|
|
410
|
+
return hash_name in lowered and "missing" in lowered
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def validate_prefetch_line(line):
|
|
414
|
+
"""Validate one prefetch line; return [(code, message)].
|
|
415
|
+
|
|
416
|
+
Wraps `validate_prefetch()` from bigfix_prefetch: its verdict is the
|
|
417
|
+
return value and its reasons are `warnings`, so the warnings are captured
|
|
418
|
+
and turned into this hook's codes. A missing sha256 is pulled out of the
|
|
419
|
+
verdict into its own E401 -- upstream treats it as optional-unless-asked,
|
|
420
|
+
this hook treats it as mandatory, and giving it a code of its own is what
|
|
421
|
+
lets `--disable E401` put it back to optional for a repo that wants that.
|
|
422
|
+
|
|
423
|
+
A sha1-less prefetch *statement* is a separate pre-check (W405): upstream
|
|
424
|
+
cannot even parse such a line (see `statement_missing_sha1()`), so it is
|
|
425
|
+
validated with a placeholder sha1 spliced in instead, to catch every
|
|
426
|
+
*other* defect through the real `validate_prefetch()` rather than one
|
|
427
|
+
reimplemented here.
|
|
428
|
+
"""
|
|
429
|
+
is_sha1_less_statement = statement_missing_sha1(line)
|
|
430
|
+
line_to_validate = _with_placeholder_sha1(line) if is_sha1_less_statement else line
|
|
431
|
+
with warnings.catch_warnings(record=True) as caught:
|
|
432
|
+
warnings.simplefilter("always") # the same reason may repeat per line
|
|
433
|
+
try:
|
|
434
|
+
valid = validate_prefetch(line_to_validate)
|
|
435
|
+
except (AttributeError, TypeError, ValueError) as err:
|
|
436
|
+
# validate_prefetch handles the unparsable-line case itself (its
|
|
437
|
+
# regexes returning None); this is the belt-and-braces path for a
|
|
438
|
+
# shape it does not, so one odd line cannot abort a whole run.
|
|
439
|
+
return [("E400", f"prefetch line could not be validated ({err})")]
|
|
440
|
+
messages = [_first_line(entry.message) for entry in caught]
|
|
441
|
+
|
|
442
|
+
issues = []
|
|
443
|
+
# a missing sha1 escalates sha256 to mandatory upstream, so the "sha256 is
|
|
444
|
+
# missing" reason can arrive as either a warning or the failure reason.
|
|
445
|
+
if any(_is_missing(message, "sha256") for message in messages):
|
|
446
|
+
issues.append(
|
|
447
|
+
(
|
|
448
|
+
"E401",
|
|
449
|
+
(
|
|
450
|
+
"prefetch has no sha256; it is technically optional, but "
|
|
451
|
+
"treated as mandatory here -- add one, or "
|
|
452
|
+
f"`{PREFETCH_MARKER}` if intentional"
|
|
453
|
+
),
|
|
454
|
+
)
|
|
455
|
+
)
|
|
456
|
+
if is_sha1_less_statement:
|
|
457
|
+
issues.append(
|
|
458
|
+
(
|
|
459
|
+
"W405",
|
|
460
|
+
(
|
|
461
|
+
"prefetch statement has no sha1; sha256 alone is accepted "
|
|
462
|
+
"by current BigFix clients, so this is valid but unusual "
|
|
463
|
+
f"-- add one, or `{PREFETCH_MARKER}` if intentional"
|
|
464
|
+
),
|
|
465
|
+
)
|
|
466
|
+
)
|
|
467
|
+
elif any(_is_missing(message, "sha1") for message in messages):
|
|
468
|
+
issues.append(
|
|
469
|
+
(
|
|
470
|
+
"W402",
|
|
471
|
+
(
|
|
472
|
+
"prefetch has no sha1, which is unusual though valid in a "
|
|
473
|
+
f"prefetch block; add `{PREFETCH_MARKER}` if intentional"
|
|
474
|
+
),
|
|
475
|
+
)
|
|
476
|
+
)
|
|
477
|
+
|
|
478
|
+
if not valid:
|
|
479
|
+
reasons = [m for m in messages if m.lower().startswith("error")] or messages
|
|
480
|
+
# the missing sha256 is already reported as E401 above; anything left
|
|
481
|
+
# is a separate defect and is what E400 is for.
|
|
482
|
+
reasons = [m for m in reasons if not _is_missing(m, "sha256")]
|
|
483
|
+
if reasons:
|
|
484
|
+
reported = "; ".join(dict.fromkeys(reasons))
|
|
485
|
+
issues.append(
|
|
486
|
+
(
|
|
487
|
+
"E400",
|
|
488
|
+
(
|
|
489
|
+
f"invalid prefetch: {reported}; add `{PREFETCH_MARKER}` "
|
|
490
|
+
"if intentional"
|
|
491
|
+
),
|
|
492
|
+
)
|
|
493
|
+
)
|
|
494
|
+
elif not issues: # failed for a reason nothing reported; do not swallow it
|
|
495
|
+
issues.append(
|
|
496
|
+
(
|
|
497
|
+
"E400",
|
|
498
|
+
(
|
|
499
|
+
"invalid prefetch: reason not reported; add "
|
|
500
|
+
f"`{PREFETCH_MARKER}` if intentional"
|
|
501
|
+
),
|
|
502
|
+
)
|
|
503
|
+
)
|
|
504
|
+
return issues
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
# E403: what a prefetch statement's fields after the name may be, and the
|
|
508
|
+
# keys an `add prefetch item` may carry
|
|
509
|
+
STATEMENT_FIELD_RE = re.compile(r"^(?:sha1|sha256|size):\S+$|://", re.IGNORECASE)
|
|
510
|
+
ITEM_FIELD_RE = re.compile(r"^(?:name|sha1|sha256|size|url)=", re.IGNORECASE)
|
|
511
|
+
# a trailing `// comment` after whitespace ends the line's syntax
|
|
512
|
+
TRAILING_COMMENT_RE = re.compile(r"\s//.*$")
|
|
513
|
+
ITEM_PREFIX_RE = re.compile(r"^\s*add\s+(?:nohash\s+)?prefetch\s+item\b", re.IGNORECASE)
|
|
514
|
+
|
|
515
|
+
|
|
516
|
+
def stray_prefetch_tokens(line):
|
|
517
|
+
"""E403: return the tokens of a prefetch line its syntax does not define.
|
|
518
|
+
|
|
519
|
+
A statement is `prefetch <name>` followed only by `sha1:`, `sha256:`,
|
|
520
|
+
`size:` fields and the URL; a block item only by `name=`, `sha1=`,
|
|
521
|
+
`sha256=`, `size=`, `url=`. The reference parser keeps what it
|
|
522
|
+
recognizes and silently ignores anything else -- so a template that
|
|
523
|
+
pastes extra text into the line (`vs_SSMS.exe; filename*=UTF-8''...`)
|
|
524
|
+
validates, and the file is saved under a name nothing else uses.
|
|
525
|
+
"""
|
|
526
|
+
line = TRAILING_COMMENT_RE.sub("", line) # a spaced `// comment` is valid
|
|
527
|
+
item = ITEM_PREFIX_RE.match(line)
|
|
528
|
+
if item:
|
|
529
|
+
return [
|
|
530
|
+
tok for tok in line[item.end() :].split() if not ITEM_FIELD_RE.match(tok)
|
|
531
|
+
]
|
|
532
|
+
tokens = line.split()[2:] # past `prefetch <name>`
|
|
533
|
+
return [tok for tok in tokens if not STATEMENT_FIELD_RE.search(tok)]
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
# E403 fix: `sha2` written for `sha256`, carrying a real sha256 value
|
|
537
|
+
FIELD_LIKE_RE = re.compile(r"^\w+[:=]")
|
|
538
|
+
SHA2_TYPO_RE = re.compile(r"(?<!\S)sha2([:=])([0-9A-Fa-f]{64})(?!\S)", re.IGNORECASE)
|
|
539
|
+
|
|
540
|
+
|
|
541
|
+
def _is_field(token, item):
|
|
542
|
+
"""Say whether `token` is a field the prefetch syntax defines."""
|
|
543
|
+
if item:
|
|
544
|
+
return bool(ITEM_FIELD_RE.match(token))
|
|
545
|
+
return bool(STATEMENT_FIELD_RE.search(token))
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
def stray_tokens_fixed(line):
|
|
549
|
+
"""E403: return `line` with its certain stray-text fixes applied, or None.
|
|
550
|
+
|
|
551
|
+
Two E403 shapes have exactly one meaning, and only these are rewritten:
|
|
552
|
+
`sha2:`/`sha2=` in front of a 64-hex value (a truncated `sha256`), and
|
|
553
|
+
tokens directly after the URL, before the next field -- an unencoded
|
|
554
|
+
space in the URL, joined back with `%20`. Stray text anywhere else (next
|
|
555
|
+
to the name, say) is left for a human, and a trailing `// comment` is
|
|
556
|
+
kept as it is.
|
|
557
|
+
"""
|
|
558
|
+
comment = TRAILING_COMMENT_RE.search(line)
|
|
559
|
+
core, tail = (
|
|
560
|
+
(line[: comment.start()], line[comment.start() :]) if comment else (line, "")
|
|
561
|
+
)
|
|
562
|
+
item = bool(ITEM_PREFIX_RE.match(core))
|
|
563
|
+
separator = "=" if item else ":"
|
|
564
|
+
core = SHA2_TYPO_RE.sub(
|
|
565
|
+
lambda m: (
|
|
566
|
+
f"sha256{separator}{m.group(2)}" if m.group(1) == separator else m.group(0)
|
|
567
|
+
),
|
|
568
|
+
core,
|
|
569
|
+
)
|
|
570
|
+
|
|
571
|
+
tokens = list(re.finditer(r"\S+", core))
|
|
572
|
+
# past `add [nohash] prefetch item` / `prefetch <name>`
|
|
573
|
+
first_field = len(ITEM_PREFIX_RE.match(core).group(0).split()) if item else 2
|
|
574
|
+
url_index = next(
|
|
575
|
+
(
|
|
576
|
+
index
|
|
577
|
+
for index in range(first_field, len(tokens))
|
|
578
|
+
if (
|
|
579
|
+
tokens[index].group(0).lower().startswith("url=")
|
|
580
|
+
if item
|
|
581
|
+
else "://" in tokens[index].group(0)
|
|
582
|
+
)
|
|
583
|
+
),
|
|
584
|
+
None,
|
|
585
|
+
)
|
|
586
|
+
if url_index is not None:
|
|
587
|
+
last = url_index
|
|
588
|
+
while last + 1 < len(tokens):
|
|
589
|
+
token = tokens[last + 1].group(0)
|
|
590
|
+
# a `key:value` / `key=value` token is a mistyped field, not URL text
|
|
591
|
+
if _is_field(token, item) or FIELD_LIKE_RE.match(token):
|
|
592
|
+
break
|
|
593
|
+
last += 1
|
|
594
|
+
if last > url_index:
|
|
595
|
+
start, end = tokens[url_index].start(), tokens[last].end()
|
|
596
|
+
span = core[start:end]
|
|
597
|
+
if "\t" not in span:
|
|
598
|
+
core = core[:start] + span.replace(" ", "%20") + core[end:]
|
|
599
|
+
|
|
600
|
+
new_line = core + tail
|
|
601
|
+
return new_line if new_line != line else None
|
|
602
|
+
|
|
603
|
+
|
|
604
|
+
def find_stray_fix_targets(raw, src, is_bes):
|
|
605
|
+
"""Return [(file_lineno, line, new_line)] for the E403 lines to rewrite.
|
|
606
|
+
|
|
607
|
+
`add nohash prefetch item` lines are included: joining a URL or fixing a
|
|
608
|
+
field name does not change what a hashless line does.
|
|
609
|
+
"""
|
|
610
|
+
if is_bes:
|
|
611
|
+
try:
|
|
612
|
+
bodies = list(_iter_actionscript_bodies(raw))
|
|
613
|
+
except etree.XMLSyntaxError:
|
|
614
|
+
return []
|
|
615
|
+
else:
|
|
616
|
+
bodies = [(1, src)]
|
|
617
|
+
targets = []
|
|
618
|
+
for sourceline, body in bodies:
|
|
619
|
+
for lineno, line, _is_nohash in find_prefetch_lines(body):
|
|
620
|
+
if stray_prefetch_tokens(line):
|
|
621
|
+
new_line = stray_tokens_fixed(line)
|
|
622
|
+
if new_line is not None:
|
|
623
|
+
targets.append((sourceline + lineno - 1, line, new_line))
|
|
624
|
+
return targets
|
|
625
|
+
|
|
626
|
+
|
|
627
|
+
def fix_stray_tokens(src, targets):
|
|
628
|
+
"""Apply `find_stray_fix_targets`' rewrites in place; return (src, fixed)."""
|
|
629
|
+
lines = src.split("\n")
|
|
630
|
+
fixed = []
|
|
631
|
+
for lineno, text, new_text in targets:
|
|
632
|
+
if _replace_on_line(lines, lineno, text, new_text):
|
|
633
|
+
fixed.append(
|
|
634
|
+
(
|
|
635
|
+
lineno,
|
|
636
|
+
"E403",
|
|
637
|
+
(
|
|
638
|
+
"rewrote stray prefetch text (`sha2` -> `sha256`, or a "
|
|
639
|
+
"space in the URL -> `%20`)"
|
|
640
|
+
),
|
|
641
|
+
)
|
|
642
|
+
)
|
|
643
|
+
return "\n".join(lines), fixed
|
|
644
|
+
|
|
645
|
+
|
|
646
|
+
# W207 network fix: the https copy of an http prefetch is checked before the
|
|
647
|
+
# URL is switched. Content-Length first, so nothing is downloaded unless the
|
|
648
|
+
# size already matches; then the download is capped at that size (plus one
|
|
649
|
+
# byte, to notice a longer file) and hashed.
|
|
650
|
+
DOWNLOAD_CHUNK = 65536
|
|
651
|
+
|
|
652
|
+
|
|
653
|
+
def https_head(url):
|
|
654
|
+
"""Return (final_url, content_length_or_None) for a HEAD of `url`.
|
|
655
|
+
|
|
656
|
+
Redirects are followed (urllib's default); the caller checks the final
|
|
657
|
+
URL is still https.
|
|
658
|
+
"""
|
|
659
|
+
request = urllib.request.Request(url, method="HEAD")
|
|
660
|
+
with urllib.request.urlopen(request, timeout=NETWORK_TIMEOUT) as response:
|
|
661
|
+
length = response.headers.get("Content-Length", "")
|
|
662
|
+
return response.geturl(), (int(length) if length.strip().isdigit() else None)
|
|
663
|
+
|
|
664
|
+
|
|
665
|
+
def https_hashes(url, limit):
|
|
666
|
+
"""Download `url` and return (size, sha1, sha256) of at most `limit`+1 bytes.
|
|
667
|
+
|
|
668
|
+
Reading stops one byte past `limit`, so a file larger than the prefetch
|
|
669
|
+
says is noticed (its size is wrong) without being streamed to the end.
|
|
670
|
+
"""
|
|
671
|
+
sha1, sha256, size = hashlib.sha1(), hashlib.sha256(), 0
|
|
672
|
+
with urllib.request.urlopen(url, timeout=NETWORK_TIMEOUT) as response:
|
|
673
|
+
if urlsplit(response.geturl()).scheme.lower() != "https":
|
|
674
|
+
raise ValueError(f"redirected to a non-https URL ({response.geturl()})")
|
|
675
|
+
while size <= limit:
|
|
676
|
+
chunk = response.read(min(DOWNLOAD_CHUNK, limit + 1 - size))
|
|
677
|
+
if not chunk:
|
|
678
|
+
break
|
|
679
|
+
size += len(chunk)
|
|
680
|
+
sha1.update(chunk)
|
|
681
|
+
sha256.update(chunk)
|
|
682
|
+
return size, sha1.hexdigest(), sha256.hexdigest()
|
|
683
|
+
|
|
684
|
+
|
|
685
|
+
def http_prefetch_facts(line):
|
|
686
|
+
"""Return (url, size, sha1, sha256) for an http prefetch W207 could fix.
|
|
687
|
+
|
|
688
|
+
None when the line is not a candidate: not http, no literal size, or no
|
|
689
|
+
hash at all to verify an https copy against.
|
|
690
|
+
"""
|
|
691
|
+
sha1_less = statement_missing_sha1(line)
|
|
692
|
+
to_parse = _with_placeholder_sha1(line) if sha1_less else line
|
|
693
|
+
with warnings.catch_warnings():
|
|
694
|
+
warnings.simplefilter("ignore")
|
|
695
|
+
try:
|
|
696
|
+
parsed = parse_prefetch(to_parse)
|
|
697
|
+
except (AttributeError, TypeError, ValueError, KeyError):
|
|
698
|
+
return None
|
|
699
|
+
url = parsed.get("download_url") or ""
|
|
700
|
+
size = str(parsed.get("file_size") or "")
|
|
701
|
+
sha1 = None if sha1_less else parsed.get("file_sha1")
|
|
702
|
+
sha256 = parsed.get("file_sha256")
|
|
703
|
+
if not url.lower().startswith("http://") or url not in line:
|
|
704
|
+
return None
|
|
705
|
+
if not size.isdigit() or not (sha1 or sha256):
|
|
706
|
+
return None
|
|
707
|
+
return url, int(size), sha1, sha256
|
|
708
|
+
|
|
709
|
+
|
|
710
|
+
def verified_https_url(url, size, sha1, sha256):
|
|
711
|
+
"""Return the https URL for `url` once its copy is proven identical.
|
|
712
|
+
|
|
713
|
+
Raises ValueError (or a network error) saying why when it is not: a
|
|
714
|
+
Content-Length that does not equal `size` -- checked before anything is
|
|
715
|
+
downloaded -- a redirect off https, or a size/sha1/sha256 mismatch.
|
|
716
|
+
"""
|
|
717
|
+
https_url = "https://" + url[len("http://") :]
|
|
718
|
+
final, length = https_head(https_url)
|
|
719
|
+
if urlsplit(final).scheme.lower() != "https":
|
|
720
|
+
raise ValueError(f"the https URL redirects to a non-https one ({final})")
|
|
721
|
+
if length != size:
|
|
722
|
+
raise ValueError(
|
|
723
|
+
f"the https copy's Content-Length ({length}) is not the prefetch's "
|
|
724
|
+
f"size ({size}); nothing was downloaded"
|
|
725
|
+
)
|
|
726
|
+
got_size, got_sha1, got_sha256 = https_hashes(https_url, size)
|
|
727
|
+
if got_size != size:
|
|
728
|
+
raise ValueError(f"the https copy is {got_size} bytes, not {size}")
|
|
729
|
+
if sha1 and got_sha1 != sha1.lower():
|
|
730
|
+
raise ValueError("the https copy's sha1 does not match the prefetch's")
|
|
731
|
+
if sha256 and got_sha256 != sha256.lower():
|
|
732
|
+
raise ValueError("the https copy's sha256 does not match the prefetch's")
|
|
733
|
+
return https_url
|
|
734
|
+
|
|
735
|
+
|
|
736
|
+
def find_https_fix_targets(raw, src, is_bes):
|
|
737
|
+
"""Return [(file_lineno, line, facts)] for the http prefetches W207 may fix."""
|
|
738
|
+
targets = []
|
|
739
|
+
for lineno, line in iter_prefetch_targets(raw, src, is_bes):
|
|
740
|
+
facts = http_prefetch_facts(line)
|
|
741
|
+
if facts:
|
|
742
|
+
targets.append((lineno, line, facts))
|
|
743
|
+
return targets
|
|
744
|
+
|
|
745
|
+
|
|
746
|
+
def fix_http_urls(src, targets, cache=None):
|
|
747
|
+
"""Switch each target's http URL to https once it is verified.
|
|
748
|
+
|
|
749
|
+
Returns (src, fixed, failed). `failed` is a list of W407s -- the line stayed http. `cache` maps a
|
|
750
|
+
prefetch's (url, size, sha1, sha256) to its verified https URL, or to the
|
|
751
|
+
reason it failed, so each URL is checked at most once per run.
|
|
752
|
+
"""
|
|
753
|
+
if cache is None:
|
|
754
|
+
cache = {}
|
|
755
|
+
lines = src.split("\n")
|
|
756
|
+
fixed = []
|
|
757
|
+
failed = []
|
|
758
|
+
for lineno, text, facts in targets:
|
|
759
|
+
key = ("https",) + facts
|
|
760
|
+
if key not in cache:
|
|
761
|
+
try:
|
|
762
|
+
cache[key] = (verified_https_url(*facts), None)
|
|
763
|
+
except Exception as err: # noqa: BLE001 -- same reasoning as E401's
|
|
764
|
+
cache[key] = (None, f"{type(err).__name__}: {err}")
|
|
765
|
+
https_url, reason = cache[key]
|
|
766
|
+
if https_url is None:
|
|
767
|
+
failed.append(
|
|
768
|
+
(
|
|
769
|
+
lineno,
|
|
770
|
+
"W407",
|
|
771
|
+
(
|
|
772
|
+
f"could not switch the prefetch URL to https ({reason}); "
|
|
773
|
+
"the line was left on http"
|
|
774
|
+
),
|
|
775
|
+
)
|
|
776
|
+
)
|
|
777
|
+
elif _replace_on_line(
|
|
778
|
+
lines, lineno, text, text.replace(facts[0], https_url, 1)
|
|
779
|
+
):
|
|
780
|
+
fixed.append(
|
|
781
|
+
(
|
|
782
|
+
lineno,
|
|
783
|
+
"W207",
|
|
784
|
+
(
|
|
785
|
+
"switched the prefetch URL to https after verifying the "
|
|
786
|
+
"https copy's size and hashes"
|
|
787
|
+
),
|
|
788
|
+
)
|
|
789
|
+
)
|
|
790
|
+
return "\n".join(lines), fixed, failed
|
|
791
|
+
|
|
792
|
+
|
|
793
|
+
def _extension(name):
|
|
794
|
+
"""Return `name`'s lowercased, alias-normalized extension, or None."""
|
|
795
|
+
match = EXTENSION_RE.search(name)
|
|
796
|
+
if not match:
|
|
797
|
+
return None
|
|
798
|
+
extension = match.group(1).lower()
|
|
799
|
+
return EXTENSION_ALIASES.get(extension, extension)
|
|
800
|
+
|
|
801
|
+
|
|
802
|
+
def extension_mismatch(line):
|
|
803
|
+
"""W406: return (name, url_file) if their extensions disagree, else None.
|
|
804
|
+
|
|
805
|
+
The URL's file is the last segment of its *path* (query and fragment
|
|
806
|
+
dropped; a bare host has none). Both must carry an extension -- a
|
|
807
|
+
sha1-named prefetch, a URL with none, or a server-side script endpoint
|
|
808
|
+
(`download.php`) says nothing either way -- and the comparison ignores
|
|
809
|
+
case and treats spellings like `tgz`/`gz` alike. A URL file that is the
|
|
810
|
+
declared name plus one more extension (`7z.dll.txt`, a common way to get
|
|
811
|
+
a binary past web filtering) is the right file, not a slip.
|
|
812
|
+
"""
|
|
813
|
+
to_parse = _with_placeholder_sha1(line) if statement_missing_sha1(line) else line
|
|
814
|
+
with warnings.catch_warnings():
|
|
815
|
+
warnings.simplefilter("ignore")
|
|
816
|
+
try:
|
|
817
|
+
parsed = parse_prefetch(to_parse)
|
|
818
|
+
except (AttributeError, TypeError, ValueError, KeyError):
|
|
819
|
+
return None
|
|
820
|
+
name, url = parsed.get("file_name"), parsed.get("download_url")
|
|
821
|
+
if not name or not url or "{" in name + url:
|
|
822
|
+
return None
|
|
823
|
+
url_file = urlsplit(url).path.rstrip("/").rsplit("/", 1)[-1]
|
|
824
|
+
name_ext, url_ext = _extension(name), _extension(url_file)
|
|
825
|
+
if not name_ext or not url_ext or url_ext in SCRIPT_EXTENSIONS:
|
|
826
|
+
return None
|
|
827
|
+
if url_file.lower().startswith(name.lower() + "."):
|
|
828
|
+
return None # `x.exe.txt`: the real name plus an extension to pass filters
|
|
829
|
+
return None if name_ext == url_ext else (name, url_file)
|
|
830
|
+
|
|
831
|
+
|
|
832
|
+
def validate_actionscript(body):
|
|
833
|
+
"""Validate every prefetch line in one ActionScript body.
|
|
834
|
+
|
|
835
|
+
Returns sorted [(lineno, code, message)] with line numbers local to the
|
|
836
|
+
body, 1-based.
|
|
837
|
+
"""
|
|
838
|
+
issues = []
|
|
839
|
+
for lineno, line, is_nohash in find_prefetch_lines(body):
|
|
840
|
+
if is_outdated_unzip(line):
|
|
841
|
+
fixable = "" if is_nohash else " (auto-fixable)"
|
|
842
|
+
issues.append(
|
|
843
|
+
(
|
|
844
|
+
lineno,
|
|
845
|
+
"E402",
|
|
846
|
+
(
|
|
847
|
+
"prefetch downloads the retired unzip-5.52.exe; "
|
|
848
|
+
f"{CURRENT_UNZIP['download_url']} is the current one"
|
|
849
|
+
f"{fixable}; add `{PREFETCH_MARKER}` if intentional"
|
|
850
|
+
),
|
|
851
|
+
)
|
|
852
|
+
)
|
|
853
|
+
stray = stray_prefetch_tokens(line)
|
|
854
|
+
if stray:
|
|
855
|
+
issues.append(
|
|
856
|
+
(
|
|
857
|
+
lineno,
|
|
858
|
+
"E403",
|
|
859
|
+
(
|
|
860
|
+
"prefetch line has text its syntax does not define ("
|
|
861
|
+
+ ", ".join(f"`{tok}`" for tok in stray)
|
|
862
|
+
+ "); it is silently ignored, and a stray token next "
|
|
863
|
+
"to the name usually means the name itself is wrong; "
|
|
864
|
+
f"add `{PREFETCH_MARKER}` if intentional"
|
|
865
|
+
),
|
|
866
|
+
)
|
|
867
|
+
)
|
|
868
|
+
mismatch = extension_mismatch(line)
|
|
869
|
+
if mismatch:
|
|
870
|
+
issues.append(
|
|
871
|
+
(
|
|
872
|
+
lineno,
|
|
873
|
+
"W406",
|
|
874
|
+
(
|
|
875
|
+
f'prefetch names the file "{mismatch[0]}" but the URL '
|
|
876
|
+
f'serves "{mismatch[1]}"; the extension decides how the '
|
|
877
|
+
"file is run or unpacked (msiexec vs a direct launch, "
|
|
878
|
+
"which extractor), so this is usually a copy-paste "
|
|
879
|
+
f"slip; add `{PREFETCH_MARKER}` if intentional"
|
|
880
|
+
),
|
|
881
|
+
)
|
|
882
|
+
)
|
|
883
|
+
if is_nohash:
|
|
884
|
+
issues.append(
|
|
885
|
+
(
|
|
886
|
+
lineno,
|
|
887
|
+
"W403",
|
|
888
|
+
(
|
|
889
|
+
"`add nohash prefetch item` downloads without a hash, so "
|
|
890
|
+
"the file cannot be verified and is not validated here; "
|
|
891
|
+
f"prefer a hashed prefetch, or add `{PREFETCH_MARKER}` if "
|
|
892
|
+
"intentional"
|
|
893
|
+
),
|
|
894
|
+
)
|
|
895
|
+
)
|
|
896
|
+
continue
|
|
897
|
+
for code, message in validate_prefetch_line(line):
|
|
898
|
+
issues.append((lineno, code, message))
|
|
899
|
+
return sorted(issues)
|
|
900
|
+
|
|
901
|
+
|
|
902
|
+
def _iter_actionscript_bodies(raw):
|
|
903
|
+
"""Yield (sourceline, body) for every BigFix ActionScript in a BES document.
|
|
904
|
+
|
|
905
|
+
Raises etree.XMLSyntaxError if the document does not parse; the callers
|
|
906
|
+
turn that into W400.
|
|
907
|
+
"""
|
|
908
|
+
root = etree.fromstring(raw)
|
|
909
|
+
for element in root.iter("ActionScript"):
|
|
910
|
+
mimetype = element.get("MIMEType")
|
|
911
|
+
if mimetype is not None and mimetype != ACTIONSCRIPT_MIMETYPE:
|
|
912
|
+
continue
|
|
913
|
+
yield element.sourceline, element.text or ""
|
|
914
|
+
|
|
915
|
+
|
|
916
|
+
def _validate_bes_xml(raw):
|
|
917
|
+
"""Validate prefetches in every ActionScript of a BES document."""
|
|
918
|
+
try:
|
|
919
|
+
bodies = list(_iter_actionscript_bodies(raw))
|
|
920
|
+
except etree.XMLSyntaxError as err:
|
|
921
|
+
return [(1, "W400", f"not parseable BES XML ({err}); skipping")]
|
|
922
|
+
issues = []
|
|
923
|
+
for sourceline, body in bodies:
|
|
924
|
+
for lineno, code, message in validate_actionscript(body):
|
|
925
|
+
issues.append((sourceline + lineno - 1, code, message))
|
|
926
|
+
return issues
|
|
927
|
+
|
|
928
|
+
|
|
929
|
+
def iter_prefetch_targets(raw, src, is_bes):
|
|
930
|
+
"""Yield (file_lineno, line) for every fixable prefetch line in a file.
|
|
931
|
+
|
|
932
|
+
Line numbers are 1-based into `src`. An `add nohash prefetch item` is left
|
|
933
|
+
out of both fixes: it is hashless on purpose, and giving it hashes would
|
|
934
|
+
change what the line does, not just which file it fetches.
|
|
935
|
+
"""
|
|
936
|
+
if is_bes:
|
|
937
|
+
try:
|
|
938
|
+
bodies = list(_iter_actionscript_bodies(raw))
|
|
939
|
+
except etree.XMLSyntaxError:
|
|
940
|
+
return # already reported as W400; nothing safe to rewrite
|
|
941
|
+
else:
|
|
942
|
+
bodies = [(1, src)]
|
|
943
|
+
|
|
944
|
+
for sourceline, body in bodies:
|
|
945
|
+
for lineno, line, is_nohash in find_prefetch_lines(body):
|
|
946
|
+
if not is_nohash:
|
|
947
|
+
yield sourceline + lineno - 1, line
|
|
948
|
+
|
|
949
|
+
|
|
950
|
+
def find_fix_targets(raw, src, is_bes):
|
|
951
|
+
"""Return [(file_lineno, line)] for the E402 lines that can be rewritten."""
|
|
952
|
+
return [
|
|
953
|
+
(lineno, line)
|
|
954
|
+
for lineno, line in iter_prefetch_targets(raw, src, is_bes)
|
|
955
|
+
if is_outdated_unzip(line)
|
|
956
|
+
]
|
|
957
|
+
|
|
958
|
+
|
|
959
|
+
def find_network_fix_targets(raw, src, is_bes):
|
|
960
|
+
"""Return [(file_lineno, line)] for the E401 lines a download could fix."""
|
|
961
|
+
return [
|
|
962
|
+
(lineno, line)
|
|
963
|
+
for lineno, line in iter_prefetch_targets(raw, src, is_bes)
|
|
964
|
+
if not has_sha256(line)
|
|
965
|
+
]
|
|
966
|
+
|
|
967
|
+
|
|
968
|
+
def _replace_on_line(lines, lineno, text, replacement):
|
|
969
|
+
"""Substitute `text` on 1-based `lineno` of `lines`; say whether it landed.
|
|
970
|
+
|
|
971
|
+
The prefetch text is replaced inside the file line rather than the line
|
|
972
|
+
being rebuilt, so indentation and anything sharing the line (a `<![CDATA[`
|
|
973
|
+
opener, say) survive. A target whose text is not found on its line -- which
|
|
974
|
+
entity decoding could in principle cause -- is left alone rather than
|
|
975
|
+
guessed at, and its issue still stands.
|
|
976
|
+
"""
|
|
977
|
+
if not 1 <= lineno <= len(lines):
|
|
978
|
+
return False
|
|
979
|
+
raw_line = lines[lineno - 1]
|
|
980
|
+
if text not in raw_line:
|
|
981
|
+
return False
|
|
982
|
+
lines[lineno - 1] = raw_line.replace(text, replacement, 1)
|
|
983
|
+
return True
|
|
984
|
+
|
|
985
|
+
|
|
986
|
+
def fix_outdated_unzip(src, targets):
|
|
987
|
+
"""Rewrite each target line to the current unzip prefetch; return (src, fixed)."""
|
|
988
|
+
lines = src.split("\n")
|
|
989
|
+
fixed = []
|
|
990
|
+
for lineno, text in targets:
|
|
991
|
+
if _replace_on_line(lines, lineno, text, current_unzip_prefetch(text)):
|
|
992
|
+
fixed.append(
|
|
993
|
+
(
|
|
994
|
+
lineno,
|
|
995
|
+
"E402",
|
|
996
|
+
(
|
|
997
|
+
"replaced the retired unzip-5.52.exe prefetch with "
|
|
998
|
+
f"{CURRENT_UNZIP['download_url']}"
|
|
999
|
+
),
|
|
1000
|
+
)
|
|
1001
|
+
)
|
|
1002
|
+
return "\n".join(lines), fixed
|
|
1003
|
+
|
|
1004
|
+
|
|
1005
|
+
def fix_missing_sha256(src, targets, cache=None):
|
|
1006
|
+
"""Add a sha256 to each target line by downloading it; return (src, fixed, failed).
|
|
1007
|
+
|
|
1008
|
+
`failed` is a list of W404s: the line stayed as it was and its E401 still
|
|
1009
|
+
stands. `cache` maps a prefetch line to the rewritten one (or to None for a
|
|
1010
|
+
line whose download already failed), so a URL repeated across a run is
|
|
1011
|
+
fetched once -- pass the same dict to every call to share it.
|
|
1012
|
+
"""
|
|
1013
|
+
if cache is None:
|
|
1014
|
+
cache = {}
|
|
1015
|
+
lines = src.split("\n")
|
|
1016
|
+
fixed = []
|
|
1017
|
+
failed = []
|
|
1018
|
+
for lineno, text in targets:
|
|
1019
|
+
if text not in cache:
|
|
1020
|
+
try:
|
|
1021
|
+
cache[text] = sha256_added_prefetch(text)
|
|
1022
|
+
except Exception as err: # noqa: BLE001 -- see below
|
|
1023
|
+
# urlopen alone raises URLError, HTTPError, socket.timeout, ssl
|
|
1024
|
+
# errors...; upstream adds AttributeError/TypeError/ValueError
|
|
1025
|
+
# for a download that does not match the line. Every one of them
|
|
1026
|
+
# means the same thing here -- no sha256 was learned -- and none
|
|
1027
|
+
# of them may take the whole run down.
|
|
1028
|
+
cache[text] = None
|
|
1029
|
+
failed.append((lineno, "W404", f"{type(err).__name__}: {err}"))
|
|
1030
|
+
continue
|
|
1031
|
+
replacement = cache[text]
|
|
1032
|
+
if replacement is None:
|
|
1033
|
+
failed.append((lineno, "W404", "the download already failed above"))
|
|
1034
|
+
elif _replace_on_line(lines, lineno, text, replacement):
|
|
1035
|
+
fixed.append((lineno, "E401", "downloaded the file and added its sha256"))
|
|
1036
|
+
failed = [
|
|
1037
|
+
(
|
|
1038
|
+
lineno,
|
|
1039
|
+
code,
|
|
1040
|
+
(
|
|
1041
|
+
f"could not add a sha256 by download ({reason}); the line was "
|
|
1042
|
+
"left as it is and its E401 stands"
|
|
1043
|
+
),
|
|
1044
|
+
)
|
|
1045
|
+
for lineno, code, reason in failed
|
|
1046
|
+
]
|
|
1047
|
+
return "\n".join(lines), fixed, failed
|
|
1048
|
+
|
|
1049
|
+
|
|
1050
|
+
def check_file( # pylint: disable=too-many-locals,too-many-arguments,too-many-positional-arguments
|
|
1051
|
+
path,
|
|
1052
|
+
disabled=frozenset(),
|
|
1053
|
+
strict=False,
|
|
1054
|
+
auto_fix=False,
|
|
1055
|
+
auto_fix_network=False,
|
|
1056
|
+
network_cache=None,
|
|
1057
|
+
):
|
|
1058
|
+
"""Check one file; return (issues, fixed), each a list of (lineno, code, msg).
|
|
1059
|
+
|
|
1060
|
+
With `auto_fix`, the certain E403 stray-text shapes are fixed and the
|
|
1061
|
+
retired unzip-5.52.exe prefetches (E402) are rewritten in place to the
|
|
1062
|
+
current unzip-6.0.exe one. With `auto_fix_network` -- which
|
|
1063
|
+
downloads files, and so is never the default -- a prefetch with no sha256
|
|
1064
|
+
(E401) gets one, or a W404 saying why it could not, and an http URL is
|
|
1065
|
+
switched to https once verified (W207), or a W407 says why not;
|
|
1066
|
+
`network_cache` is a
|
|
1067
|
+
dict shared across files so a repeated URL is fetched once. Both report
|
|
1068
|
+
under `fixed`, and the file's line endings are preserved (CRLF in, CRLF
|
|
1069
|
+
out). `strict` is accepted for parity with the sibling hooks and does not
|
|
1070
|
+
change what is reported (the caller decides whether warnings fail).
|
|
1071
|
+
"""
|
|
1072
|
+
del strict # reported issues are the same either way
|
|
1073
|
+
if not os.path.isfile(path):
|
|
1074
|
+
return [(1, "W400", "file not found; skipping")], []
|
|
1075
|
+
|
|
1076
|
+
original, src, was_crlf = read_source(path)
|
|
1077
|
+
|
|
1078
|
+
if SKIP_MARKER in src:
|
|
1079
|
+
return [], []
|
|
1080
|
+
if MUSTACHE_RE.search(src):
|
|
1081
|
+
return [], []
|
|
1082
|
+
|
|
1083
|
+
is_bes = path.endswith(BES_EXTENSIONS)
|
|
1084
|
+
opted_out = PREFETCH_MARKER in src
|
|
1085
|
+
|
|
1086
|
+
raw = original
|
|
1087
|
+
# each fix pass is kept only if it leaves a schema-valid file valid (E404)
|
|
1088
|
+
guard = bes_common.SchemaGuard(
|
|
1089
|
+
src, "E404", path, validate=is_bes and "E404" not in disabled
|
|
1090
|
+
)
|
|
1091
|
+
failed = []
|
|
1092
|
+
if not opted_out and auto_fix and "E403" not in disabled:
|
|
1093
|
+
new_src, got = fix_stray_tokens(src, find_stray_fix_targets(raw, src, is_bes))
|
|
1094
|
+
if guard.apply(new_src, got):
|
|
1095
|
+
src, raw = new_src, encode(new_src, was_crlf)
|
|
1096
|
+
if not opted_out and auto_fix and "E402" not in disabled:
|
|
1097
|
+
new_src, got = fix_outdated_unzip(src, find_fix_targets(raw, src, is_bes))
|
|
1098
|
+
if guard.apply(new_src, got):
|
|
1099
|
+
src, raw = new_src, encode(new_src, was_crlf)
|
|
1100
|
+
if not opted_out and auto_fix_network and "E401" not in disabled:
|
|
1101
|
+
# re-found on the current text: an E402 fix above may already have
|
|
1102
|
+
# brought a sha256 with it, leaving nothing here to download.
|
|
1103
|
+
new_src, got, failed = fix_missing_sha256(
|
|
1104
|
+
src, find_network_fix_targets(raw, src, is_bes), network_cache
|
|
1105
|
+
)
|
|
1106
|
+
if guard.apply(new_src, got):
|
|
1107
|
+
src, raw = new_src, encode(new_src, was_crlf)
|
|
1108
|
+
if not opted_out and auto_fix_network and "W407" not in disabled:
|
|
1109
|
+
# after E401, so a sha256 just added is one more hash to verify against
|
|
1110
|
+
new_src, got, https_failed = fix_http_urls(
|
|
1111
|
+
src, find_https_fix_targets(raw, src, is_bes), network_cache
|
|
1112
|
+
)
|
|
1113
|
+
failed += https_failed
|
|
1114
|
+
if guard.apply(new_src, got):
|
|
1115
|
+
src, raw = new_src, encode(new_src, was_crlf)
|
|
1116
|
+
fixed = guard.fixed
|
|
1117
|
+
if raw != original:
|
|
1118
|
+
with open(path, "wb") as handle:
|
|
1119
|
+
handle.write(raw)
|
|
1120
|
+
|
|
1121
|
+
if is_bes:
|
|
1122
|
+
issues = _validate_bes_xml(raw)
|
|
1123
|
+
else:
|
|
1124
|
+
issues = validate_actionscript(src)
|
|
1125
|
+
issues += failed + guard.refused
|
|
1126
|
+
|
|
1127
|
+
issues = [
|
|
1128
|
+
(lineno, code, message)
|
|
1129
|
+
for lineno, code, message in issues
|
|
1130
|
+
if code not in disabled and not opted_out
|
|
1131
|
+
]
|
|
1132
|
+
return sorted(issues), fixed
|
|
1133
|
+
|
|
1134
|
+
|
|
1135
|
+
def check_files(
|
|
1136
|
+
paths, disabled=frozenset(), strict=False, auto_fix=False, auto_fix_network=False
|
|
1137
|
+
):
|
|
1138
|
+
"""Check several files; return a list of (path, issues, fixed) tuples.
|
|
1139
|
+
|
|
1140
|
+
This is the programmatic entry point: it does no printing. The download
|
|
1141
|
+
cache is made here, so a URL repeated across the files is fetched once.
|
|
1142
|
+
"""
|
|
1143
|
+
network_cache = {}
|
|
1144
|
+
return [
|
|
1145
|
+
(
|
|
1146
|
+
path,
|
|
1147
|
+
*check_file(
|
|
1148
|
+
path,
|
|
1149
|
+
disabled=disabled,
|
|
1150
|
+
strict=strict,
|
|
1151
|
+
auto_fix=auto_fix,
|
|
1152
|
+
auto_fix_network=auto_fix_network,
|
|
1153
|
+
network_cache=network_cache,
|
|
1154
|
+
),
|
|
1155
|
+
)
|
|
1156
|
+
for path in paths
|
|
1157
|
+
]
|
|
1158
|
+
|
|
1159
|
+
|
|
1160
|
+
def discover_bes_files(root="."):
|
|
1161
|
+
"""Return all .bes files under `root`, pruning hidden and noise directories."""
|
|
1162
|
+
return bes_common.discover_bes_files(root)
|
|
1163
|
+
|
|
1164
|
+
|
|
1165
|
+
def main(argv=None):
|
|
1166
|
+
"""Execution starts here.
|
|
1167
|
+
|
|
1168
|
+
argv defaults to None so this works both as a console_scripts entry point
|
|
1169
|
+
(pre-commit calls it with no arguments; argparse then reads sys.argv) and
|
|
1170
|
+
when called directly as `main(sys.argv[1:])`.
|
|
1171
|
+
"""
|
|
1172
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
1173
|
+
parser.add_argument(
|
|
1174
|
+
"--strict",
|
|
1175
|
+
action="store_true",
|
|
1176
|
+
help="treat warnings as failures (non-zero exit); default: advisory",
|
|
1177
|
+
)
|
|
1178
|
+
parser.add_argument(
|
|
1179
|
+
"--auto-fix",
|
|
1180
|
+
choices=["yes", "no"],
|
|
1181
|
+
default=None,
|
|
1182
|
+
help=(
|
|
1183
|
+
"rewrite the retired unzip-5.52.exe prefetches (E402) and fix "
|
|
1184
|
+
"`sha2:` / a space in the URL (E403) in place "
|
|
1185
|
+
"(default: yes when files are given, no when auto-discovering)"
|
|
1186
|
+
),
|
|
1187
|
+
)
|
|
1188
|
+
parser.add_argument(
|
|
1189
|
+
"--auto-fix-network",
|
|
1190
|
+
choices=["yes", "no"],
|
|
1191
|
+
default="no",
|
|
1192
|
+
help=(
|
|
1193
|
+
"add a sha256 to the prefetches that have none (E401) by "
|
|
1194
|
+
"DOWNLOADING each file to hash it, and switch http prefetch URLs "
|
|
1195
|
+
"to https once the https copy's size and hashes match (W207); off "
|
|
1196
|
+
"by default, since it fetches whatever URLs the content names"
|
|
1197
|
+
),
|
|
1198
|
+
)
|
|
1199
|
+
parser.add_argument(
|
|
1200
|
+
"--disable",
|
|
1201
|
+
default="",
|
|
1202
|
+
metavar="CODES",
|
|
1203
|
+
help=(
|
|
1204
|
+
"comma-separated check IDs to skip entirely, e.g. --disable E401 "
|
|
1205
|
+
"for a repo that still treats sha256 as optional"
|
|
1206
|
+
),
|
|
1207
|
+
)
|
|
1208
|
+
parser.add_argument(
|
|
1209
|
+
"files",
|
|
1210
|
+
nargs="*",
|
|
1211
|
+
help=(
|
|
1212
|
+
"files to check (.bes/.ojo are read as BES XML, anything else as "
|
|
1213
|
+
"raw ActionScript); if omitted, all *.bes files in the current "
|
|
1214
|
+
"folder and below are checked"
|
|
1215
|
+
),
|
|
1216
|
+
)
|
|
1217
|
+
args = parser.parse_args(argv)
|
|
1218
|
+
|
|
1219
|
+
disabled = {
|
|
1220
|
+
code.strip().upper() for code in args.disable.split(",") if code.strip()
|
|
1221
|
+
}
|
|
1222
|
+
unknown = disabled - KNOWN_CODES
|
|
1223
|
+
if unknown:
|
|
1224
|
+
print(
|
|
1225
|
+
f"warning: ignoring unknown --disable code(s): {', '.join(sorted(unknown))}"
|
|
1226
|
+
)
|
|
1227
|
+
|
|
1228
|
+
# auto-fix defaults to yes for explicit files, no when auto-discovering; an
|
|
1229
|
+
# explicit --auto-fix always wins. This is the sibling hooks' rule, and
|
|
1230
|
+
# pre-commit always passes files, so under pre-commit the default is yes.
|
|
1231
|
+
if args.auto_fix is not None:
|
|
1232
|
+
auto_fix = args.auto_fix == "yes"
|
|
1233
|
+
else:
|
|
1234
|
+
auto_fix = bool(args.files)
|
|
1235
|
+
paths = args.files if args.files else discover_bes_files(".")
|
|
1236
|
+
|
|
1237
|
+
issue_count, warning_count, fix_count = bes_common.report(
|
|
1238
|
+
check_files(
|
|
1239
|
+
paths,
|
|
1240
|
+
disabled=disabled,
|
|
1241
|
+
strict=args.strict,
|
|
1242
|
+
auto_fix=auto_fix,
|
|
1243
|
+
auto_fix_network=args.auto_fix_network == "yes",
|
|
1244
|
+
)
|
|
1245
|
+
)
|
|
1246
|
+
|
|
1247
|
+
if fix_count:
|
|
1248
|
+
print(f"\nauto-fixed {fix_count} issue(s); review and re-stage the changes.")
|
|
1249
|
+
if warning_count:
|
|
1250
|
+
print(f"{warning_count} prefetch warning(s).")
|
|
1251
|
+
if issue_count:
|
|
1252
|
+
print(f"{issue_count} prefetch issue(s).")
|
|
1253
|
+
# E-codes and any fix always fail; warnings fail only under --strict
|
|
1254
|
+
return 1 if (issue_count or fix_count or (warning_count and args.strict)) else 0
|
|
1255
|
+
|
|
1256
|
+
|
|
1257
|
+
if __name__ == "__main__":
|
|
1258
|
+
sys.exit(main(sys.argv[1:]))
|