pre-commit-bigfix 2.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pre_commit_bigfix/__init__.py +3 -0
- pre_commit_bigfix/bes_actionscript_lint_schclass.py +937 -0
- pre_commit_bigfix/bes_actionscript_validate_prefetch.py +1258 -0
- pre_commit_bigfix/bes_actionscript_validate_script.py +2871 -0
- pre_commit_bigfix/bes_common.py +271 -0
- pre_commit_bigfix/bes_conventions_check.py +3484 -0
- pre_commit_bigfix/bes_relevance_convert_group.py +407 -0
- pre_commit_bigfix/bes_relevance_lint.py +402 -0
- pre_commit_bigfix/bes_schema_validate.py +84 -0
- pre_commit_bigfix/bes_siterepo_validate_layout.py +1203 -0
- pre_commit_bigfix/run_all.py +305 -0
- pre_commit_bigfix/schclass.py +284 -0
- pre_commit_bigfix/schclass_data/ExpandedActionScript.schclass +412 -0
- pre_commit_bigfix/schclass_data/bigfix_overrides.schclass +43 -0
- pre_commit_bigfix/schclass_tokenizer.py +321 -0
- pre_commit_bigfix-2.0.1.dist-info/METADATA +821 -0
- pre_commit_bigfix-2.0.1.dist-info/RECORD +21 -0
- pre_commit_bigfix-2.0.1.dist-info/WHEEL +5 -0
- pre_commit_bigfix-2.0.1.dist-info/entry_points.txt +10 -0
- pre_commit_bigfix-2.0.1.dist-info/licenses/LICENSE +21 -0
- pre_commit_bigfix-2.0.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,3484 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Pre-commit hook: check BigFix BES files for opinionated conventions.
|
|
3
|
+
|
|
4
|
+
This is the BES-content companion to the `bes-schema-validate` hook, which
|
|
5
|
+
only checks that a file is well-formed XML that satisfies the BES.xsd schema --
|
|
6
|
+
it says nothing about the *content* being conventional or correct. This tool
|
|
7
|
+
goes further, with PICKY, OPINIONATED checks (several AUTO-FIXABLE) that the XSD
|
|
8
|
+
cannot express.
|
|
9
|
+
|
|
10
|
+
A BES file is XML rooted at <BES> holding one or more content objects: Task,
|
|
11
|
+
Fixlet, Analysis, ComputerGroup, Baseline, SingleAction, ... The checks are
|
|
12
|
+
scoped by what actually makes sense for each: value-format checks apply wherever
|
|
13
|
+
the field appears; presence checks apply only to Task/Fixlet (the "publishable"
|
|
14
|
+
content -- Analysis and ComputerGroup legitimately have no SourceReleaseDate or
|
|
15
|
+
modification time).
|
|
16
|
+
|
|
17
|
+
Checks:
|
|
18
|
+
E200 an <ActionScript> MIMEType is missing or not one of the allowed set
|
|
19
|
+
(fixable -> a missing one is added as the default
|
|
20
|
+
application/x-Fixlet-Windows-Shell, and an allowed one in the
|
|
21
|
+
wrong case is respelled; any other value is left alone)
|
|
22
|
+
E201 a <SourceReleaseDate> is present but not in YYYY-MM-DD format
|
|
23
|
+
(fixable only when unambiguous -> a year-first `2026-7-4`,
|
|
24
|
+
`2026/07/14`, `2026.07.14`, or `2026-07-14T10:20:30Z` is rewritten
|
|
25
|
+
as YYYY-MM-DD; a month/day-first date is never guessed at)
|
|
26
|
+
E202 an x-fixlet-modification-time value is not a valid RFC 5322 date-time
|
|
27
|
+
(e.g. `Tue, 14 Jul 2026 18:32:35 +0000`, or `14 Jul 2026 18:32:35
|
|
28
|
+
+0000` without the optional day-of-week) -- see "Timestamp fields"
|
|
29
|
+
below for exactly what that requires (fixable when only the
|
|
30
|
+
formatting is off -> day/hour zero-padded, month and weekday
|
|
31
|
+
capitalized or abbreviated, a GMT/UTC/UT/Z zone or `+HH:MM` offset
|
|
32
|
+
written `+0000`/`+HHMM`, and a day-of-week that is wrong for the
|
|
33
|
+
date DROPPED, not recomputed; an unparsable value is left alone)
|
|
34
|
+
E203 a <DownloadSize> is not empty and not 0-or-a-positive-integer (fixable
|
|
35
|
+
-> 0)
|
|
36
|
+
E204 a content object's <Description> still contains the boilerplate
|
|
37
|
+
placeholder "enter a description of the" (case-insensitive), e.g.
|
|
38
|
+
"Enter a description of the Task here." or "...of the Analysis here."
|
|
39
|
+
E205 an x-fixlet-cpe23-item-name value is not a valid CPE 2.3 string
|
|
40
|
+
E206 an action-ui-metadata value is not well-formed
|
|
41
|
+
E207 a Description / Relevance / ActionScript entity-escapes a character
|
|
42
|
+
(< > &) that requires <![CDATA[ ... ]]> instead (fixable -> unescaped
|
|
43
|
+
and CDATA-wrapped)
|
|
44
|
+
E208 the file is not CRLF throughout -- BES files must use CRLF line
|
|
45
|
+
endings (fixable -> the whole file is normalized to CRLF)
|
|
46
|
+
E209 a <CVENames> value is not a valid CVE id (e.g. CVE-2021-44228), or
|
|
47
|
+
more than one <CVENames> element is present in a single content
|
|
48
|
+
object -- multiple CVE ids in one <CVENames> may be separated by
|
|
49
|
+
commas, semicolons, and/or whitespace, and a lone `Unspecified` is
|
|
50
|
+
accepted as the "no CVEs" sentinel (fixable -> a lowercase or
|
|
51
|
+
mixed-case `cve-` prefix is uppercased; several <CVENames> are NOT
|
|
52
|
+
merged, and any other bad value is left alone)
|
|
53
|
+
E210 two <MIMEField> entries in one content object share the same <Name>
|
|
54
|
+
E211 a <Title> is a default placeholder ("Custom Fixlet"/"Custom Task"/
|
|
55
|
+
"Custom Baseline"/"Custom Analysis")
|
|
56
|
+
E212 a <Relevance> is the literal `true` (case-insensitive) -- it targets
|
|
57
|
+
every endpoint. Comments and wrapping parentheses are ignored, and
|
|
58
|
+
a top-level `true OR ...` operand counts too: it is always true,
|
|
59
|
+
whatever the rest says (typically a test hack left in)
|
|
60
|
+
E213 a <Relevance> is empty or whitespace only
|
|
61
|
+
E214 the file has no XML declaration, or its declaration does not specify
|
|
62
|
+
encoding="UTF-8" (fixable -> declaration inserted / encoding set)
|
|
63
|
+
E215 an <ActionScript> has whitespace before its close tag, e.g. an
|
|
64
|
+
indented `]]></ActionScript>` (CDATA body) or a bare indented
|
|
65
|
+
`</ActionScript>` (non-CDATA body) -- that whitespace is inside the
|
|
66
|
+
ActionScript and becomes a bogus whitespace-only last line of the
|
|
67
|
+
action (fixable -> stripped to a flush `]]></ActionScript>` or
|
|
68
|
+
`</ActionScript>`)
|
|
69
|
+
E216 an x-fixlet-first-propagation value is not a valid RFC 5322 date-time
|
|
70
|
+
-- the same rule E202 applies to x-fixlet-modification-time, applied
|
|
71
|
+
to this separate field (see "Timestamp fields" below; fixable the
|
|
72
|
+
same way as E202)
|
|
73
|
+
E217 a <SuccessCriteria Option="CustomRelevance"> body is empty or the
|
|
74
|
+
literal `false` (can never succeed), or a non-CustomRelevance
|
|
75
|
+
<SuccessCriteria> has a non-empty body (silently ignored by BigFix)
|
|
76
|
+
E218 two <Action ID="..."> / <DefaultAction ID="..."> in one content
|
|
77
|
+
object share the same ID
|
|
78
|
+
E219 an x-relevance-evaluation-period value is not a valid HH:MM:SS
|
|
79
|
+
duration (matched case-insensitively; MM and SS must each be 00-59;
|
|
80
|
+
fixable -> zero-padded, `6:5:0` -> `06:05:00`, when MM and SS are
|
|
81
|
+
already below 60)
|
|
82
|
+
E220 two <Property> entries in one <Analysis> share a Name or an ID --
|
|
83
|
+
reporting cannot tell two same-named properties apart, and the API
|
|
84
|
+
addresses a property by its ID
|
|
85
|
+
E221 a <Description>'s <script> block uses a `// comment` style
|
|
86
|
+
JavaScript comment (fixable -> rewritten as `/* comment */`). When a
|
|
87
|
+
BES file is converted to FXF (the path Prop-tool / Git-based sites
|
|
88
|
+
take), newlines inside the Description are collapsed onto one line;
|
|
89
|
+
everything after a `//` on that collapsed line -- including real
|
|
90
|
+
code that follows it -- is then silently swallowed as part of the
|
|
91
|
+
comment. A `/* ... */` block comment survives the collapse. Only
|
|
92
|
+
`//` outside a string/template literal, regex literal, and existing
|
|
93
|
+
`/* */` block comment is flagged; a comment whose text itself
|
|
94
|
+
contains `*/` cannot be wrapped in a single block comment and is
|
|
95
|
+
left as an unfixed error
|
|
96
|
+
E222 a group of auto-fixes was not written, because it would make a
|
|
97
|
+
schema-valid file fail BES.xsd validation; the groups are all the
|
|
98
|
+
per-block fixes, the trailing-whitespace strip, and the XML
|
|
99
|
+
declaration, and the other groups are still written (--disable
|
|
100
|
+
E222 writes it anyway)
|
|
101
|
+
W200 the file is not parseable BES XML; skipped (advisory --
|
|
102
|
+
bes-schema-validate is the authority on file validity)
|
|
103
|
+
W201 a Task/Fixlet has no x-fixlet-modification-time MIMEField (fixable ->
|
|
104
|
+
the moment the linter ran)
|
|
105
|
+
W202 a Task/Fixlet has no <SourceReleaseDate> (fixable -> today)
|
|
106
|
+
W203 a Task/Fixlet has DownloadSize > 0 but no download/prefetch keyword in
|
|
107
|
+
any ActionScript
|
|
108
|
+
W204 an <ActionScript> body is not wrapped in <![CDATA[ ... ]]> (fixable,
|
|
109
|
+
but ONLY under --strict, since wrapping already-escaped content can
|
|
110
|
+
change its meaning)
|
|
111
|
+
W205 an <ActionScript> has more than one blank line before </ActionScript>
|
|
112
|
+
(fixable -> collapsed to one)
|
|
113
|
+
W206 a prefetch / "add prefetch item" line does not match the expected
|
|
114
|
+
shape; a prefetch statement's sha1 is optional when it has a sha256
|
|
115
|
+
(current BigFix clients accept a sha256-only statement). The URL
|
|
116
|
+
scheme must be one of http/https/SWDProtocol/RedHatProtocol
|
|
117
|
+
(case-insensitive) -- override with --prefetch-url-schemes
|
|
118
|
+
W207 a prefetch / "add prefetch item" URL is not https
|
|
119
|
+
W208 an <ActionScript> body is empty (only blank lines and //-comments)
|
|
120
|
+
W209 a <Title> has leading/trailing whitespace/newlines, embedded tabs,
|
|
121
|
+
or an internal run of 2+ spaces -- the last is usually where a
|
|
122
|
+
substitution expanded to nothing (fixable -> trimmed, tabs replaced
|
|
123
|
+
with spaces, repeated spaces collapsed to one)
|
|
124
|
+
W210 a line has trailing whitespace (fixable -> stripped)
|
|
125
|
+
W211 an <ActionScript> uses a dynamic `download` statement (a line whose
|
|
126
|
+
first non-whitespace token is `download`); prefer a static prefetch
|
|
127
|
+
W212 a <Relevance> is the literal `false` (case-insensitive) -- it never
|
|
128
|
+
applies to any endpoint; likewise a top-level `false AND ...`
|
|
129
|
+
W213 a <Relevance> has leading/trailing whitespace (fixable -> trimmed;
|
|
130
|
+
a CDATA-wrapped Relevance is left untouched, as with Title/W209)
|
|
131
|
+
W214 a <Title> contains a TODO or FIXME marker
|
|
132
|
+
W215 a Task/Fixlet <Description> is empty or missing (distinct from
|
|
133
|
+
E204's boilerplate-placeholder check)
|
|
134
|
+
W216 a non-empty <SourceSeverity> is not one of Low/Moderate/Important/
|
|
135
|
+
High/Critical (exact case) -- override with --severity-values.
|
|
136
|
+
`Unspecified` (any case, or the console's bracketed
|
|
137
|
+
`<Unspecified>`) carries no information (fixable -> emptied),
|
|
138
|
+
unless --severity-values lists it; a value that differs from an
|
|
139
|
+
allowed one only in case is fixable too (`HIGH` -> `High`)
|
|
140
|
+
W217 (only with --check-filename) a file's basename does not match its
|
|
141
|
+
first content object's <Title>, sanitized for filename-illegal
|
|
142
|
+
characters (/, backslash, :, *, ?, ", <, >, | -> _)
|
|
143
|
+
W218 a <PreLink>/<Link>/<PostLink> contains a run of 2+ spaces; the three
|
|
144
|
+
are assembled into one console sentence, and the gap is almost
|
|
145
|
+
always where a generator substituted an empty product name (e.g.
|
|
146
|
+
"<PostLink> to deploy v11.3.2.</PostLink>"). Not auto-fixed: the
|
|
147
|
+
missing word is what needs restoring, not the space
|
|
148
|
+
W219 a <SourceReleaseDate> later than the x-fixlet-modification-time:
|
|
149
|
+
content cannot be released after it was last modified, so one of
|
|
150
|
+
the two was likely missed in a hand edit or copied from another
|
|
151
|
+
file (compared with the later of the timestamp's local and UTC
|
|
152
|
+
dates, so a late-evening edit is not flagged)
|
|
153
|
+
W220 a <DownloadSize> that does not match the total of the prefetches'
|
|
154
|
+
sizes (the unzip.exe utility may be left out) -- a stale value,
|
|
155
|
+
often copied from another fixlet. Checked only when the total is
|
|
156
|
+
knowable: one action prefetches, none of its prefetches sits in an
|
|
157
|
+
`if`, every size is a literal, and there is no dynamic `download`
|
|
158
|
+
(fixable -> set to that total, the unzip utility left out)
|
|
159
|
+
W221 a Relevance that is true *until* a client setting has value V --
|
|
160
|
+
`not exists settings "N" whose ("V" = value of it) of client`, or
|
|
161
|
+
`value of setting "N" of client != "V"` -- while the action sets N
|
|
162
|
+
to something else, so the content stays relevant after it runs (a
|
|
163
|
+
typo, or a value edited in one place only). A positive test of the
|
|
164
|
+
current value, and a value set from a substitution, are not compared
|
|
165
|
+
|
|
166
|
+
Timestamp fields (E202, E216): x-fixlet-modification-time and
|
|
167
|
+
x-fixlet-first-propagation are both RFC 5322 date-times, e.g.
|
|
168
|
+
`Tue, 14 Jul 2026 18:32:35 +0000`:
|
|
169
|
+
- the leading day-of-week + comma is OPTIONAL (RFC 5322 itself makes it so),
|
|
170
|
+
but if present it must be the REAL day of week for the date given --
|
|
171
|
+
`Fri, 06 Aug 2026 12:43:34 +0000` is rejected because 6 Aug 2026 is a
|
|
172
|
+
Thursday, not a Friday
|
|
173
|
+
- the day of month is exactly 2 digits (zero-padded: `06`, not `6`)
|
|
174
|
+
- the month is exactly 3 letters with RFC 5322's exact capitalization
|
|
175
|
+
(`Aug`, not `aug` or `AUG`)
|
|
176
|
+
- the year is exactly 4 digits
|
|
177
|
+
- the time is `HH:MM:SS`, each exactly 2 digits
|
|
178
|
+
- a `+HHMM`/`-HHMM` UTC offset is required and must be a real one (a bare
|
|
179
|
+
zone name like `GMT`/`UTC`/`Z` is not accepted, and neither is an
|
|
180
|
+
out-of-range offset like `+9999` or one at/beyond 24 hours)
|
|
181
|
+
SourceReleaseDate (E201) is unrelated to the two above: it is a plain
|
|
182
|
+
YYYY-MM-DD date (e.g. `2026-07-14`), no time or timezone component at all.
|
|
183
|
+
|
|
184
|
+
The allowed <ActionScript> MIMETypes are:
|
|
185
|
+
application/x-Fixlet-Windows-Shell (the DEFAULT BigFix ActionScript type
|
|
186
|
+
for ALL platforms -- despite the name
|
|
187
|
+
it is not Windows-specific)
|
|
188
|
+
application/x-sh (shell, e.g. macOS / Linux)
|
|
189
|
+
application/x-AppleScript (macOS AppleScript)
|
|
190
|
+
application/x-Fixlet-Windows-PowerShell (Windows-specific PowerShell)
|
|
191
|
+
text/x-uri (open a URL)
|
|
192
|
+
|
|
193
|
+
E-codes are real issues and fail the hook. W-codes are advisory and do NOT fail
|
|
194
|
+
the hook unless --strict is given; wire the hook with `verbose: true` to surface
|
|
195
|
+
them. W200 is how the tool stays out of bes-schema-validate's lane: an
|
|
196
|
+
unparsable file is skipped, not failed, here.
|
|
197
|
+
|
|
198
|
+
--auto-fix rewrites the fixable conventions in place: an invalid/empty
|
|
199
|
+
DownloadSize -> 0 (E203), then a DownloadSize that disagrees with a knowable
|
|
200
|
+
prefetch total -> that total (W220); a missing ActionScript MIMEType -> the
|
|
201
|
+
default, or an allowed one's case (E200); an unambiguous year-first
|
|
202
|
+
SourceReleaseDate -> YYYY-MM-DD (E201); a loosely-formatted modification /
|
|
203
|
+
first-propagation timestamp -> RFC 5322 (E202/E216, a wrong day-of-week
|
|
204
|
+
dropped); a lowercase `cve-` prefix -> `CVE-` (E209); an unpadded evaluation
|
|
205
|
+
period -> HH:MM:SS (E219); a case-only SourceSeverity mismatch -> the allowed
|
|
206
|
+
spelling (W216); a missing SourceReleaseDate -> today (W202); a missing
|
|
207
|
+
x-fixlet-modification-time -> the moment the linter ran (W201); collapsed blank
|
|
208
|
+
lines before </ActionScript> (W205); a Title trimmed with tabs replaced by
|
|
209
|
+
spaces (W209); a Relevance trimmed of leading/trailing whitespace (W213);
|
|
210
|
+
trailing whitespace stripped from every line (W210); whitespace
|
|
211
|
+
around an ActionScript CDATA terminator stripped (E215); a missing or
|
|
212
|
+
non-UTF-8 XML declaration inserted / normalized (E214); a
|
|
213
|
+
Description/Relevance/ActionScript that entity-escapes < > & is unescaped and
|
|
214
|
+
CDATA-wrapped (E207); and a `// comment` in a Description <script> block
|
|
215
|
+
rewritten as `/* comment */` (E221, applied after E207 in the same pass so an
|
|
216
|
+
entity-escaped Description is unescaped first). One fix is gated behind
|
|
217
|
+
--strict: wrapping an otherwise-plain ActionScript body in <![CDATA[ ... ]]>
|
|
218
|
+
(W204). The file-level
|
|
219
|
+
fixers (W210 then E214) run after the per-block ones. Finally, CRLF
|
|
220
|
+
normalization runs LAST: whenever --auto-fix is on, the whole file is rewritten
|
|
221
|
+
with CRLF line endings (E208), so any fix -- and any file that was not already
|
|
222
|
+
all-CRLF -- ends up entirely CRLF rather than preserving a mix. --auto-fix
|
|
223
|
+
defaults to yes when files are given
|
|
224
|
+
explicitly, but to no when auto-discovering, so a bare run is read-only. An
|
|
225
|
+
auto-fixed file fails the hook so the change is reviewed and re-staged.
|
|
226
|
+
|
|
227
|
+
Usage:
|
|
228
|
+
bes_conventions_check.py [--strict] [--errors-only] [--auto-fix=yes|no]
|
|
229
|
+
[--disable E200,W201] [--check-filename]
|
|
230
|
+
[--severity-values Low,Moderate,Important,High,Critical]
|
|
231
|
+
[--prefetch-url-schemes http,https,RedHatProtocol,SWDProtocol]
|
|
232
|
+
[file.bes ...]
|
|
233
|
+
|
|
234
|
+
--check-filename is OFF by default: it enables W217 (a file's basename must
|
|
235
|
+
match its first content object's Title, sanitized for filename-illegal
|
|
236
|
+
characters). It is opt-in because many repos deliberately version or
|
|
237
|
+
otherwise diverge a Title from its filename.
|
|
238
|
+
|
|
239
|
+
--severity-values overrides the vocabulary W216 accepts for a non-empty
|
|
240
|
+
<SourceSeverity> (default: Low, Moderate, Important, High, Critical, matched
|
|
241
|
+
exact-case). Pass a comma-separated list of the values
|
|
242
|
+
this repo considers valid, e.g. --severity-values "low,medium,high,critical";
|
|
243
|
+
an empty value in the list is ignored, and an empty <SourceSeverity> is always
|
|
244
|
+
allowed regardless of this setting.
|
|
245
|
+
|
|
246
|
+
--prefetch-url-schemes overrides the URL schemes W206 accepts in a prefetch
|
|
247
|
+
statement or `add prefetch item` line (default: http, https, RedHatProtocol,
|
|
248
|
+
SWDProtocol, matched case-insensitively). It replaces the default rather than
|
|
249
|
+
extending it, like --severity-values.
|
|
250
|
+
|
|
251
|
+
With no file arguments, all *.bes files in the current folder and below are
|
|
252
|
+
checked. --disable takes a comma-separated list of check IDs to skip entirely.
|
|
253
|
+
--errors-only keeps every check (and every auto-fix) running but leaves W-codes
|
|
254
|
+
out of the report, so only E-codes and auto-fix lines are printed; unlike
|
|
255
|
+
--disable it does not turn off the W-code fixers.
|
|
256
|
+
|
|
257
|
+
A file can opt out of all checks with an XML comment anywhere in it:
|
|
258
|
+
<!-- pre-commit-skip: bes-conventions-check -->
|
|
259
|
+
or out of a single check family with the matching marker (also anywhere in the
|
|
260
|
+
file, e.g. in an XML comment):
|
|
261
|
+
mimetype-ok (E200)
|
|
262
|
+
source-release-date-ok (E201, W202, and W219)
|
|
263
|
+
modification-time-ok (E202 and W201)
|
|
264
|
+
first-propagation-ok (E216)
|
|
265
|
+
download-size-ok (E203, W203, and W220)
|
|
266
|
+
setting-value-ok (W221)
|
|
267
|
+
description-ok (E204)
|
|
268
|
+
cpe-ok (E205)
|
|
269
|
+
action-ui-metadata-ok (E206)
|
|
270
|
+
cdata-ok (W204 and E207)
|
|
271
|
+
cdata-close-ok (E215 -- CDATA terminator or bare ActionScript close)
|
|
272
|
+
action-blank-lines-ok (W205)
|
|
273
|
+
prefetch-ok (W206)
|
|
274
|
+
prefetch-https-ok (W207)
|
|
275
|
+
actionscript-empty-ok (W208)
|
|
276
|
+
title-ok (E211, W209, and W214)
|
|
277
|
+
relevance-ok (E212, E213, W212, and W213)
|
|
278
|
+
xml-declaration-ok (E214)
|
|
279
|
+
trailing-whitespace-ok (W210)
|
|
280
|
+
download-ok (W211)
|
|
281
|
+
cve-names-ok (E209)
|
|
282
|
+
mimefield-name-ok (E210)
|
|
283
|
+
analysis-property-ok (E220)
|
|
284
|
+
link-text-ok (W218)
|
|
285
|
+
success-criteria-ok (E217)
|
|
286
|
+
action-id-ok (E218)
|
|
287
|
+
evaluation-period-ok (E219)
|
|
288
|
+
severity-ok (W216)
|
|
289
|
+
filename-ok (W217, only relevant with --check-filename)
|
|
290
|
+
script-comment-ok (E221)
|
|
291
|
+
|
|
292
|
+
description-ok also covers W215 (an empty/missing Task/Fixlet Description) --
|
|
293
|
+
it is the same marker as E204 since both describe a Description problem.
|
|
294
|
+
|
|
295
|
+
Files that look like mustache templates (containing a `{{ placeholder }}`,
|
|
296
|
+
e.g. the `*.bes.mustache` sources that ContentFromTemplate renders) are skipped
|
|
297
|
+
silently: they are not valid BES XML until rendered, and their own output is
|
|
298
|
+
what should be linted. Only an identifier-like placeholder counts -- `{{` is
|
|
299
|
+
also the ActionScript escape for a literal `{`, so a heredoc payload (YARA,
|
|
300
|
+
JSON, C#) containing it is real content and is still checked.
|
|
301
|
+
|
|
302
|
+
Exit codes:
|
|
303
|
+
0 no E-code issues and nothing auto-fixed (and, without --strict, regardless
|
|
304
|
+
of warnings)
|
|
305
|
+
1 an E-code issue was found, a file was auto-fixed, or a warning was found
|
|
306
|
+
while --strict is set (--errors-only suppresses the warnings, so they
|
|
307
|
+
cannot fail the run even under --strict)
|
|
308
|
+
"""
|
|
309
|
+
|
|
310
|
+
import argparse
|
|
311
|
+
import functools
|
|
312
|
+
import json
|
|
313
|
+
import os
|
|
314
|
+
import re
|
|
315
|
+
import sys
|
|
316
|
+
from datetime import date, datetime, timezone
|
|
317
|
+
from xml.etree import ElementTree
|
|
318
|
+
|
|
319
|
+
if __package__ in (None, ""): # run directly as a script, not as a module
|
|
320
|
+
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
321
|
+
|
|
322
|
+
from pre_commit_bigfix import bes_common
|
|
323
|
+
from pre_commit_bigfix.bes_common import MUSTACHE_RE, read_source
|
|
324
|
+
|
|
325
|
+
# the fixers' name for the shared helper
|
|
326
|
+
_lineno = bes_common.lineno
|
|
327
|
+
|
|
328
|
+
SKIP_MARKER = "pre-commit-skip: bes-conventions-check"
|
|
329
|
+
|
|
330
|
+
# per-check opt-out markers (matched anywhere in the file text)
|
|
331
|
+
MIMETYPE_MARKER = "mimetype-ok" # E200
|
|
332
|
+
SOURCE_RELEASE_DATE_MARKER = "source-release-date-ok" # E201, W202, W219
|
|
333
|
+
MODIFICATION_TIME_MARKER = "modification-time-ok" # E202, W201
|
|
334
|
+
FIRST_PROPAGATION_MARKER = "first-propagation-ok" # E216
|
|
335
|
+
DOWNLOAD_SIZE_MARKER = "download-size-ok" # E203, W203, W220
|
|
336
|
+
SETTING_VALUE_MARKER = "setting-value-ok" # W221
|
|
337
|
+
DESCRIPTION_MARKER = "description-ok" # E204
|
|
338
|
+
CPE_MARKER = "cpe-ok" # E205
|
|
339
|
+
ACTION_UI_METADATA_MARKER = "action-ui-metadata-ok" # E206
|
|
340
|
+
CDATA_MARKER = "cdata-ok" # W204
|
|
341
|
+
CDATA_CLOSE_MARKER = "cdata-close-ok" # E215 (CDATA terminator or bare close)
|
|
342
|
+
ACTION_BLANK_LINES_MARKER = "action-blank-lines-ok" # W205
|
|
343
|
+
PREFETCH_MARKER = "prefetch-ok" # W206
|
|
344
|
+
PREFETCH_HTTPS_MARKER = "prefetch-https-ok" # W207
|
|
345
|
+
ACTIONSCRIPT_EMPTY_MARKER = "actionscript-empty-ok" # W208
|
|
346
|
+
TITLE_MARKER = "title-ok" # E211, W209
|
|
347
|
+
RELEVANCE_MARKER = "relevance-ok" # E212, E213
|
|
348
|
+
XML_DECL_MARKER = "xml-declaration-ok" # E214
|
|
349
|
+
TRAILING_WS_MARKER = "trailing-whitespace-ok" # W210
|
|
350
|
+
DOWNLOAD_MARKER = "download-ok" # W211
|
|
351
|
+
CVE_NAMES_MARKER = "cve-names-ok" # E209
|
|
352
|
+
MIMEFIELD_DUP_MARKER = "mimefield-name-ok" # E210
|
|
353
|
+
ANALYSIS_PROPERTY_MARKER = "analysis-property-ok" # E220
|
|
354
|
+
LINK_TEXT_MARKER = "link-text-ok" # W218
|
|
355
|
+
SUCCESS_CRITERIA_MARKER = "success-criteria-ok" # E217
|
|
356
|
+
ACTION_ID_MARKER = "action-id-ok" # E218
|
|
357
|
+
EVALUATION_PERIOD_MARKER = "evaluation-period-ok" # E219
|
|
358
|
+
SEVERITY_MARKER = "severity-ok" # W216
|
|
359
|
+
FILENAME_MARKER = "filename-ok" # W217 (only with --check-filename)
|
|
360
|
+
SCRIPT_COMMENT_MARKER = "script-comment-ok" # E221
|
|
361
|
+
|
|
362
|
+
BES_EXTENSIONS = (".bes",)
|
|
363
|
+
|
|
364
|
+
# the ActionScript MIMETypes BigFix supports (and this repo allows). The
|
|
365
|
+
# "Windows-Shell" one is the platform-agnostic default despite its name; only
|
|
366
|
+
# "Windows-PowerShell" is genuinely Windows-only.
|
|
367
|
+
ALLOWED_MIMETYPES = frozenset(
|
|
368
|
+
[
|
|
369
|
+
"application/x-Fixlet-Windows-Shell",
|
|
370
|
+
"application/x-sh",
|
|
371
|
+
"application/x-AppleScript",
|
|
372
|
+
"application/x-Fixlet-Windows-PowerShell",
|
|
373
|
+
"text/x-uri",
|
|
374
|
+
]
|
|
375
|
+
)
|
|
376
|
+
|
|
377
|
+
# content objects that live directly under <BES>
|
|
378
|
+
CONTENT_TAGS = frozenset(
|
|
379
|
+
["Task", "Fixlet", "Analysis", "ComputerGroup", "Baseline", "SingleAction"]
|
|
380
|
+
)
|
|
381
|
+
# only these are expected to carry a SourceReleaseDate / modification time and a
|
|
382
|
+
# download action; the description placeholder check (E204) applies to every
|
|
383
|
+
# content object, since Analysis/Baseline/... carry the same boilerplate
|
|
384
|
+
DATED_CONTENT_TAGS = frozenset(["Task", "Fixlet"])
|
|
385
|
+
|
|
386
|
+
MODIFICATION_TIME_NAME = "x-fixlet-modification-time"
|
|
387
|
+
FIRST_PROPAGATION_NAME = "x-fixlet-first-propagation"
|
|
388
|
+
DESCRIPTION_PLACEHOLDER = "enter a description of the"
|
|
389
|
+
|
|
390
|
+
# An RFC 5322 date-time, e.g. "Tue, 14 Jul 2026 18:32:35 +0000" -- shared by
|
|
391
|
+
# x-fixlet-modification-time (E202) and x-fixlet-first-propagation (E216). The
|
|
392
|
+
# day-of-week + comma is an optional group (RFC 5322 makes it so); `rest` is
|
|
393
|
+
# everything after it, captured separately so _valid_timestamp() can re-parse
|
|
394
|
+
# just that part with strptime without re-deriving the split index.
|
|
395
|
+
TIMESTAMP_RE = re.compile(
|
|
396
|
+
r"^(?:(?P<dow>[A-Za-z]{3}), )?"
|
|
397
|
+
r"(?P<rest>\d{2} (?P<mon>[A-Za-z]{3}) \d{4} \d{2}:\d{2}:\d{2} [+-]\d{4})$"
|
|
398
|
+
)
|
|
399
|
+
# index 0 = Monday, matching datetime.weekday() -- used to cross-check a
|
|
400
|
+
# SUPPLIED day-of-week against the one the date actually falls on, not just
|
|
401
|
+
# that it is spelled like a real weekday.
|
|
402
|
+
WEEKDAY_ABBREVS = ("Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun")
|
|
403
|
+
WEEKDAYS = frozenset(WEEKDAY_ABBREVS)
|
|
404
|
+
MONTHS = frozenset(
|
|
405
|
+
["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]
|
|
406
|
+
)
|
|
407
|
+
|
|
408
|
+
SOURCE_RELEASE_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
|
|
409
|
+
DOWNLOAD_SIZE_RE = re.compile(r"^\d+$")
|
|
410
|
+
|
|
411
|
+
# a CPE 2.3 formatted string: cpe:2.3: then 11 colon-separated components
|
|
412
|
+
# (part vendor product version update edition language sw_edition target_sw
|
|
413
|
+
# target_hw other); colons inside a component are backslash-escaped.
|
|
414
|
+
CPE23_RE = re.compile(r"^cpe:2\.3:[aho*\-](:([^:\\]|\\.)+){10}$", re.IGNORECASE)
|
|
415
|
+
|
|
416
|
+
# action-ui-metadata: the JSON object BigFixSetupTemplateDictionary emits. The
|
|
417
|
+
# value is real JSON, so whitespace and key order carry no meaning and it is
|
|
418
|
+
# parsed rather than pattern-matched; only the keys and their shapes are checked.
|
|
419
|
+
ACTION_UI_METADATA_KEYS = frozenset(["version", "size", "icon"])
|
|
420
|
+
ACTION_UI_METADATA_VERSION_RE = re.compile(r"^\d+(\.\d+)*$")
|
|
421
|
+
# an inline data URI, e.g. data:image/png;base64,iVBORw0KGgo... -- the payload
|
|
422
|
+
# (usually a base64-encoded icon) is not inspected beyond being non-empty.
|
|
423
|
+
DATA_URI_RE = re.compile(r"^data:[\w.+-]+/[\w.+-]+(?:;[\w.+-]+)*,.+$", re.DOTALL)
|
|
424
|
+
# how much of an offending value to quote in a message; an action-ui-metadata
|
|
425
|
+
# icon is a base64 blob that would otherwise bury the rest of the report.
|
|
426
|
+
VALUE_QUOTE_LIMIT = 120
|
|
427
|
+
|
|
428
|
+
# a prefetch statement or an "add prefetch item" line (user-supplied shape).
|
|
429
|
+
# A statement's sha1 is optional -- but only when it has a sha256, current
|
|
430
|
+
# BigFix clients accept a sha256-only statement (bes-actionscript-validate-
|
|
431
|
+
# prefetch's W405 is the same judgement); a block item's sha1 is unaffected
|
|
432
|
+
# here, that is W402's business in the sibling hook, not a shape issue.
|
|
433
|
+
# The URL schemes a well-formed prefetch may use (case-insensitive, like URL
|
|
434
|
+
# schemes themselves); override with --prefetch-url-schemes. SWDProtocol is
|
|
435
|
+
# the console's own upload scheme (`SWDProtocol://127.0.0.1:52311/Uploads/...`)
|
|
436
|
+
# and RedHatProtocol the RHEL patching plug-in's -- both common in real
|
|
437
|
+
# content. http stays allowed here; W207 separately nudges it toward https.
|
|
438
|
+
DEFAULT_PREFETCH_URL_SCHEMES = frozenset(
|
|
439
|
+
["http", "https", "SWDProtocol", "RedHatProtocol"]
|
|
440
|
+
)
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
# compiled W206 shapes, keyed by frozenset of schemes (a plain dict rather
|
|
444
|
+
# than functools.cache, which needs Python 3.9+)
|
|
445
|
+
_PREFETCH_OK_RES = {}
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
def _prefetch_ok_re(schemes):
|
|
449
|
+
"""Compile the W206 prefetch-line shape for a frozenset of URL schemes.
|
|
450
|
+
|
|
451
|
+
An empty set matches no URL at all (`(?!)`), rather than an empty
|
|
452
|
+
alternation that would accept a scheme-less `://host/x`.
|
|
453
|
+
"""
|
|
454
|
+
if schemes in _PREFETCH_OK_RES:
|
|
455
|
+
return _PREFETCH_OK_RES[schemes]
|
|
456
|
+
alternatives = "|".join(re.escape(item) for item in sorted(schemes, key=str.lower))
|
|
457
|
+
scheme = f"(?i:{alternatives})" if alternatives else "(?!)"
|
|
458
|
+
_PREFETCH_OK_RES[schemes] = re.compile(
|
|
459
|
+
rf"(^prefetch \S+ (sha1:\S{{40}} )?size:\d+ {scheme}:\/\/\S+ sha256:\S{{64}}$"
|
|
460
|
+
rf"|^\s+add prefetch item name=\S+ sha1=\S{{40}} size=\d+ url={scheme}:\/\/\S+"
|
|
461
|
+
r" sha256=\S{64}$)"
|
|
462
|
+
)
|
|
463
|
+
return _PREFETCH_OK_RES[schemes]
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
DOWNLOAD_KEYWORD_RE = re.compile(r"prefetch|download|add prefetch item", re.IGNORECASE)
|
|
467
|
+
|
|
468
|
+
# a line whose first non-whitespace token is the `download` action verb (a
|
|
469
|
+
# dynamic download); a `download` appearing later on a line, or in a comment,
|
|
470
|
+
# is not matched -- only a real statement is.
|
|
471
|
+
DOWNLOAD_STMT_RE = re.compile(r"^[ \t]*download\b", re.IGNORECASE)
|
|
472
|
+
# the URL inside a prefetch / add-prefetch-item line (used to check the scheme)
|
|
473
|
+
PREFETCH_URL_RE = re.compile(r"\b(?:url=|https?://)", re.IGNORECASE)
|
|
474
|
+
PREFETCH_URL_SCHEME_RE = re.compile(r"(?:url=)?(https?)://", re.IGNORECASE)
|
|
475
|
+
|
|
476
|
+
# <Title>, <Relevance>, and <CVENames> element bodies
|
|
477
|
+
# a run of 2+ spaces, the visible trace of a substitution that expanded to
|
|
478
|
+
# nothing (e.g. "<PostLink> to deploy v11.3.2.</PostLink>")
|
|
479
|
+
DOUBLE_SPACE_RE = re.compile(r" +")
|
|
480
|
+
# the three action link-text elements, checked together for that same trace
|
|
481
|
+
LINK_TEXT_RE = re.compile(
|
|
482
|
+
r"<(PreLink|Link|PostLink)>(.*?)</\1>", re.DOTALL | re.IGNORECASE
|
|
483
|
+
)
|
|
484
|
+
TITLE_TAG_RE = re.compile(r"<Title>(.*?)</Title>", re.DOTALL)
|
|
485
|
+
RELEVANCE_TAG_RE = re.compile(r"<Relevance>(.*?)</Relevance>", re.DOTALL)
|
|
486
|
+
CVENAMES_TAG_RE = re.compile(r"<CVENames>(.*?)</CVENames>", re.DOTALL)
|
|
487
|
+
|
|
488
|
+
# a single CVE identifier, e.g. CVE-2021-44228 (4-digit year, 4+ digit sequence)
|
|
489
|
+
CVE_RE = re.compile(r"^CVE-\d{4}-\d{4,}$")
|
|
490
|
+
# the sentinel BigFix content uses for "this object has no associated CVEs";
|
|
491
|
+
# the same word the console writes into SourceID/SourceSeverity/SANSID, so it
|
|
492
|
+
# is accepted as a whole CVENames value (exact case) rather than as a token --
|
|
493
|
+
# "CVE-2021-44228 Unspecified" is still a malformed list.
|
|
494
|
+
CVE_UNSPECIFIED = "Unspecified"
|
|
495
|
+
|
|
496
|
+
# the default placeholder titles the BigFix console emits for new content
|
|
497
|
+
DEFAULT_TITLES = frozenset(
|
|
498
|
+
["custom fixlet", "custom task", "custom baseline", "custom analysis"]
|
|
499
|
+
)
|
|
500
|
+
# a TODO/FIXME marker left in a Title, e.g. "... TODO:testing" or "... FIXME"
|
|
501
|
+
TODO_RE = re.compile(r"\bTODO\b|\bFIXME\b", re.IGNORECASE)
|
|
502
|
+
|
|
503
|
+
# a <SuccessCriteria Option="..."> ... </SuccessCriteria> action element; the
|
|
504
|
+
# body is a RelevanceString, meaningful only when Option="CustomRelevance"
|
|
505
|
+
SUCCESS_CRITERIA_RE = re.compile(
|
|
506
|
+
r"<SuccessCriteria\b([^>]*)>(.*?)</SuccessCriteria>", re.DOTALL
|
|
507
|
+
)
|
|
508
|
+
SUCCESS_CRITERIA_OPTION_RE = re.compile(r'Option\s*=\s*"([^"]*)"')
|
|
509
|
+
|
|
510
|
+
# an <Action ID="..."> or <DefaultAction ID="..."> open tag
|
|
511
|
+
ACTION_ID_RE = re.compile(r'<(?:Default)?Action\s+ID\s*=\s*"([^"]*)"')
|
|
512
|
+
|
|
513
|
+
# an x-relevance-evaluation-period value: an HH:MM:SS duration (the hour
|
|
514
|
+
# component is a plain duration, not a clock hour, so it is not capped at 23)
|
|
515
|
+
EVALUATION_PERIOD_RE = re.compile(r"^\d{2,}:[0-5]\d:[0-5]\d$")
|
|
516
|
+
|
|
517
|
+
# <SourceSeverity> body
|
|
518
|
+
SOURCE_SEVERITY_RE = re.compile(r"<SourceSeverity>(.*?)</SourceSeverity>", re.DOTALL)
|
|
519
|
+
# the canonical, exact-case severity vocabulary; empty is always allowed
|
|
520
|
+
# "High" is here for third-party application content (Mozilla, Adobe, Zoom,
|
|
521
|
+
# ...), which rates that way rather than on the Microsoft patch scale.
|
|
522
|
+
CANONICAL_SEVERITIES = frozenset(["Low", "Moderate", "Important", "High", "Critical"])
|
|
523
|
+
|
|
524
|
+
# characters not allowed in a filename on common filesystems -- used to derive
|
|
525
|
+
# the expected filename stem from a content object's Title (W217)
|
|
526
|
+
FILENAME_ILLEGAL_RE = re.compile(r'[\\/:*?"<>|]')
|
|
527
|
+
|
|
528
|
+
# an XML declaration at the very start of the document, and its encoding value
|
|
529
|
+
XML_DECL_RE = re.compile("^\\ufeff?\\s*<\\?xml\\b([^>]*)\\?>")
|
|
530
|
+
XML_DECL_ENCODING_RE = re.compile(r'encoding\s*=\s*"([^"]*)"', re.IGNORECASE)
|
|
531
|
+
XML_DECL_VERSION_RE = re.compile(r'version\s*=\s*"([^"]*)"', re.IGNORECASE)
|
|
532
|
+
# trailing spaces/tabs at the end of any line (LF-normalized text)
|
|
533
|
+
TRAILING_WS_RE = re.compile(r"[ \t]+$", re.MULTILINE)
|
|
534
|
+
|
|
535
|
+
# an entity reference for a character that requires CDATA (or escaping): < > &
|
|
536
|
+
# -- either the named entity or a decimal/hex numeric reference to it. A literal
|
|
537
|
+
# `>` is valid XML text and does NOT require CDATA, so it is deliberately absent.
|
|
538
|
+
SPECIAL_ENTITY_RE = re.compile(
|
|
539
|
+
r"&(?:lt|gt|amp|#0*(?:60|62|38)|#x0*(?:3c|3e|26));", re.IGNORECASE
|
|
540
|
+
)
|
|
541
|
+
# text elements whose content should be CDATA-wrapped when it has special chars
|
|
542
|
+
CDATA_ELEMENT_RE = re.compile(
|
|
543
|
+
r"<(Description|Relevance|ActionScript)\b[^>]*>(.*?)</\1>", re.DOTALL
|
|
544
|
+
)
|
|
545
|
+
# a real child element open tag (distinguishes an action <Description>, whose
|
|
546
|
+
# body is <PreLink>/<Link>/<PostLink> markup, from an entity-escaped text body)
|
|
547
|
+
CHILD_ELEMENT_RE = re.compile(r"<[A-Za-z]")
|
|
548
|
+
|
|
549
|
+
# a <Description> element (content-object or action), used to scope E221 to
|
|
550
|
+
# Description bodies only -- Relevance/ActionScript are not scanned for it
|
|
551
|
+
DESCRIPTION_ELEMENT_RE = re.compile(
|
|
552
|
+
r"<Description\b[^>]*>(.*?)</Description>", re.DOTALL
|
|
553
|
+
)
|
|
554
|
+
# a <script> block within a (possibly CDATA-wrapped, possibly entity-escaped)
|
|
555
|
+
# Description body
|
|
556
|
+
SCRIPT_BLOCK_RE = re.compile(
|
|
557
|
+
r"<script\b[^>]*>(.*?)</script>", re.IGNORECASE | re.DOTALL
|
|
558
|
+
)
|
|
559
|
+
SCRIPT_OPEN_RE = re.compile(r"<script\b", re.IGNORECASE)
|
|
560
|
+
|
|
561
|
+
ACTIONSCRIPT_OPEN_RE = re.compile(r"<ActionScript\b([^>]*)>")
|
|
562
|
+
ACTIONSCRIPT_FULL_RE = re.compile(
|
|
563
|
+
r"<ActionScript\b([^>]*)>(.*?)</ActionScript>", re.DOTALL
|
|
564
|
+
)
|
|
565
|
+
MIMETYPE_ATTR_RE = re.compile(r'MIMEType\s*=\s*"([^"]*)"')
|
|
566
|
+
SRD_RE = re.compile(r"<SourceReleaseDate>(.*?)</SourceReleaseDate>", re.DOTALL)
|
|
567
|
+
DOWNLOAD_SIZE_TAG_RE = re.compile(r"<DownloadSize>(.*?)</DownloadSize>", re.DOTALL)
|
|
568
|
+
MODTIME_VALUE_RE = re.compile(
|
|
569
|
+
r"<Name>\s*"
|
|
570
|
+
+ re.escape(MODIFICATION_TIME_NAME)
|
|
571
|
+
+ r"\s*</Name>\s*<Value>(.*?)</Value>",
|
|
572
|
+
re.DOTALL,
|
|
573
|
+
)
|
|
574
|
+
FIRST_PROP_VALUE_RE = re.compile(
|
|
575
|
+
r"<Name>\s*"
|
|
576
|
+
+ re.escape(FIRST_PROPAGATION_NAME)
|
|
577
|
+
+ r"\s*</Name>\s*<Value>(.*?)</Value>",
|
|
578
|
+
re.DOTALL,
|
|
579
|
+
)
|
|
580
|
+
# an Analysis <Property>, with its Name and (optional) ID attributes; the two
|
|
581
|
+
# may appear in either order, so ID is looked for across the whole tag
|
|
582
|
+
ANALYSIS_PROPERTY_RE = re.compile(
|
|
583
|
+
r'<Property\b[^>]*?\bName="([^"]*)"(?:[^>]*?\bID="([^"]*)")?[^>]*>',
|
|
584
|
+
re.IGNORECASE,
|
|
585
|
+
)
|
|
586
|
+
NAMED_MIMEFIELD_RE = re.compile(
|
|
587
|
+
r"<Name>\s*([^<]*?)\s*</Name>\s*<Value>(.*?)</Value>", re.DOTALL
|
|
588
|
+
)
|
|
589
|
+
CONTENT_OPEN_RE = re.compile(r"<(" + "|".join(sorted(CONTENT_TAGS)) + r")\b")
|
|
590
|
+
CONTENT_BLOCK_RE = re.compile(r"(<(Task|Fixlet)\b[^>]*>)(.*?)(</\2>)", re.DOTALL)
|
|
591
|
+
# a whole content object (open tag through matching close), used to split a <BES>
|
|
592
|
+
# into the independent entities it contains so each is checked/fixed on its own.
|
|
593
|
+
CONTENT_OBJECT_SPAN_RE = re.compile(
|
|
594
|
+
r"<(" + "|".join(sorted(CONTENT_TAGS)) + r")\b[^>]*>.*?</\1>", re.DOTALL
|
|
595
|
+
)
|
|
596
|
+
CDATA_RE = re.compile(r"^<!\[CDATA\[(.*)\]\]>$", re.DOTALL)
|
|
597
|
+
# 2+ blank lines immediately before a </ActionScript> close (an optional CDATA
|
|
598
|
+
# terminator may sit between the blank lines and the close tag)
|
|
599
|
+
BLANK_BEFORE_CLOSE_RE = re.compile(
|
|
600
|
+
r"(\n)(?:[ \t]*\n){2,}([ \t]*(?:\]\]>)?[ \t]*</ActionScript>)"
|
|
601
|
+
)
|
|
602
|
+
# an ActionScript close tag -- either the CDATA terminator `]]>` or, for a
|
|
603
|
+
# non-CDATA body, the bare close itself -- with the whitespace that may
|
|
604
|
+
# surround it. Whitespace before `]]>` (or before a bare close) is INSIDE the
|
|
605
|
+
# ActionScript (so it is action content), and whitespace after `]]>` is
|
|
606
|
+
# element text that BigFix appends to the same body -- either way it is a
|
|
607
|
+
# spurious whitespace-only last line of the action (E215).
|
|
608
|
+
CDATA_CLOSE_RE = re.compile(r"[ \t]*(?:\]\]>)?[ \t]*</ActionScript>")
|
|
609
|
+
CDATA_CLOSE_CANONICAL = "]]></ActionScript>"
|
|
610
|
+
CDATA_CLOSE_CANONICAL_PLAIN = "</ActionScript>"
|
|
611
|
+
|
|
612
|
+
# where a new SourceReleaseDate / modification-time MIMEField may be inserted so
|
|
613
|
+
# the result still satisfies the BES.xsd element ordering
|
|
614
|
+
SRD_ANCHORS = (
|
|
615
|
+
"<SourceSeverity",
|
|
616
|
+
"<CVENames",
|
|
617
|
+
"<SANSID",
|
|
618
|
+
"<MIMEField",
|
|
619
|
+
"<Domain",
|
|
620
|
+
"<DefaultAction",
|
|
621
|
+
"<Action",
|
|
622
|
+
)
|
|
623
|
+
MODTIME_ANCHORS = ("<Domain", "<DefaultAction", "<Action", "<SingleAction")
|
|
624
|
+
|
|
625
|
+
KNOWN_CODES = frozenset(
|
|
626
|
+
[
|
|
627
|
+
"E200", # ActionScript MIMEType missing / not allowed
|
|
628
|
+
"E201", # SourceReleaseDate not YYYY-MM-DD
|
|
629
|
+
"E202", # modification-time value not in expected format
|
|
630
|
+
"E203", # DownloadSize not 0-or-positive-integer
|
|
631
|
+
"E204", # Description contains the boilerplate placeholder
|
|
632
|
+
"E205", # x-fixlet-cpe23-item-name not a valid CPE 2.3 string
|
|
633
|
+
"E206", # action-ui-metadata not well-formed
|
|
634
|
+
"E207", # entity-escaped special chars where CDATA is required
|
|
635
|
+
"E208", # file is not entirely CRLF-terminated
|
|
636
|
+
"E209", # CVENames value invalid or multiple CVENames elements
|
|
637
|
+
"E210", # duplicate MIMEField Name within one content object
|
|
638
|
+
"E220", # duplicate Analysis <Property> Name or ID
|
|
639
|
+
"W218", # action link text has a run of 2+ spaces
|
|
640
|
+
"W219", # SourceReleaseDate later than x-fixlet-modification-time
|
|
641
|
+
"W220", # DownloadSize does not match the prefetches' total
|
|
642
|
+
"W221", # relevance's target setting value differs from what is set
|
|
643
|
+
"E211", # Title is a default placeholder value
|
|
644
|
+
"E212", # Relevance is the literal `true`
|
|
645
|
+
"E213", # Relevance is empty / whitespace only
|
|
646
|
+
"E214", # XML declaration missing or not encoding="UTF-8"
|
|
647
|
+
"E215", # whitespace around an ActionScript CDATA terminator
|
|
648
|
+
"E216", # x-fixlet-first-propagation value not a valid RFC 5322 date-time
|
|
649
|
+
"E217", # SuccessCriteria body/Option consistency
|
|
650
|
+
"E218", # duplicate Action ID within one content object
|
|
651
|
+
"E219", # x-relevance-evaluation-period value not a valid HH:MM:SS duration
|
|
652
|
+
"E221", # Description <script> block uses a `//` line comment
|
|
653
|
+
"E222", # auto-fix not written: it would break BES.xsd validity
|
|
654
|
+
"W200", # not parseable BES XML; skipped
|
|
655
|
+
"W201", # Task/Fixlet missing x-fixlet-modification-time
|
|
656
|
+
"W202", # Task/Fixlet missing SourceReleaseDate
|
|
657
|
+
"W203", # DownloadSize > 0 but no download keyword in an ActionScript
|
|
658
|
+
"W204", # ActionScript body not wrapped in CDATA
|
|
659
|
+
"W205", # more than one blank line before </ActionScript>
|
|
660
|
+
"W206", # prefetch / add-prefetch-item line malformed
|
|
661
|
+
"W207", # prefetch / add-prefetch-item URL is not https
|
|
662
|
+
"W208", # ActionScript body empty (only blank lines and //-comments)
|
|
663
|
+
"W209", # Title has leading/trailing whitespace or embedded tabs
|
|
664
|
+
"W210", # a line has trailing whitespace
|
|
665
|
+
"W211", # ActionScript uses a dynamic `download` statement
|
|
666
|
+
"W212", # Relevance is the literal `false`
|
|
667
|
+
"W213", # Relevance has leading/trailing whitespace
|
|
668
|
+
"W214", # Title contains a TODO/FIXME marker
|
|
669
|
+
"W215", # Task/Fixlet Description is empty or missing
|
|
670
|
+
"W216", # SourceSeverity not in the canonical vocabulary
|
|
671
|
+
"W217", # (--check-filename only) filename does not match Title
|
|
672
|
+
]
|
|
673
|
+
)
|
|
674
|
+
|
|
675
|
+
|
|
676
|
+
def _now():
|
|
677
|
+
"""Return the current UTC time (isolated so tests can monkeypatch it)."""
|
|
678
|
+
return datetime.now(timezone.utc)
|
|
679
|
+
|
|
680
|
+
|
|
681
|
+
def _today_str(now=None):
|
|
682
|
+
"""Return today's date as YYYY-MM-DD."""
|
|
683
|
+
return (now or _now()).strftime("%Y-%m-%d")
|
|
684
|
+
|
|
685
|
+
|
|
686
|
+
def _modtime_str(now=None):
|
|
687
|
+
"""Return the current time as e.g. `Tue, 14 Jul 2026 18:32:35 +0000`."""
|
|
688
|
+
return (now or _now()).strftime("%a, %d %b %Y %H:%M:%S %z")
|
|
689
|
+
|
|
690
|
+
|
|
691
|
+
def _is_all_crlf(raw):
|
|
692
|
+
"""True if the raw bytes use CRLF throughout (no lone LF and no lone CR).
|
|
693
|
+
|
|
694
|
+
A file with no line breaks at all is trivially CRLF-consistent.
|
|
695
|
+
"""
|
|
696
|
+
crlf = raw.count(b"\r\n")
|
|
697
|
+
return raw.count(b"\n") == crlf and raw.count(b"\r") == crlf
|
|
698
|
+
|
|
699
|
+
|
|
700
|
+
def _to_crlf(text):
|
|
701
|
+
"""Return `text` with every line ending normalized to CRLF."""
|
|
702
|
+
return text.replace("\r\n", "\n").replace("\r", "\n").replace("\n", "\r\n")
|
|
703
|
+
|
|
704
|
+
|
|
705
|
+
def _strip_cdata(text):
|
|
706
|
+
"""Return `text` trimmed, with a wrapping <![CDATA[ ... ]]> removed if present."""
|
|
707
|
+
text = text.strip()
|
|
708
|
+
match = CDATA_RE.match(text)
|
|
709
|
+
return match.group(1).strip() if match else text
|
|
710
|
+
|
|
711
|
+
|
|
712
|
+
def _xml_unescape(text):
|
|
713
|
+
"""Decode XML entity references (named and numeric) to their characters.
|
|
714
|
+
|
|
715
|
+
`&` is decoded last so `&lt;` becomes the literal text `<`, not `<`.
|
|
716
|
+
"""
|
|
717
|
+
text = (
|
|
718
|
+
text.replace("<", "<")
|
|
719
|
+
.replace(">", ">")
|
|
720
|
+
.replace(""", '"')
|
|
721
|
+
.replace("'", "'")
|
|
722
|
+
)
|
|
723
|
+
text = re.sub(r"&#x([0-9a-fA-F]+);", lambda m: chr(int(m.group(1), 16)), text)
|
|
724
|
+
text = re.sub(r"&#(\d+);", lambda m: chr(int(m.group(1))), text)
|
|
725
|
+
return text.replace("&", "&")
|
|
726
|
+
|
|
727
|
+
|
|
728
|
+
def _valid_source_release_date(value):
|
|
729
|
+
"""True if `value` is a real YYYY-MM-DD date."""
|
|
730
|
+
if not SOURCE_RELEASE_DATE_RE.match(value):
|
|
731
|
+
return False
|
|
732
|
+
try:
|
|
733
|
+
date.fromisoformat(value) # parsed only to reject impossible dates
|
|
734
|
+
except ValueError:
|
|
735
|
+
return False
|
|
736
|
+
return True
|
|
737
|
+
|
|
738
|
+
|
|
739
|
+
def _valid_timestamp(value):
|
|
740
|
+
"""True if `value` is a valid RFC 5322 date-time (E202 / E216)."""
|
|
741
|
+
return _parse_timestamp(value) is not None
|
|
742
|
+
|
|
743
|
+
|
|
744
|
+
def _parse_timestamp(value):
|
|
745
|
+
"""Return `value` as an aware datetime if it is a valid RFC 5322 date-time.
|
|
746
|
+
|
|
747
|
+
`Tue, 14 Jul 2026 18:32:35 +0000` and `14 Jul 2026 18:32:35 +0000` (no
|
|
748
|
+
day-of-week) are both valid; `Fri, 14 Jul 2026 18:32:35 +0000` is not,
|
|
749
|
+
because 14 Jul 2026 is actually a Tuesday. See the module docstring's
|
|
750
|
+
"Timestamp fields" section for the full rule.
|
|
751
|
+
"""
|
|
752
|
+
match = TIMESTAMP_RE.match(value)
|
|
753
|
+
if not match:
|
|
754
|
+
return None
|
|
755
|
+
if match.group("mon") not in MONTHS:
|
|
756
|
+
return None
|
|
757
|
+
dow = match.group("dow")
|
|
758
|
+
if dow is not None and dow not in WEEKDAYS:
|
|
759
|
+
return None
|
|
760
|
+
try:
|
|
761
|
+
# Parsed from `rest` (the day-of-week, if any, already matched and
|
|
762
|
+
# verified above) so a correctly-spelled but wrong-for-the-date
|
|
763
|
+
# weekday can't slip past strptime, which does not check that.
|
|
764
|
+
# %z also rejects a bare zone name (no "GMT"/"UTC"/"Z") and an
|
|
765
|
+
# out-of-range offset (magnitude >= 24 hours), which is exactly the
|
|
766
|
+
# "must represent a valid offset" requirement.
|
|
767
|
+
parsed = datetime.strptime(match.group("rest"), "%d %b %Y %H:%M:%S %z")
|
|
768
|
+
except ValueError:
|
|
769
|
+
return None
|
|
770
|
+
if dow is not None and WEEKDAY_ABBREVS[parsed.weekday()] != dow:
|
|
771
|
+
return None
|
|
772
|
+
return parsed
|
|
773
|
+
|
|
774
|
+
|
|
775
|
+
def _valid_cpe23(value):
|
|
776
|
+
"""True if `value` is a CPE 2.3 formatted string."""
|
|
777
|
+
return bool(CPE23_RE.match(value))
|
|
778
|
+
|
|
779
|
+
|
|
780
|
+
def _valid_action_ui_metadata(value):
|
|
781
|
+
"""True if `value` is a well-formed action-ui-metadata object.
|
|
782
|
+
|
|
783
|
+
It must be a JSON object with a dotted-numeric `version` string and a
|
|
784
|
+
non-negative integer `size` (quoted or bare -- the console emits both),
|
|
785
|
+
plus an optional `icon` data URI. Whitespace and key order are
|
|
786
|
+
insignificant; keys outside that set are not recognised.
|
|
787
|
+
"""
|
|
788
|
+
try:
|
|
789
|
+
data = json.loads(value)
|
|
790
|
+
except ValueError:
|
|
791
|
+
return False
|
|
792
|
+
if not isinstance(data, dict) or not ACTION_UI_METADATA_KEYS.issuperset(data):
|
|
793
|
+
return False
|
|
794
|
+
|
|
795
|
+
version = data.get("version")
|
|
796
|
+
if not isinstance(version, str) or not ACTION_UI_METADATA_VERSION_RE.match(version):
|
|
797
|
+
return False
|
|
798
|
+
|
|
799
|
+
size = data.get("size")
|
|
800
|
+
if isinstance(size, str):
|
|
801
|
+
if not DOWNLOAD_SIZE_RE.match(size):
|
|
802
|
+
return False
|
|
803
|
+
elif not isinstance(size, int) or isinstance(size, bool) or size < 0:
|
|
804
|
+
return False
|
|
805
|
+
|
|
806
|
+
icon = data.get("icon")
|
|
807
|
+
return icon is None or (isinstance(icon, str) and bool(DATA_URI_RE.match(icon)))
|
|
808
|
+
|
|
809
|
+
|
|
810
|
+
def _quote(value, limit=VALUE_QUOTE_LIMIT):
|
|
811
|
+
"""`value` shortened for use in a message, with an elision marker if cut."""
|
|
812
|
+
value = value.strip()
|
|
813
|
+
if len(value) <= limit:
|
|
814
|
+
return value
|
|
815
|
+
return value[:limit] + "..."
|
|
816
|
+
|
|
817
|
+
|
|
818
|
+
# --------------------------------------------------------------------------
|
|
819
|
+
# checks (each returns a list of (lineno, code, message))
|
|
820
|
+
# --------------------------------------------------------------------------
|
|
821
|
+
|
|
822
|
+
|
|
823
|
+
def check_action_mimetypes(src):
|
|
824
|
+
"""E200: every <ActionScript> must carry a MIMEType from the allowed set."""
|
|
825
|
+
issues = []
|
|
826
|
+
for match in ACTIONSCRIPT_OPEN_RE.finditer(src):
|
|
827
|
+
attrs = match.group(1)
|
|
828
|
+
lineno = _lineno(src, match.start())
|
|
829
|
+
mime_match = MIMETYPE_ATTR_RE.search(attrs)
|
|
830
|
+
if mime_match is None:
|
|
831
|
+
issues.append(
|
|
832
|
+
(
|
|
833
|
+
lineno,
|
|
834
|
+
"E200",
|
|
835
|
+
(
|
|
836
|
+
"ActionScript has no MIMEType; add one of "
|
|
837
|
+
f"{sorted(ALLOWED_MIMETYPES)}; add `{MIMETYPE_MARKER}` if intentional"
|
|
838
|
+
),
|
|
839
|
+
)
|
|
840
|
+
)
|
|
841
|
+
elif mime_match.group(1) not in ALLOWED_MIMETYPES:
|
|
842
|
+
issues.append(
|
|
843
|
+
(
|
|
844
|
+
lineno,
|
|
845
|
+
"E200",
|
|
846
|
+
(
|
|
847
|
+
f'ActionScript MIMEType "{mime_match.group(1)}" is not one of '
|
|
848
|
+
f"{sorted(ALLOWED_MIMETYPES)}; add `{MIMETYPE_MARKER}` if intentional"
|
|
849
|
+
),
|
|
850
|
+
)
|
|
851
|
+
)
|
|
852
|
+
return issues
|
|
853
|
+
|
|
854
|
+
|
|
855
|
+
def check_source_release_date_format(src):
|
|
856
|
+
"""E201: every <SourceReleaseDate> present must be YYYY-MM-DD (empty allowed)."""
|
|
857
|
+
issues = []
|
|
858
|
+
for match in SRD_RE.finditer(src):
|
|
859
|
+
value = _strip_cdata(match.group(1))
|
|
860
|
+
if value == "":
|
|
861
|
+
continue
|
|
862
|
+
if not _valid_source_release_date(value):
|
|
863
|
+
issues.append(
|
|
864
|
+
(
|
|
865
|
+
_lineno(src, match.start()),
|
|
866
|
+
"E201",
|
|
867
|
+
(
|
|
868
|
+
f'SourceReleaseDate "{value}" is not in YYYY-MM-DD format; '
|
|
869
|
+
f"add `{SOURCE_RELEASE_DATE_MARKER}` if intentional"
|
|
870
|
+
),
|
|
871
|
+
)
|
|
872
|
+
)
|
|
873
|
+
return issues
|
|
874
|
+
|
|
875
|
+
|
|
876
|
+
def check_modification_time_format(src):
|
|
877
|
+
"""E202: every x-fixlet-modification-time value must be a valid RFC 5322 date-
|
|
878
|
+
time.
|
|
879
|
+
"""
|
|
880
|
+
issues = []
|
|
881
|
+
for match in MODTIME_VALUE_RE.finditer(src):
|
|
882
|
+
value = _strip_cdata(match.group(1))
|
|
883
|
+
if not _valid_timestamp(value):
|
|
884
|
+
issues.append(
|
|
885
|
+
(
|
|
886
|
+
_lineno(src, match.start()),
|
|
887
|
+
"E202",
|
|
888
|
+
(
|
|
889
|
+
f'x-fixlet-modification-time "{value}" is not a valid RFC 5322 '
|
|
890
|
+
"date-time (e.g. `Tue, 14 Jul 2026 18:32:35 +0000`); add "
|
|
891
|
+
f"`{MODIFICATION_TIME_MARKER}` if intentional"
|
|
892
|
+
),
|
|
893
|
+
)
|
|
894
|
+
)
|
|
895
|
+
return issues
|
|
896
|
+
|
|
897
|
+
|
|
898
|
+
def check_release_after_modification(src):
|
|
899
|
+
"""W219: <SourceReleaseDate> is later than x-fixlet-modification-time.
|
|
900
|
+
|
|
901
|
+
Content cannot be released after it was last modified, so the two
|
|
902
|
+
disagreeing is a sign the file was hand-edited or its metadata copied
|
|
903
|
+
from another file with a field missed. Only two valid values are compared
|
|
904
|
+
(malformed ones are E201/E202's). SourceReleaseDate has no time zone, so
|
|
905
|
+
it is compared with the later of the modification time's local and UTC
|
|
906
|
+
dates -- a late-evening timestamp is never flagged for a next-day date.
|
|
907
|
+
"""
|
|
908
|
+
srd_match = SRD_RE.search(src)
|
|
909
|
+
mod_match = MODTIME_VALUE_RE.search(src)
|
|
910
|
+
if not srd_match or not mod_match:
|
|
911
|
+
return []
|
|
912
|
+
srd = _strip_cdata(srd_match.group(1))
|
|
913
|
+
modified = _parse_timestamp(_strip_cdata(mod_match.group(1)))
|
|
914
|
+
if not srd or not _valid_source_release_date(srd) or modified is None:
|
|
915
|
+
return []
|
|
916
|
+
latest = max(modified.date(), modified.astimezone(timezone.utc).date())
|
|
917
|
+
if date.fromisoformat(srd) <= latest:
|
|
918
|
+
return []
|
|
919
|
+
return [
|
|
920
|
+
(
|
|
921
|
+
_lineno(src, srd_match.start()),
|
|
922
|
+
"W219",
|
|
923
|
+
(
|
|
924
|
+
f"SourceReleaseDate {srd} is later than x-fixlet-modification-time "
|
|
925
|
+
f"{_strip_cdata(mod_match.group(1))}; one of them was likely "
|
|
926
|
+
"missed in an edit; add "
|
|
927
|
+
f"`{SOURCE_RELEASE_DATE_MARKER}` if intentional"
|
|
928
|
+
),
|
|
929
|
+
)
|
|
930
|
+
]
|
|
931
|
+
|
|
932
|
+
|
|
933
|
+
def check_first_propagation_format(src):
|
|
934
|
+
"""E216: every x-fixlet-first-propagation value must be a valid RFC 5322 date-
|
|
935
|
+
time.
|
|
936
|
+
"""
|
|
937
|
+
issues = []
|
|
938
|
+
for match in FIRST_PROP_VALUE_RE.finditer(src):
|
|
939
|
+
value = _strip_cdata(match.group(1))
|
|
940
|
+
if not _valid_timestamp(value):
|
|
941
|
+
issues.append(
|
|
942
|
+
(
|
|
943
|
+
_lineno(src, match.start()),
|
|
944
|
+
"E216",
|
|
945
|
+
(
|
|
946
|
+
f'x-fixlet-first-propagation "{value}" is not a valid RFC 5322 '
|
|
947
|
+
"date-time (e.g. `Tue, 14 Jul 2026 18:32:35 +0000`); add "
|
|
948
|
+
f"`{FIRST_PROPAGATION_MARKER}` if intentional"
|
|
949
|
+
),
|
|
950
|
+
)
|
|
951
|
+
)
|
|
952
|
+
return issues
|
|
953
|
+
|
|
954
|
+
|
|
955
|
+
# W220: a prefetch statement / block item, its size, and if-structure lines
|
|
956
|
+
PREFETCH_LINE_RE = re.compile(
|
|
957
|
+
r"^[ \t]*(?:prefetch\s|add\s+(?:nohash\s+)?prefetch\s+item\b)", re.IGNORECASE
|
|
958
|
+
)
|
|
959
|
+
PREFETCH_SIZE_RE = re.compile(r"\bsize[:=](\S+)", re.IGNORECASE)
|
|
960
|
+
PREFETCH_HASH_RE = re.compile(r"\bsha(?:1|256)[:=]([0-9A-Fa-f]+)", re.IGNORECASE)
|
|
961
|
+
IF_OPEN_RE = re.compile(r"^[ \t]*if\b", re.IGNORECASE)
|
|
962
|
+
IF_CLOSE_RE = re.compile(r"^[ \t]*endif\b", re.IGNORECASE)
|
|
963
|
+
# the BigFix unzip utility, which the console leaves out of DownloadSize
|
|
964
|
+
UNZIP_PREFETCH_RE = re.compile(r"\bunzip[\w.-]*\.exe\b", re.IGNORECASE)
|
|
965
|
+
|
|
966
|
+
|
|
967
|
+
def _prefetch_total(body):
|
|
968
|
+
"""Return (total, unzip_total) of a body's prefetch sizes, or None.
|
|
969
|
+
|
|
970
|
+
None when the total is unknowable: no prefetch at all, a prefetch inside
|
|
971
|
+
an `if` (which one runs depends on the endpoint), a size that is not a
|
|
972
|
+
literal integer, or a dynamic `download` statement. A prefetch with the
|
|
973
|
+
same sha1/sha256 and size as one already counted is a mirror URL for the same file and
|
|
974
|
+
is not counted again.
|
|
975
|
+
"""
|
|
976
|
+
depth, total, unzip, seen = 0, 0, 0, False
|
|
977
|
+
counted = set() # (hashes, size) already summed: a mirror URL is the same file
|
|
978
|
+
for line in body.split("\n"):
|
|
979
|
+
if IF_OPEN_RE.match(line):
|
|
980
|
+
depth += 1
|
|
981
|
+
elif IF_CLOSE_RE.match(line):
|
|
982
|
+
depth = max(depth - 1, 0)
|
|
983
|
+
elif DOWNLOAD_STMT_RE.match(line):
|
|
984
|
+
return None
|
|
985
|
+
elif PREFETCH_LINE_RE.match(line):
|
|
986
|
+
size = PREFETCH_SIZE_RE.search(line)
|
|
987
|
+
if depth or not size or not size.group(1).isdigit():
|
|
988
|
+
return None
|
|
989
|
+
seen = True
|
|
990
|
+
hashes = frozenset(h.lower() for h in PREFETCH_HASH_RE.findall(line))
|
|
991
|
+
key = (hashes, size.group(1))
|
|
992
|
+
if hashes and key in counted:
|
|
993
|
+
continue # the same file (hashes and size) from another URL
|
|
994
|
+
counted.add(key)
|
|
995
|
+
total += int(size.group(1))
|
|
996
|
+
if UNZIP_PREFETCH_RE.search(line):
|
|
997
|
+
unzip += int(size.group(1))
|
|
998
|
+
return (total, unzip) if seen else None
|
|
999
|
+
|
|
1000
|
+
|
|
1001
|
+
def check_download_size_total(src):
|
|
1002
|
+
"""W220: <DownloadSize> does not match the prefetches' total size.
|
|
1003
|
+
|
|
1004
|
+
Only when the total is knowable: exactly one action prefetches, none of
|
|
1005
|
+
its prefetches is conditional, and every size is a literal integer. The
|
|
1006
|
+
unzip.exe utility may be counted or left out (the console leaves it
|
|
1007
|
+
out). A stale value -- often copied from another fixlet -- shows the
|
|
1008
|
+
wrong download size in the console.
|
|
1009
|
+
"""
|
|
1010
|
+
mismatch = _download_size_mismatch(src)
|
|
1011
|
+
if mismatch is None:
|
|
1012
|
+
return []
|
|
1013
|
+
size_match, declared, total, _unzip = mismatch
|
|
1014
|
+
return [
|
|
1015
|
+
(
|
|
1016
|
+
_lineno(src, size_match.start()),
|
|
1017
|
+
"W220",
|
|
1018
|
+
(
|
|
1019
|
+
f"DownloadSize {declared} does not match the prefetches' total of "
|
|
1020
|
+
f"{total} bytes; add `{DOWNLOAD_SIZE_MARKER}` if intentional"
|
|
1021
|
+
),
|
|
1022
|
+
)
|
|
1023
|
+
]
|
|
1024
|
+
|
|
1025
|
+
|
|
1026
|
+
def _download_size_mismatch(src):
|
|
1027
|
+
"""Return (size_match, declared, total, unzip) when W220 applies, else None.
|
|
1028
|
+
|
|
1029
|
+
Shared by check_download_size_total and fix_download_size_total so the
|
|
1030
|
+
fixer rewrites exactly what the check reports.
|
|
1031
|
+
"""
|
|
1032
|
+
size_match = DOWNLOAD_SIZE_TAG_RE.search(src)
|
|
1033
|
+
if not size_match:
|
|
1034
|
+
return None
|
|
1035
|
+
declared = _strip_cdata(size_match.group(1))
|
|
1036
|
+
if not DOWNLOAD_SIZE_RE.match(declared):
|
|
1037
|
+
return None # E203's business
|
|
1038
|
+
totals = []
|
|
1039
|
+
for match in ACTIONSCRIPT_FULL_RE.finditer(src):
|
|
1040
|
+
body = match.group(2)
|
|
1041
|
+
cdata = CDATA_RE.match(body.strip())
|
|
1042
|
+
body = cdata.group(1) if cdata else _xml_unescape(body)
|
|
1043
|
+
if PREFETCH_LINE_RE.search(body) or any(
|
|
1044
|
+
PREFETCH_LINE_RE.match(line) for line in body.split("\n")
|
|
1045
|
+
):
|
|
1046
|
+
totals.append(_prefetch_total(body))
|
|
1047
|
+
if len(totals) != 1 or totals[0] is None:
|
|
1048
|
+
return None
|
|
1049
|
+
total, unzip = totals[0]
|
|
1050
|
+
if int(declared) in (total, total - unzip):
|
|
1051
|
+
return None
|
|
1052
|
+
return size_match, declared, total, unzip
|
|
1053
|
+
|
|
1054
|
+
|
|
1055
|
+
# W221: an action `setting "N"="V" on ...` line, and the two relevance shapes
|
|
1056
|
+
# that mean "relevant until setting N is V"
|
|
1057
|
+
SETTING_SET_RE = re.compile(
|
|
1058
|
+
r'^[ \t]*setting\s+"([^"]+)"\s*=\s*"([^"]*)"\s+on\b', re.IGNORECASE | re.MULTILINE
|
|
1059
|
+
)
|
|
1060
|
+
_VALUE_IT = r"value\s+of\s+it"
|
|
1061
|
+
SETTING_UNTIL_RES = (
|
|
1062
|
+
re.compile(
|
|
1063
|
+
r'\bnot\s+exists\s+settings?\s+"([^"]+)"\s+whose\s*\(\s*'
|
|
1064
|
+
r'(?:"([^"]*)"\s*=\s*' + _VALUE_IT + r"|" + _VALUE_IT + r'\s*=\s*"([^"]*)")'
|
|
1065
|
+
r"\s*\)\s+of\s+client\b",
|
|
1066
|
+
re.IGNORECASE,
|
|
1067
|
+
),
|
|
1068
|
+
re.compile(
|
|
1069
|
+
r'\bvalue\s+of\s+setting\s+"([^"]+)"\s+of\s+client\s*!=\s*"([^"]*)"',
|
|
1070
|
+
re.IGNORECASE,
|
|
1071
|
+
),
|
|
1072
|
+
re.compile(
|
|
1073
|
+
r'(?<![\w"])"([^"]*)"\s*!=\s*value\s+of\s+setting\s+"([^"]+)"\s+of\s+client\b',
|
|
1074
|
+
re.IGNORECASE,
|
|
1075
|
+
),
|
|
1076
|
+
)
|
|
1077
|
+
|
|
1078
|
+
|
|
1079
|
+
def _setting_targets(relevance):
|
|
1080
|
+
"""Yield (setting name, value) for each "relevant until N is V" test."""
|
|
1081
|
+
first, second, third = SETTING_UNTIL_RES
|
|
1082
|
+
for match in first.finditer(relevance):
|
|
1083
|
+
yield match.group(1), (
|
|
1084
|
+
match.group(2) if match.group(2) is not None else match.group(3)
|
|
1085
|
+
)
|
|
1086
|
+
for match in second.finditer(relevance):
|
|
1087
|
+
yield match.group(1), match.group(2)
|
|
1088
|
+
for match in third.finditer(relevance):
|
|
1089
|
+
yield match.group(2), match.group(1)
|
|
1090
|
+
|
|
1091
|
+
|
|
1092
|
+
def check_setting_value_target(src):
|
|
1093
|
+
"""W221: the relevance waits for a setting value the action never sets.
|
|
1094
|
+
|
|
1095
|
+
`not exists settings "N" whose ("V" = value of it) of client` (or
|
|
1096
|
+
`value of setting "N" of client != "V"`) is relevant *until* N is V, so
|
|
1097
|
+
the action that fixes it must set exactly V; an action setting N to
|
|
1098
|
+
anything else leaves the content relevant forever after it runs --
|
|
1099
|
+
usually a typo or a value edited in one place only. Relevance branches
|
|
1100
|
+
may wait for several values; it is enough that the action sets one of
|
|
1101
|
+
them (e.g. each under its own `if`). A positive test of
|
|
1102
|
+
the current value (`exists setting ... = "bad"`) is legitimate and not
|
|
1103
|
+
compared, nor is a value set from a `{...}` substitution.
|
|
1104
|
+
"""
|
|
1105
|
+
targets = {}
|
|
1106
|
+
for match in RELEVANCE_TAG_RE.finditer(src):
|
|
1107
|
+
inner = match.group(1)
|
|
1108
|
+
cdata = CDATA_RE.match(inner.strip())
|
|
1109
|
+
relevance = cdata.group(1) if cdata else _xml_unescape(inner)
|
|
1110
|
+
for name, value in _setting_targets(relevance):
|
|
1111
|
+
entry = targets.setdefault(
|
|
1112
|
+
name.lower(), [name, set(), _lineno(src, match.start())]
|
|
1113
|
+
)
|
|
1114
|
+
entry[1].add(value)
|
|
1115
|
+
if not targets:
|
|
1116
|
+
return []
|
|
1117
|
+
set_values = {} # lowercased name -> values the actions set
|
|
1118
|
+
for match in ACTIONSCRIPT_FULL_RE.finditer(src):
|
|
1119
|
+
body = match.group(2)
|
|
1120
|
+
cdata = CDATA_RE.match(body.strip())
|
|
1121
|
+
body = cdata.group(1) if cdata else _xml_unescape(body)
|
|
1122
|
+
for set_match in SETTING_SET_RE.finditer(body):
|
|
1123
|
+
set_values.setdefault(set_match.group(1).lower(), set()).add(
|
|
1124
|
+
set_match.group(2)
|
|
1125
|
+
)
|
|
1126
|
+
issues = []
|
|
1127
|
+
for key, (name, wanted, lineno) in targets.items():
|
|
1128
|
+
values = set_values.get(key)
|
|
1129
|
+
# set from a substitution: unknowable; any match: some branch fixes it
|
|
1130
|
+
if not values or any("{" in value for value in values) or values & wanted:
|
|
1131
|
+
continue
|
|
1132
|
+
issues.append(
|
|
1133
|
+
(
|
|
1134
|
+
lineno,
|
|
1135
|
+
"W221",
|
|
1136
|
+
(
|
|
1137
|
+
f'Relevance is true until setting "{name}" is '
|
|
1138
|
+
f'{" or ".join(sorted(repr(v) for v in wanted))}, but the '
|
|
1139
|
+
f'action sets it to {" or ".join(sorted(repr(v) for v in values))}, '
|
|
1140
|
+
"so the content stays relevant after it runs; add "
|
|
1141
|
+
f"`{SETTING_VALUE_MARKER}` if intentional"
|
|
1142
|
+
),
|
|
1143
|
+
)
|
|
1144
|
+
)
|
|
1145
|
+
return issues
|
|
1146
|
+
|
|
1147
|
+
|
|
1148
|
+
def check_download_size_value(src):
|
|
1149
|
+
"""E203: every <DownloadSize> present must be 0 or a positive integer."""
|
|
1150
|
+
issues = []
|
|
1151
|
+
for match in DOWNLOAD_SIZE_TAG_RE.finditer(src):
|
|
1152
|
+
value = _strip_cdata(match.group(1))
|
|
1153
|
+
if value == "" or not DOWNLOAD_SIZE_RE.match(value):
|
|
1154
|
+
issues.append(
|
|
1155
|
+
(
|
|
1156
|
+
_lineno(src, match.start()),
|
|
1157
|
+
"E203",
|
|
1158
|
+
(
|
|
1159
|
+
f'DownloadSize "{value}" is not 0 or a positive integer; '
|
|
1160
|
+
f"add `{DOWNLOAD_SIZE_MARKER}` if intentional"
|
|
1161
|
+
),
|
|
1162
|
+
)
|
|
1163
|
+
)
|
|
1164
|
+
return issues
|
|
1165
|
+
|
|
1166
|
+
|
|
1167
|
+
def check_action_ui_metadata(src):
|
|
1168
|
+
"""E206: an action-ui-metadata value must be well-formed."""
|
|
1169
|
+
issues = []
|
|
1170
|
+
for match in NAMED_MIMEFIELD_RE.finditer(src):
|
|
1171
|
+
if match.group(1).strip() != "action-ui-metadata":
|
|
1172
|
+
continue
|
|
1173
|
+
raw = match.group(2)
|
|
1174
|
+
value = _strip_cdata(raw)
|
|
1175
|
+
# outside CDATA the JSON's quotes are written as `"`; parse the
|
|
1176
|
+
# decoded text the agent actually reads (inside CDATA it is literal)
|
|
1177
|
+
decoded = value if CDATA_RE.match(raw.strip()) else _xml_unescape(value)
|
|
1178
|
+
if not _valid_action_ui_metadata(decoded):
|
|
1179
|
+
issues.append(
|
|
1180
|
+
(
|
|
1181
|
+
_lineno(src, match.start()),
|
|
1182
|
+
"E206",
|
|
1183
|
+
(
|
|
1184
|
+
f'action-ui-metadata "{_quote(value)}" is not well-formed; add '
|
|
1185
|
+
f"`{ACTION_UI_METADATA_MARKER}` if intentional"
|
|
1186
|
+
),
|
|
1187
|
+
)
|
|
1188
|
+
)
|
|
1189
|
+
return issues
|
|
1190
|
+
|
|
1191
|
+
|
|
1192
|
+
def check_cpe23(src):
|
|
1193
|
+
"""E205: an x-fixlet-cpe23-item-name value must be a valid CPE 2.3 string."""
|
|
1194
|
+
issues = []
|
|
1195
|
+
for match in NAMED_MIMEFIELD_RE.finditer(src):
|
|
1196
|
+
if match.group(1).strip().lower() != "x-fixlet-cpe23-item-name":
|
|
1197
|
+
continue
|
|
1198
|
+
value = _strip_cdata(match.group(2))
|
|
1199
|
+
if not _valid_cpe23(value):
|
|
1200
|
+
issues.append(
|
|
1201
|
+
(
|
|
1202
|
+
_lineno(src, match.start()),
|
|
1203
|
+
"E205",
|
|
1204
|
+
(
|
|
1205
|
+
f'x-fixlet-cpe23-item-name "{value}" is not a valid CPE 2.3 '
|
|
1206
|
+
f"string; add `{CPE_MARKER}` if intentional"
|
|
1207
|
+
),
|
|
1208
|
+
)
|
|
1209
|
+
)
|
|
1210
|
+
return issues
|
|
1211
|
+
|
|
1212
|
+
|
|
1213
|
+
def check_cdata_required(src):
|
|
1214
|
+
"""E207: a Description/Relevance/ActionScript that entity-escapes < > &.
|
|
1215
|
+
|
|
1216
|
+
Such content should use <![CDATA[ ... ]]> instead. Elements already wrapped in
|
|
1217
|
+
CDATA are fine, and elements whose body is real child markup (an action's
|
|
1218
|
+
<Description> of <PreLink>/<Link>/<PostLink>) are not text bodies at all, so
|
|
1219
|
+
both are skipped.
|
|
1220
|
+
"""
|
|
1221
|
+
issues = []
|
|
1222
|
+
for match in CDATA_ELEMENT_RE.finditer(src):
|
|
1223
|
+
tag, inner = match.group(1), match.group(2)
|
|
1224
|
+
if "<![CDATA[" in inner or CHILD_ELEMENT_RE.search(inner):
|
|
1225
|
+
continue
|
|
1226
|
+
if SPECIAL_ENTITY_RE.search(inner):
|
|
1227
|
+
issues.append(
|
|
1228
|
+
(
|
|
1229
|
+
_lineno(src, match.start()),
|
|
1230
|
+
"E207",
|
|
1231
|
+
(
|
|
1232
|
+
f"{tag} entity-escapes a character (< > &) that requires "
|
|
1233
|
+
f"<![CDATA[ ... ]]>; add `{CDATA_MARKER}` if intentional"
|
|
1234
|
+
),
|
|
1235
|
+
)
|
|
1236
|
+
)
|
|
1237
|
+
return issues
|
|
1238
|
+
|
|
1239
|
+
|
|
1240
|
+
def _js_line_comments(js):
|
|
1241
|
+
"""Find `//` end-of-line comments in `js` outside strings/templates/regex
|
|
1242
|
+
literals/block comments; return [(start, end, body), ...].
|
|
1243
|
+
|
|
1244
|
+
`start`/`end` are offsets into `js` spanning the comment from its opening
|
|
1245
|
+
`//` up to (not including) the line terminator or end of string; `body` is
|
|
1246
|
+
the text after `//` on that line. This is a lightweight tokenizer, good
|
|
1247
|
+
enough for typical BigFix Description scripts -- not a full ECMAScript
|
|
1248
|
+
parser. A `/` starts a regex literal unless the previous significant token
|
|
1249
|
+
was a value (an identifier/number, a closing string/template/regex, or `)`
|
|
1250
|
+
/ `]`); this is the standard division-vs-regex heuristic and can misjudge
|
|
1251
|
+
a keyword-suffixed regex literal (e.g. `return /x/`) as division, which is
|
|
1252
|
+
not expected to appear in a BES Description script. The legacy `<!-- ... `
|
|
1253
|
+
/ `//-->` HTML-comment-hiding idiom is recognized (a comment whose body,
|
|
1254
|
+
stripped, is exactly `-->`) and is not reported -- rewriting it would
|
|
1255
|
+
break the hiding trick, and it is boilerplate, not content.
|
|
1256
|
+
"""
|
|
1257
|
+
hits = []
|
|
1258
|
+
i = 0
|
|
1259
|
+
n = len(js)
|
|
1260
|
+
prev_value = False
|
|
1261
|
+
while i < n:
|
|
1262
|
+
c = js[i]
|
|
1263
|
+
if c in ("'", '"'):
|
|
1264
|
+
i += 1
|
|
1265
|
+
while i < n and js[i] != c:
|
|
1266
|
+
i += 2 if js[i] == "\\" and i + 1 < n else 1
|
|
1267
|
+
i += 1
|
|
1268
|
+
prev_value = True
|
|
1269
|
+
continue
|
|
1270
|
+
if c == "`":
|
|
1271
|
+
i += 1
|
|
1272
|
+
while i < n and js[i] != "`":
|
|
1273
|
+
i += 2 if js[i] == "\\" and i + 1 < n else 1
|
|
1274
|
+
i += 1
|
|
1275
|
+
prev_value = True
|
|
1276
|
+
continue
|
|
1277
|
+
if c == "/" and i + 1 < n and js[i + 1] == "*":
|
|
1278
|
+
end = js.find("*/", i + 2)
|
|
1279
|
+
i = end + 2 if end != -1 else n
|
|
1280
|
+
continue
|
|
1281
|
+
if c == "/" and i + 1 < n and js[i + 1] == "/":
|
|
1282
|
+
eol = js.find("\n", i)
|
|
1283
|
+
if eol == -1:
|
|
1284
|
+
eol = n
|
|
1285
|
+
body = js[i + 2 : eol]
|
|
1286
|
+
if body.strip() != "-->":
|
|
1287
|
+
hits.append((i, eol, body))
|
|
1288
|
+
i = eol
|
|
1289
|
+
prev_value = False
|
|
1290
|
+
continue
|
|
1291
|
+
if c == "/":
|
|
1292
|
+
if prev_value:
|
|
1293
|
+
i += 1
|
|
1294
|
+
prev_value = True
|
|
1295
|
+
continue
|
|
1296
|
+
j = i + 1
|
|
1297
|
+
in_class = False
|
|
1298
|
+
while j < n:
|
|
1299
|
+
ch = js[j]
|
|
1300
|
+
if ch == "\\" and j + 1 < n:
|
|
1301
|
+
j += 2
|
|
1302
|
+
continue
|
|
1303
|
+
if ch == "\n":
|
|
1304
|
+
break
|
|
1305
|
+
if ch == "[":
|
|
1306
|
+
in_class = True
|
|
1307
|
+
elif ch == "]":
|
|
1308
|
+
in_class = False
|
|
1309
|
+
elif ch == "/" and not in_class:
|
|
1310
|
+
j += 1
|
|
1311
|
+
break
|
|
1312
|
+
j += 1
|
|
1313
|
+
while j < n and js[j].isalpha():
|
|
1314
|
+
j += 1
|
|
1315
|
+
i = j
|
|
1316
|
+
prev_value = True
|
|
1317
|
+
continue
|
|
1318
|
+
if c.isalnum() or c in "_$":
|
|
1319
|
+
j = i
|
|
1320
|
+
while j < n and (js[j].isalnum() or js[j] in "_$"):
|
|
1321
|
+
j += 1
|
|
1322
|
+
i = j
|
|
1323
|
+
prev_value = True
|
|
1324
|
+
continue
|
|
1325
|
+
if c in ")]":
|
|
1326
|
+
prev_value = True
|
|
1327
|
+
i += 1
|
|
1328
|
+
continue
|
|
1329
|
+
if c.isspace():
|
|
1330
|
+
i += 1
|
|
1331
|
+
continue
|
|
1332
|
+
prev_value = False
|
|
1333
|
+
i += 1
|
|
1334
|
+
return hits
|
|
1335
|
+
|
|
1336
|
+
|
|
1337
|
+
def check_script_comments(src):
|
|
1338
|
+
"""E221: a `//` line comment inside a Description <script> block.
|
|
1339
|
+
|
|
1340
|
+
<script> may appear as literal (CDATA-wrapped or plain) markup, in which
|
|
1341
|
+
case it is scanned directly for an exact file line number, or hidden
|
|
1342
|
+
behind entity escapes (a Description that should have been CDATA-wrapped,
|
|
1343
|
+
per E207), in which case it is unescaped first -- newline count is
|
|
1344
|
+
unaffected by entity decoding, so the reported line stays accurate. Only
|
|
1345
|
+
<script> blocks are scanned: ordinary Description prose (a bare
|
|
1346
|
+
`http://...` URL, a `//`-prefixed protocol-relative href) is never
|
|
1347
|
+
special-cased away because it is never scanned in the first place.
|
|
1348
|
+
"""
|
|
1349
|
+
issues = []
|
|
1350
|
+
for desc_match in DESCRIPTION_ELEMENT_RE.finditer(src):
|
|
1351
|
+
body = desc_match.group(1)
|
|
1352
|
+
body_start = desc_match.start(1)
|
|
1353
|
+
if SCRIPT_OPEN_RE.search(body):
|
|
1354
|
+
for script_match in SCRIPT_BLOCK_RE.finditer(body):
|
|
1355
|
+
script_body = script_match.group(1)
|
|
1356
|
+
script_body_start = body_start + script_match.start(1)
|
|
1357
|
+
for start, _end, _comment_body in _js_line_comments(script_body):
|
|
1358
|
+
issues.append(_e221_issue(_lineno(src, script_body_start + start)))
|
|
1359
|
+
continue
|
|
1360
|
+
decoded = _xml_unescape(body)
|
|
1361
|
+
if not SCRIPT_OPEN_RE.search(decoded):
|
|
1362
|
+
continue
|
|
1363
|
+
base_line = _lineno(src, body_start)
|
|
1364
|
+
for script_match in SCRIPT_BLOCK_RE.finditer(decoded):
|
|
1365
|
+
script_body = script_match.group(1)
|
|
1366
|
+
script_base_line = base_line + decoded.count("\n", 0, script_match.start(1))
|
|
1367
|
+
for start, _end, _comment_body in _js_line_comments(script_body):
|
|
1368
|
+
lineno = script_base_line + script_body.count("\n", 0, start)
|
|
1369
|
+
issues.append(_e221_issue(lineno))
|
|
1370
|
+
return issues
|
|
1371
|
+
|
|
1372
|
+
|
|
1373
|
+
def _e221_issue(lineno):
|
|
1374
|
+
return (
|
|
1375
|
+
lineno,
|
|
1376
|
+
"E221",
|
|
1377
|
+
(
|
|
1378
|
+
"Description <script> uses a `//` comment, which is lost when the "
|
|
1379
|
+
"file is converted to FXF (Description newlines are collapsed); "
|
|
1380
|
+
f"use `/* ... */` instead; add `{SCRIPT_COMMENT_MARKER}` if intentional"
|
|
1381
|
+
),
|
|
1382
|
+
)
|
|
1383
|
+
|
|
1384
|
+
|
|
1385
|
+
def check_actionscript_cdata(src):
|
|
1386
|
+
"""W204: an ActionScript body should be wrapped in <![CDATA[ ... ]]>.
|
|
1387
|
+
|
|
1388
|
+
Bodies that entity-escape special chars are owned by E207, not warned here.
|
|
1389
|
+
"""
|
|
1390
|
+
issues = []
|
|
1391
|
+
for match in ACTIONSCRIPT_FULL_RE.finditer(src):
|
|
1392
|
+
body = match.group(2)
|
|
1393
|
+
if "<![CDATA[" in body or SPECIAL_ENTITY_RE.search(body):
|
|
1394
|
+
continue
|
|
1395
|
+
issues.append(
|
|
1396
|
+
(
|
|
1397
|
+
_lineno(src, match.start()),
|
|
1398
|
+
"W204",
|
|
1399
|
+
(
|
|
1400
|
+
"ActionScript body is not wrapped in <![CDATA[ ... ]]>; add "
|
|
1401
|
+
f"`{CDATA_MARKER}` if intentional (auto-fixable under --strict)"
|
|
1402
|
+
),
|
|
1403
|
+
)
|
|
1404
|
+
)
|
|
1405
|
+
return issues
|
|
1406
|
+
|
|
1407
|
+
|
|
1408
|
+
def check_actionscript_blank_lines(src):
|
|
1409
|
+
"""W205: no more than one blank line before </ActionScript>."""
|
|
1410
|
+
issues = []
|
|
1411
|
+
for match in BLANK_BEFORE_CLOSE_RE.finditer(src):
|
|
1412
|
+
issues.append(
|
|
1413
|
+
(
|
|
1414
|
+
_lineno(src, match.start()),
|
|
1415
|
+
"W205",
|
|
1416
|
+
(
|
|
1417
|
+
"more than one blank line before </ActionScript>; add "
|
|
1418
|
+
f"`{ACTION_BLANK_LINES_MARKER}` if intentional"
|
|
1419
|
+
),
|
|
1420
|
+
)
|
|
1421
|
+
)
|
|
1422
|
+
return issues
|
|
1423
|
+
|
|
1424
|
+
|
|
1425
|
+
def check_cdata_close(src):
|
|
1426
|
+
"""E215: no whitespace may surround an ActionScript close tag.
|
|
1427
|
+
|
|
1428
|
+
Applies to both the CDATA terminator (`]]></ActionScript>`) and, for a
|
|
1429
|
+
non-CDATA body, the bare close (`</ActionScript>`).
|
|
1430
|
+
"""
|
|
1431
|
+
issues = []
|
|
1432
|
+
for match in CDATA_CLOSE_RE.finditer(src):
|
|
1433
|
+
canonical = (
|
|
1434
|
+
CDATA_CLOSE_CANONICAL
|
|
1435
|
+
if "]]>" in match.group(0)
|
|
1436
|
+
else CDATA_CLOSE_CANONICAL_PLAIN
|
|
1437
|
+
)
|
|
1438
|
+
if match.group(0) == canonical:
|
|
1439
|
+
continue
|
|
1440
|
+
issues.append(
|
|
1441
|
+
(
|
|
1442
|
+
_lineno(src, match.start()),
|
|
1443
|
+
"E215",
|
|
1444
|
+
(
|
|
1445
|
+
"whitespace before the ActionScript close tag (it becomes "
|
|
1446
|
+
f"a whitespace-only last action line); add `{CDATA_CLOSE_MARKER}` "
|
|
1447
|
+
"if intentional"
|
|
1448
|
+
),
|
|
1449
|
+
)
|
|
1450
|
+
)
|
|
1451
|
+
return issues
|
|
1452
|
+
|
|
1453
|
+
|
|
1454
|
+
def check_prefetch_lines(src, schemes=DEFAULT_PREFETCH_URL_SCHEMES):
|
|
1455
|
+
"""W206: a prefetch / add-prefetch-item line must match the expected shape.
|
|
1456
|
+
|
|
1457
|
+
`schemes` is the set of URL schemes the URL may use (see
|
|
1458
|
+
--prefetch-url-schemes); matched case-insensitively.
|
|
1459
|
+
"""
|
|
1460
|
+
prefetch_ok_re = _prefetch_ok_re(frozenset(schemes))
|
|
1461
|
+
issues = []
|
|
1462
|
+
for match in ACTIONSCRIPT_FULL_RE.finditer(src):
|
|
1463
|
+
body = match.group(2)
|
|
1464
|
+
base = _lineno(src, match.start(2))
|
|
1465
|
+
for offset, raw in enumerate(body.splitlines()):
|
|
1466
|
+
line = raw.rstrip("\r")
|
|
1467
|
+
stripped = line.strip()
|
|
1468
|
+
is_prefetch = stripped.startswith("prefetch ")
|
|
1469
|
+
is_add = "add prefetch item" in stripped
|
|
1470
|
+
if not (is_prefetch or is_add):
|
|
1471
|
+
continue
|
|
1472
|
+
if prefetch_ok_re.fullmatch(line) or prefetch_ok_re.fullmatch(stripped):
|
|
1473
|
+
continue
|
|
1474
|
+
issues.append(
|
|
1475
|
+
(
|
|
1476
|
+
base + offset,
|
|
1477
|
+
"W206",
|
|
1478
|
+
(
|
|
1479
|
+
f'prefetch line "{stripped[:60]}" does not match the expected '
|
|
1480
|
+
f"shape; add `{PREFETCH_MARKER}` if intentional"
|
|
1481
|
+
),
|
|
1482
|
+
)
|
|
1483
|
+
)
|
|
1484
|
+
return issues
|
|
1485
|
+
|
|
1486
|
+
|
|
1487
|
+
def check_prefetch_https(src):
|
|
1488
|
+
"""W207: a prefetch / add-prefetch-item URL should use https, not http."""
|
|
1489
|
+
issues = []
|
|
1490
|
+
for match in ACTIONSCRIPT_FULL_RE.finditer(src):
|
|
1491
|
+
body = match.group(2)
|
|
1492
|
+
base = _lineno(src, match.start(2))
|
|
1493
|
+
for offset, raw in enumerate(body.splitlines()):
|
|
1494
|
+
line = raw.rstrip("\r")
|
|
1495
|
+
stripped = line.strip()
|
|
1496
|
+
is_prefetch = stripped.startswith("prefetch ")
|
|
1497
|
+
is_add = "add prefetch item" in stripped
|
|
1498
|
+
if not (is_prefetch or is_add):
|
|
1499
|
+
continue
|
|
1500
|
+
scheme = PREFETCH_URL_SCHEME_RE.search(line)
|
|
1501
|
+
if scheme is not None and scheme.group(1).lower() == "http":
|
|
1502
|
+
issues.append(
|
|
1503
|
+
(
|
|
1504
|
+
base + offset,
|
|
1505
|
+
"W207",
|
|
1506
|
+
(
|
|
1507
|
+
f'prefetch URL in "{stripped[:60]}" uses http, not https; '
|
|
1508
|
+
f"add `{PREFETCH_HTTPS_MARKER}` if intentional"
|
|
1509
|
+
),
|
|
1510
|
+
)
|
|
1511
|
+
)
|
|
1512
|
+
return issues
|
|
1513
|
+
|
|
1514
|
+
|
|
1515
|
+
def _actionscript_is_empty(body):
|
|
1516
|
+
"""True if an ActionScript body has only blank lines and //-comment lines."""
|
|
1517
|
+
content = _strip_cdata(body)
|
|
1518
|
+
for raw in content.splitlines():
|
|
1519
|
+
line = raw.strip()
|
|
1520
|
+
if line and not line.startswith("//"):
|
|
1521
|
+
return False
|
|
1522
|
+
return True
|
|
1523
|
+
|
|
1524
|
+
|
|
1525
|
+
def check_empty_actionscript(src):
|
|
1526
|
+
"""W208: an ActionScript body must contain more than blank/// lines.
|
|
1527
|
+
|
|
1528
|
+
Skipped when the item's Title contains "warning" or "test"
|
|
1529
|
+
(case-insensitive): warning-only and test content commonly has no
|
|
1530
|
+
actionable ActionScript body.
|
|
1531
|
+
"""
|
|
1532
|
+
issues = []
|
|
1533
|
+
title_match = TITLE_TAG_RE.search(src)
|
|
1534
|
+
if title_match:
|
|
1535
|
+
title_lower = _strip_cdata(title_match.group(1)).lower()
|
|
1536
|
+
if "warning" in title_lower or "test" in title_lower:
|
|
1537
|
+
return issues
|
|
1538
|
+
for match in ACTIONSCRIPT_FULL_RE.finditer(src):
|
|
1539
|
+
if _actionscript_is_empty(match.group(2)):
|
|
1540
|
+
issues.append(
|
|
1541
|
+
(
|
|
1542
|
+
_lineno(src, match.start()),
|
|
1543
|
+
"W208",
|
|
1544
|
+
(
|
|
1545
|
+
"ActionScript body is empty (only blank lines and "
|
|
1546
|
+
f"//-comments); add `{ACTIONSCRIPT_EMPTY_MARKER}` if intentional"
|
|
1547
|
+
),
|
|
1548
|
+
)
|
|
1549
|
+
)
|
|
1550
|
+
return issues
|
|
1551
|
+
|
|
1552
|
+
|
|
1553
|
+
def check_dynamic_download(src):
|
|
1554
|
+
"""W211: warn on a dynamic `download` statement (prefer a static prefetch)."""
|
|
1555
|
+
issues = []
|
|
1556
|
+
for match in ACTIONSCRIPT_FULL_RE.finditer(src):
|
|
1557
|
+
body = match.group(2)
|
|
1558
|
+
base = _lineno(src, match.start(2))
|
|
1559
|
+
for offset, raw in enumerate(body.splitlines()):
|
|
1560
|
+
line = raw.rstrip("\r")
|
|
1561
|
+
if DOWNLOAD_STMT_RE.match(line):
|
|
1562
|
+
issues.append(
|
|
1563
|
+
(
|
|
1564
|
+
base + offset,
|
|
1565
|
+
"W211",
|
|
1566
|
+
(
|
|
1567
|
+
f'ActionScript uses a dynamic download statement "'
|
|
1568
|
+
f'{line.strip()[:60]}"; prefer a static prefetch; add '
|
|
1569
|
+
f"`{DOWNLOAD_MARKER}` if intentional"
|
|
1570
|
+
),
|
|
1571
|
+
)
|
|
1572
|
+
)
|
|
1573
|
+
return issues
|
|
1574
|
+
|
|
1575
|
+
|
|
1576
|
+
def check_cve_names(src):
|
|
1577
|
+
"""E209: each <CVENames> value must be a valid CVE id; only one CVENames.
|
|
1578
|
+
|
|
1579
|
+
A single <CVENames> may hold several comma/semicolon/space-separated CVE
|
|
1580
|
+
ids, but a content object must not carry more than one <CVENames>
|
|
1581
|
+
element. The lone value `Unspecified` is accepted as the "no CVEs"
|
|
1582
|
+
sentinel; `N/A` and friends are not.
|
|
1583
|
+
"""
|
|
1584
|
+
issues = []
|
|
1585
|
+
matches = list(CVENAMES_TAG_RE.finditer(src))
|
|
1586
|
+
if len(matches) > 1:
|
|
1587
|
+
issues.append(
|
|
1588
|
+
(
|
|
1589
|
+
_lineno(src, matches[1].start()),
|
|
1590
|
+
"E209",
|
|
1591
|
+
(
|
|
1592
|
+
"more than one <CVENames> element in a content object; combine "
|
|
1593
|
+
f"into one comma-separated CVENames; add `{CVE_NAMES_MARKER}` "
|
|
1594
|
+
"if intentional"
|
|
1595
|
+
),
|
|
1596
|
+
)
|
|
1597
|
+
)
|
|
1598
|
+
for match in matches:
|
|
1599
|
+
value = _strip_cdata(match.group(1))
|
|
1600
|
+
if value.strip() == CVE_UNSPECIFIED:
|
|
1601
|
+
continue
|
|
1602
|
+
for token in re.split(r"[,;\s]+", value):
|
|
1603
|
+
if token and not CVE_RE.match(token):
|
|
1604
|
+
issues.append(
|
|
1605
|
+
(
|
|
1606
|
+
_lineno(src, match.start()),
|
|
1607
|
+
"E209",
|
|
1608
|
+
(
|
|
1609
|
+
f'CVENames value "{token}" is not a valid CVE id (e.g. '
|
|
1610
|
+
f"CVE-2021-44228); add `{CVE_NAMES_MARKER}` if intentional"
|
|
1611
|
+
),
|
|
1612
|
+
)
|
|
1613
|
+
)
|
|
1614
|
+
return issues
|
|
1615
|
+
|
|
1616
|
+
|
|
1617
|
+
def check_duplicate_mimefield_names(src):
|
|
1618
|
+
"""E210: two <MIMEField> entries in one object must not share a <Name>."""
|
|
1619
|
+
issues = []
|
|
1620
|
+
seen = {}
|
|
1621
|
+
for match in NAMED_MIMEFIELD_RE.finditer(src):
|
|
1622
|
+
name = match.group(1).strip()
|
|
1623
|
+
if name in seen:
|
|
1624
|
+
issues.append(
|
|
1625
|
+
(
|
|
1626
|
+
_lineno(src, match.start()),
|
|
1627
|
+
"E210",
|
|
1628
|
+
(
|
|
1629
|
+
f'duplicate MIMEField <Name> "{name}" in one content object; '
|
|
1630
|
+
f"add `{MIMEFIELD_DUP_MARKER}` if intentional"
|
|
1631
|
+
),
|
|
1632
|
+
)
|
|
1633
|
+
)
|
|
1634
|
+
else:
|
|
1635
|
+
seen[name] = match.start()
|
|
1636
|
+
return issues
|
|
1637
|
+
|
|
1638
|
+
|
|
1639
|
+
def check_link_text(src):
|
|
1640
|
+
"""W218: action link text should not contain a run of 2+ spaces.
|
|
1641
|
+
|
|
1642
|
+
`<PreLink>Click </PreLink><Link>here</Link><PostLink> to deploy this
|
|
1643
|
+
action.</PostLink>` is assembled into one sentence shown in the console.
|
|
1644
|
+
A double space in it is almost always where a generator substituted an
|
|
1645
|
+
empty product name, so the sentence reads "to deploy v11.3.2." -- the
|
|
1646
|
+
gap is the symptom, and there is nothing to auto-fix: the missing word
|
|
1647
|
+
is what needs restoring.
|
|
1648
|
+
"""
|
|
1649
|
+
issues = []
|
|
1650
|
+
for match in LINK_TEXT_RE.finditer(src):
|
|
1651
|
+
tag, inner = match.group(1), match.group(2)
|
|
1652
|
+
if "<![CDATA[" in inner:
|
|
1653
|
+
continue
|
|
1654
|
+
if DOUBLE_SPACE_RE.search(inner):
|
|
1655
|
+
issues.append(
|
|
1656
|
+
(
|
|
1657
|
+
_lineno(src, match.start()),
|
|
1658
|
+
"W218",
|
|
1659
|
+
(
|
|
1660
|
+
f"<{tag}> has a run of 2+ spaces, usually where a "
|
|
1661
|
+
"substitution expanded to nothing; add "
|
|
1662
|
+
f"`{LINK_TEXT_MARKER}` if intentional"
|
|
1663
|
+
),
|
|
1664
|
+
)
|
|
1665
|
+
)
|
|
1666
|
+
return issues
|
|
1667
|
+
|
|
1668
|
+
|
|
1669
|
+
def check_duplicate_analysis_properties(src):
|
|
1670
|
+
"""E220: <Property> entries in one Analysis must not share a Name or ID.
|
|
1671
|
+
|
|
1672
|
+
The console lets an edit introduce a second property with an existing
|
|
1673
|
+
name, and reporting then cannot tell the two apart; a repeated ID is the
|
|
1674
|
+
same collision in the field the API addresses properties by. Runs per
|
|
1675
|
+
content object, so two analyses may each carry the same property name.
|
|
1676
|
+
"""
|
|
1677
|
+
issues = []
|
|
1678
|
+
seen_names = set()
|
|
1679
|
+
seen_ids = set()
|
|
1680
|
+
for match in ANALYSIS_PROPERTY_RE.finditer(src):
|
|
1681
|
+
lineno = _lineno(src, match.start())
|
|
1682
|
+
name = match.group(1).strip()
|
|
1683
|
+
prop_id = (match.group(2) or "").strip()
|
|
1684
|
+
if name in seen_names:
|
|
1685
|
+
issues.append(
|
|
1686
|
+
(
|
|
1687
|
+
lineno,
|
|
1688
|
+
"E220",
|
|
1689
|
+
(
|
|
1690
|
+
f'duplicate Analysis <Property> Name "{name}" in one '
|
|
1691
|
+
f"Analysis; add `{ANALYSIS_PROPERTY_MARKER}` if intentional"
|
|
1692
|
+
),
|
|
1693
|
+
)
|
|
1694
|
+
)
|
|
1695
|
+
else:
|
|
1696
|
+
seen_names.add(name)
|
|
1697
|
+
if prop_id and prop_id in seen_ids:
|
|
1698
|
+
issues.append(
|
|
1699
|
+
(
|
|
1700
|
+
lineno,
|
|
1701
|
+
"E220",
|
|
1702
|
+
(
|
|
1703
|
+
f'duplicate Analysis <Property> ID "{prop_id}" in one '
|
|
1704
|
+
f"Analysis; add `{ANALYSIS_PROPERTY_MARKER}` if intentional"
|
|
1705
|
+
),
|
|
1706
|
+
)
|
|
1707
|
+
)
|
|
1708
|
+
elif prop_id:
|
|
1709
|
+
seen_ids.add(prop_id)
|
|
1710
|
+
return issues
|
|
1711
|
+
|
|
1712
|
+
|
|
1713
|
+
def check_success_criteria(src):
|
|
1714
|
+
"""E217: a <SuccessCriteria> body/Option combination must be consistent.
|
|
1715
|
+
|
|
1716
|
+
A CustomRelevance body must be real relevance (not empty, not the literal
|
|
1717
|
+
`false`, which can never succeed); a non-CustomRelevance SuccessCriteria
|
|
1718
|
+
(OriginalRelevance, RunToCompletion, or no Option at all) must have an
|
|
1719
|
+
empty body, since BigFix silently ignores anything else there.
|
|
1720
|
+
"""
|
|
1721
|
+
issues = []
|
|
1722
|
+
for match in SUCCESS_CRITERIA_RE.finditer(src):
|
|
1723
|
+
attrs, body = match.group(1), match.group(2)
|
|
1724
|
+
lineno = _lineno(src, match.start())
|
|
1725
|
+
option_match = SUCCESS_CRITERIA_OPTION_RE.search(attrs)
|
|
1726
|
+
option = option_match.group(1) if option_match else None
|
|
1727
|
+
value = _strip_cdata(body).strip()
|
|
1728
|
+
if option == "CustomRelevance":
|
|
1729
|
+
if value == "":
|
|
1730
|
+
issues.append(
|
|
1731
|
+
(
|
|
1732
|
+
lineno,
|
|
1733
|
+
"E217",
|
|
1734
|
+
(
|
|
1735
|
+
'SuccessCriteria Option="CustomRelevance" has an empty '
|
|
1736
|
+
f"relevance body; add `{SUCCESS_CRITERIA_MARKER}` if "
|
|
1737
|
+
"intentional"
|
|
1738
|
+
),
|
|
1739
|
+
)
|
|
1740
|
+
)
|
|
1741
|
+
elif value.lower() == "false":
|
|
1742
|
+
issues.append(
|
|
1743
|
+
(
|
|
1744
|
+
lineno,
|
|
1745
|
+
"E217",
|
|
1746
|
+
(
|
|
1747
|
+
'SuccessCriteria Option="CustomRelevance" body is the '
|
|
1748
|
+
"literal `false`; it can never succeed; add "
|
|
1749
|
+
f"`{SUCCESS_CRITERIA_MARKER}` if intentional"
|
|
1750
|
+
),
|
|
1751
|
+
)
|
|
1752
|
+
)
|
|
1753
|
+
elif value != "":
|
|
1754
|
+
issues.append(
|
|
1755
|
+
(
|
|
1756
|
+
lineno,
|
|
1757
|
+
"E217",
|
|
1758
|
+
(
|
|
1759
|
+
f'SuccessCriteria has a relevance body but Option is "{option}"'
|
|
1760
|
+
', not "CustomRelevance"; the body is silently ignored; add '
|
|
1761
|
+
f"`{SUCCESS_CRITERIA_MARKER}` if intentional"
|
|
1762
|
+
),
|
|
1763
|
+
)
|
|
1764
|
+
)
|
|
1765
|
+
return issues
|
|
1766
|
+
|
|
1767
|
+
|
|
1768
|
+
def check_duplicate_action_ids(src):
|
|
1769
|
+
"""E218: two Action/DefaultAction elements in one object must not share an ID."""
|
|
1770
|
+
issues = []
|
|
1771
|
+
seen = {}
|
|
1772
|
+
for match in ACTION_ID_RE.finditer(src):
|
|
1773
|
+
action_id = match.group(1)
|
|
1774
|
+
if action_id in seen:
|
|
1775
|
+
issues.append(
|
|
1776
|
+
(
|
|
1777
|
+
_lineno(src, match.start()),
|
|
1778
|
+
"E218",
|
|
1779
|
+
(
|
|
1780
|
+
f'duplicate Action ID "{action_id}" in one content object; '
|
|
1781
|
+
f"add `{ACTION_ID_MARKER}` if intentional"
|
|
1782
|
+
),
|
|
1783
|
+
)
|
|
1784
|
+
)
|
|
1785
|
+
else:
|
|
1786
|
+
seen[action_id] = match.start()
|
|
1787
|
+
return issues
|
|
1788
|
+
|
|
1789
|
+
|
|
1790
|
+
def check_evaluation_period(src):
|
|
1791
|
+
"""E219: an x-relevance-evaluation-period value must be a valid HH:MM:SS."""
|
|
1792
|
+
issues = []
|
|
1793
|
+
for match in NAMED_MIMEFIELD_RE.finditer(src):
|
|
1794
|
+
if match.group(1).strip().lower() != "x-relevance-evaluation-period":
|
|
1795
|
+
continue
|
|
1796
|
+
value = _strip_cdata(match.group(2))
|
|
1797
|
+
if not EVALUATION_PERIOD_RE.match(value):
|
|
1798
|
+
issues.append(
|
|
1799
|
+
(
|
|
1800
|
+
_lineno(src, match.start()),
|
|
1801
|
+
"E219",
|
|
1802
|
+
(
|
|
1803
|
+
f'x-relevance-evaluation-period "{value}" is not a valid '
|
|
1804
|
+
f"HH:MM:SS duration; add `{EVALUATION_PERIOD_MARKER}` if "
|
|
1805
|
+
"intentional"
|
|
1806
|
+
),
|
|
1807
|
+
)
|
|
1808
|
+
)
|
|
1809
|
+
return issues
|
|
1810
|
+
|
|
1811
|
+
|
|
1812
|
+
def _is_unspecified_severity(inner):
|
|
1813
|
+
"""True if a <SourceSeverity>'s raw text is `Unspecified` in any spelling.
|
|
1814
|
+
|
|
1815
|
+
Matches case-insensitively, CDATA-wrapped or not, and in the console's
|
|
1816
|
+
bracketed `<Unspecified>` form (entity-escaped outside CDATA).
|
|
1817
|
+
"""
|
|
1818
|
+
value = _strip_cdata(inner)
|
|
1819
|
+
if not CDATA_RE.match(inner.strip()):
|
|
1820
|
+
value = _xml_unescape(value)
|
|
1821
|
+
value = value.strip()
|
|
1822
|
+
if value.startswith("<") and value.endswith(">"):
|
|
1823
|
+
value = value[1:-1].strip()
|
|
1824
|
+
return value.lower() == "unspecified"
|
|
1825
|
+
|
|
1826
|
+
|
|
1827
|
+
def check_source_severity(src, allowed=CANONICAL_SEVERITIES):
|
|
1828
|
+
"""W216: a non-empty <SourceSeverity> must be in the allowed vocabulary.
|
|
1829
|
+
|
|
1830
|
+
`allowed` defaults to CANONICAL_SEVERITIES but can be overridden (see
|
|
1831
|
+
--severity-values) to whatever exact-case values a repo wants to permit.
|
|
1832
|
+
`Unspecified` (any case, bracketed or not) says nothing an empty
|
|
1833
|
+
SourceSeverity does not, so unless `allowed` lists it, it is reported as
|
|
1834
|
+
auto-fixable to empty (see fix_source_severity).
|
|
1835
|
+
"""
|
|
1836
|
+
issues = []
|
|
1837
|
+
for match in SOURCE_SEVERITY_RE.finditer(src):
|
|
1838
|
+
value = _strip_cdata(match.group(1)).strip()
|
|
1839
|
+
if value == "" or value in allowed:
|
|
1840
|
+
continue
|
|
1841
|
+
if _is_unspecified_severity(match.group(1)):
|
|
1842
|
+
issues.append(
|
|
1843
|
+
(
|
|
1844
|
+
_lineno(src, match.start()),
|
|
1845
|
+
"W216",
|
|
1846
|
+
(
|
|
1847
|
+
f'SourceSeverity "{value}" carries no information; '
|
|
1848
|
+
"leave it empty instead (auto-fixable); add "
|
|
1849
|
+
f"`{SEVERITY_MARKER}` if intentional"
|
|
1850
|
+
),
|
|
1851
|
+
)
|
|
1852
|
+
)
|
|
1853
|
+
continue
|
|
1854
|
+
issues.append(
|
|
1855
|
+
(
|
|
1856
|
+
_lineno(src, match.start()),
|
|
1857
|
+
"W216",
|
|
1858
|
+
(
|
|
1859
|
+
f'SourceSeverity "{value}" is not one of '
|
|
1860
|
+
f"{sorted(allowed)}; add `{SEVERITY_MARKER}` if intentional"
|
|
1861
|
+
),
|
|
1862
|
+
)
|
|
1863
|
+
)
|
|
1864
|
+
return issues
|
|
1865
|
+
|
|
1866
|
+
|
|
1867
|
+
def check_title(src):
|
|
1868
|
+
"""E211/W209/W214: a <Title> placeholder, stray whitespace, or TODO marker."""
|
|
1869
|
+
issues = []
|
|
1870
|
+
for match in TITLE_TAG_RE.finditer(src):
|
|
1871
|
+
inner = match.group(1)
|
|
1872
|
+
lineno = _lineno(src, match.start())
|
|
1873
|
+
value = _strip_cdata(inner)
|
|
1874
|
+
if value.strip().lower() in DEFAULT_TITLES:
|
|
1875
|
+
issues.append(
|
|
1876
|
+
(
|
|
1877
|
+
lineno,
|
|
1878
|
+
"E211",
|
|
1879
|
+
(
|
|
1880
|
+
f'Title "{value.strip()}" is a default placeholder; give it a '
|
|
1881
|
+
f"real title; add `{TITLE_MARKER}` if intentional"
|
|
1882
|
+
),
|
|
1883
|
+
)
|
|
1884
|
+
)
|
|
1885
|
+
if "<![CDATA[" not in inner and (inner != inner.strip() or "\t" in inner):
|
|
1886
|
+
issues.append(
|
|
1887
|
+
(
|
|
1888
|
+
lineno,
|
|
1889
|
+
"W209",
|
|
1890
|
+
(
|
|
1891
|
+
"Title has leading/trailing whitespace or embedded tabs; add "
|
|
1892
|
+
f"`{TITLE_MARKER}` if intentional"
|
|
1893
|
+
),
|
|
1894
|
+
)
|
|
1895
|
+
)
|
|
1896
|
+
if "<![CDATA[" not in inner and DOUBLE_SPACE_RE.search(inner.strip()):
|
|
1897
|
+
issues.append(
|
|
1898
|
+
(
|
|
1899
|
+
lineno,
|
|
1900
|
+
"W209",
|
|
1901
|
+
(
|
|
1902
|
+
f'Title "{value.strip()}" has a run of 2+ spaces, usually '
|
|
1903
|
+
"where a substitution expanded to nothing; add "
|
|
1904
|
+
f"`{TITLE_MARKER}` if intentional"
|
|
1905
|
+
),
|
|
1906
|
+
)
|
|
1907
|
+
)
|
|
1908
|
+
if TODO_RE.search(value):
|
|
1909
|
+
issues.append(
|
|
1910
|
+
(
|
|
1911
|
+
lineno,
|
|
1912
|
+
"W214",
|
|
1913
|
+
(
|
|
1914
|
+
f'Title "{value.strip()}" contains a TODO/FIXME marker; add '
|
|
1915
|
+
f"`{TITLE_MARKER}` if intentional"
|
|
1916
|
+
),
|
|
1917
|
+
)
|
|
1918
|
+
)
|
|
1919
|
+
return issues
|
|
1920
|
+
|
|
1921
|
+
|
|
1922
|
+
# a relevance `/* ... */` comment (relevance has no line comments)
|
|
1923
|
+
RELEVANCE_COMMENT_RE = re.compile(r"/\*.*?\*/", re.DOTALL)
|
|
1924
|
+
|
|
1925
|
+
|
|
1926
|
+
def _relevance_literal(value):
|
|
1927
|
+
"""Return `value` lowercased with comments and wrapping parens removed.
|
|
1928
|
+
|
|
1929
|
+
`true /* old clause */` and `((TRUE))` are still the literal `true`, so
|
|
1930
|
+
E212/W212 compare this rather than the raw text. Only parentheses that
|
|
1931
|
+
wrap the whole expression are removed: `(true) and (false)` is kept.
|
|
1932
|
+
"""
|
|
1933
|
+
value = RELEVANCE_COMMENT_RE.sub(" ", value).strip()
|
|
1934
|
+
while value.startswith("(") and value.endswith(")"):
|
|
1935
|
+
depth = 0
|
|
1936
|
+
for index, char in enumerate(value):
|
|
1937
|
+
depth += {"(": 1, ")": -1}.get(char, 0)
|
|
1938
|
+
if depth == 0 and index < len(value) - 1:
|
|
1939
|
+
return value.lower() # the first `(` closes before the end
|
|
1940
|
+
value = value[1:-1].strip()
|
|
1941
|
+
return value.lower()
|
|
1942
|
+
|
|
1943
|
+
|
|
1944
|
+
def _decoded_relevance(inner):
|
|
1945
|
+
"""A Relevance element's text as the evaluator sees it (CDATA or not)."""
|
|
1946
|
+
cdata = CDATA_RE.match(inner.strip())
|
|
1947
|
+
return cdata.group(1) if cdata else _xml_unescape(inner)
|
|
1948
|
+
|
|
1949
|
+
|
|
1950
|
+
def _top_level_operands(value, keyword):
|
|
1951
|
+
"""Split relevance on a top-level `and`/`or` (outside parens and strings).
|
|
1952
|
+
|
|
1953
|
+
Returns the operand texts, or a single-item list when `keyword` does not
|
|
1954
|
+
occur at the top level. Comments are removed and wrapping parentheses
|
|
1955
|
+
dropped first (see _relevance_literal).
|
|
1956
|
+
"""
|
|
1957
|
+
text = RELEVANCE_COMMENT_RE.sub(" ", value).strip()
|
|
1958
|
+
while True: # drop wrapping parens, as _relevance_literal does
|
|
1959
|
+
unwrapped = _relevance_literal(text)
|
|
1960
|
+
if unwrapped == text.lower():
|
|
1961
|
+
break
|
|
1962
|
+
text = text.strip()[1:-1].strip()
|
|
1963
|
+
operands, depth, quoted, start = [], 0, False, 0
|
|
1964
|
+
pattern = re.compile(r"\s" + keyword + r"\s", re.IGNORECASE)
|
|
1965
|
+
index = 0
|
|
1966
|
+
while index < len(text):
|
|
1967
|
+
char = text[index]
|
|
1968
|
+
if char == '"':
|
|
1969
|
+
quoted = not quoted
|
|
1970
|
+
elif not quoted and char == "(":
|
|
1971
|
+
depth += 1
|
|
1972
|
+
elif not quoted and char == ")":
|
|
1973
|
+
depth -= 1
|
|
1974
|
+
elif not quoted and not depth:
|
|
1975
|
+
match = pattern.match(text, index)
|
|
1976
|
+
if match:
|
|
1977
|
+
operands.append(text[start:index])
|
|
1978
|
+
index = start = match.end()
|
|
1979
|
+
continue
|
|
1980
|
+
index += 1
|
|
1981
|
+
operands.append(text[start:])
|
|
1982
|
+
return operands
|
|
1983
|
+
|
|
1984
|
+
|
|
1985
|
+
def _short_circuit(value):
|
|
1986
|
+
"""Return "true"/"false" if a top-level literal operand decides `value`.
|
|
1987
|
+
|
|
1988
|
+
`true OR x` is always true and `false AND x` always false, whatever x is
|
|
1989
|
+
-- typically a test hack or a leftover that silently disables the real
|
|
1990
|
+
condition. `true AND x` / `x OR false` are just x and not reported.
|
|
1991
|
+
"""
|
|
1992
|
+
operands = _top_level_operands(value, "or")
|
|
1993
|
+
if len(operands) > 1:
|
|
1994
|
+
return (
|
|
1995
|
+
"true" if any(_relevance_literal(op) == "true" for op in operands) else None
|
|
1996
|
+
)
|
|
1997
|
+
operands = _top_level_operands(value, "and")
|
|
1998
|
+
if len(operands) > 1 and any(_relevance_literal(op) == "false" for op in operands):
|
|
1999
|
+
return "false"
|
|
2000
|
+
return None
|
|
2001
|
+
|
|
2002
|
+
|
|
2003
|
+
def check_relevance(src):
|
|
2004
|
+
"""E212/E213/W212/W213: Relevance empty/`true`/`false`, or stray whitespace."""
|
|
2005
|
+
issues = []
|
|
2006
|
+
for match in RELEVANCE_TAG_RE.finditer(src):
|
|
2007
|
+
inner = match.group(1)
|
|
2008
|
+
value = _strip_cdata(inner).strip()
|
|
2009
|
+
lineno = _lineno(src, match.start())
|
|
2010
|
+
if value == "":
|
|
2011
|
+
issues.append(
|
|
2012
|
+
(
|
|
2013
|
+
lineno,
|
|
2014
|
+
"E213",
|
|
2015
|
+
(
|
|
2016
|
+
"Relevance is empty; give it a real relevance clause; add "
|
|
2017
|
+
f"`{RELEVANCE_MARKER}` if intentional"
|
|
2018
|
+
),
|
|
2019
|
+
)
|
|
2020
|
+
)
|
|
2021
|
+
elif _relevance_literal(value) == "true":
|
|
2022
|
+
issues.append(
|
|
2023
|
+
(
|
|
2024
|
+
lineno,
|
|
2025
|
+
"E212",
|
|
2026
|
+
(
|
|
2027
|
+
"Relevance is the literal `true`; it targets every endpoint; "
|
|
2028
|
+
f"add `{RELEVANCE_MARKER}` if intentional"
|
|
2029
|
+
),
|
|
2030
|
+
)
|
|
2031
|
+
)
|
|
2032
|
+
elif _short_circuit(_decoded_relevance(inner)) == "true":
|
|
2033
|
+
issues.append(
|
|
2034
|
+
(
|
|
2035
|
+
lineno,
|
|
2036
|
+
"E212",
|
|
2037
|
+
(
|
|
2038
|
+
"Relevance is always true: a top-level `true OR ...` "
|
|
2039
|
+
"operand decides it whatever the rest says, so it targets "
|
|
2040
|
+
f"every endpoint; add `{RELEVANCE_MARKER}` if intentional"
|
|
2041
|
+
),
|
|
2042
|
+
)
|
|
2043
|
+
)
|
|
2044
|
+
elif _short_circuit(_decoded_relevance(inner)) == "false":
|
|
2045
|
+
issues.append(
|
|
2046
|
+
(
|
|
2047
|
+
lineno,
|
|
2048
|
+
"W212",
|
|
2049
|
+
(
|
|
2050
|
+
"Relevance is always false: a top-level `false AND ...` "
|
|
2051
|
+
"operand decides it, so it never applies; add "
|
|
2052
|
+
f"`{RELEVANCE_MARKER}` if intentional"
|
|
2053
|
+
),
|
|
2054
|
+
)
|
|
2055
|
+
)
|
|
2056
|
+
elif _relevance_literal(value) == "false":
|
|
2057
|
+
issues.append(
|
|
2058
|
+
(
|
|
2059
|
+
lineno,
|
|
2060
|
+
"W212",
|
|
2061
|
+
(
|
|
2062
|
+
"Relevance is the literal `false`; it never applies to any "
|
|
2063
|
+
f"endpoint; add `{RELEVANCE_MARKER}` if intentional"
|
|
2064
|
+
),
|
|
2065
|
+
)
|
|
2066
|
+
)
|
|
2067
|
+
if "<![CDATA[" not in inner and inner != inner.strip():
|
|
2068
|
+
issues.append(
|
|
2069
|
+
(
|
|
2070
|
+
lineno,
|
|
2071
|
+
"W213",
|
|
2072
|
+
(
|
|
2073
|
+
"Relevance has leading/trailing whitespace; add "
|
|
2074
|
+
f"`{RELEVANCE_MARKER}` if intentional"
|
|
2075
|
+
),
|
|
2076
|
+
)
|
|
2077
|
+
)
|
|
2078
|
+
return issues
|
|
2079
|
+
|
|
2080
|
+
|
|
2081
|
+
def _content_object_blocks(root, src):
|
|
2082
|
+
"""Return ordered [(start, end, element)] for the content objects in `src`.
|
|
2083
|
+
|
|
2084
|
+
The regex spans (in document order) are zipped with the ElementTree
|
|
2085
|
+
content-tag children (same order) so each block carries its parsed element.
|
|
2086
|
+
Each block is an independent entity: it is checked and fixed on its own, and
|
|
2087
|
+
a marker only governs the block it sits in (see `_outside_text`).
|
|
2088
|
+
"""
|
|
2089
|
+
spans = [(m.start(), m.end()) for m in CONTENT_OBJECT_SPAN_RE.finditer(src)]
|
|
2090
|
+
children = [
|
|
2091
|
+
child
|
|
2092
|
+
for child in list(root)
|
|
2093
|
+
if isinstance(child.tag, str) and child.tag in CONTENT_TAGS
|
|
2094
|
+
]
|
|
2095
|
+
return [
|
|
2096
|
+
(start, end, children[i] if i < len(children) else None)
|
|
2097
|
+
for i, (start, end) in enumerate(spans)
|
|
2098
|
+
]
|
|
2099
|
+
|
|
2100
|
+
|
|
2101
|
+
def _outside_text(src, blocks):
|
|
2102
|
+
"""Return `src` with every content-object block removed.
|
|
2103
|
+
|
|
2104
|
+
A marker appearing here is outside all objects, so it is file-level and
|
|
2105
|
+
governs every object; a marker inside a block governs only that block.
|
|
2106
|
+
"""
|
|
2107
|
+
parts = []
|
|
2108
|
+
cursor = 0
|
|
2109
|
+
for start, end, _ in blocks:
|
|
2110
|
+
parts.append(src[cursor:start])
|
|
2111
|
+
cursor = end
|
|
2112
|
+
parts.append(src[cursor:])
|
|
2113
|
+
return "".join(parts)
|
|
2114
|
+
|
|
2115
|
+
|
|
2116
|
+
def _has_modification_time(element):
|
|
2117
|
+
"""True if `element` has a MIMEField named x-fixlet-modification-time."""
|
|
2118
|
+
for mimefield in element.findall("MIMEField"):
|
|
2119
|
+
name = mimefield.find("Name")
|
|
2120
|
+
if name is not None and (name.text or "").strip() == MODIFICATION_TIME_NAME:
|
|
2121
|
+
return True
|
|
2122
|
+
return False
|
|
2123
|
+
|
|
2124
|
+
|
|
2125
|
+
def _action_scripts_text(element):
|
|
2126
|
+
"""Return the concatenated text of every ActionScript under `element`."""
|
|
2127
|
+
return "\n".join((a.text or "") for a in element.iter("ActionScript"))
|
|
2128
|
+
|
|
2129
|
+
|
|
2130
|
+
def _check_element(tag, element, disabled):
|
|
2131
|
+
"""E204 for any content object plus W201/W202/W203 for a Task/Fixlet.
|
|
2132
|
+
|
|
2133
|
+
Issues are reported at local line 1 -- the block's open tag; the caller
|
|
2134
|
+
offsets it to the file line.
|
|
2135
|
+
"""
|
|
2136
|
+
if element is None or tag not in CONTENT_TAGS:
|
|
2137
|
+
return []
|
|
2138
|
+
issues = []
|
|
2139
|
+
title = element.find("Title")
|
|
2140
|
+
label = (title.text or "").strip() if title is not None else ""
|
|
2141
|
+
where = f' ("{label}")' if label else ""
|
|
2142
|
+
|
|
2143
|
+
if "E204" not in disabled:
|
|
2144
|
+
description = element.find("Description")
|
|
2145
|
+
text = "".join(description.itertext()) if description is not None else ""
|
|
2146
|
+
if DESCRIPTION_PLACEHOLDER in text.lower():
|
|
2147
|
+
issues.append(
|
|
2148
|
+
(
|
|
2149
|
+
1,
|
|
2150
|
+
"E204",
|
|
2151
|
+
(
|
|
2152
|
+
f"{tag}{where} Description contains the placeholder "
|
|
2153
|
+
f'"{DESCRIPTION_PLACEHOLDER}"; add `{DESCRIPTION_MARKER}` '
|
|
2154
|
+
"if intentional"
|
|
2155
|
+
),
|
|
2156
|
+
)
|
|
2157
|
+
)
|
|
2158
|
+
if tag not in DATED_CONTENT_TAGS:
|
|
2159
|
+
return issues
|
|
2160
|
+
|
|
2161
|
+
if "W215" not in disabled:
|
|
2162
|
+
description = element.find("Description")
|
|
2163
|
+
text = (
|
|
2164
|
+
"".join(description.itertext()).strip() if description is not None else ""
|
|
2165
|
+
)
|
|
2166
|
+
if text == "":
|
|
2167
|
+
issues.append(
|
|
2168
|
+
(
|
|
2169
|
+
1,
|
|
2170
|
+
"W215",
|
|
2171
|
+
(
|
|
2172
|
+
f"{tag}{where} has an empty or missing Description; add "
|
|
2173
|
+
f"`{DESCRIPTION_MARKER}` if intentional"
|
|
2174
|
+
),
|
|
2175
|
+
)
|
|
2176
|
+
)
|
|
2177
|
+
if "W201" not in disabled and not _has_modification_time(element):
|
|
2178
|
+
issues.append(
|
|
2179
|
+
(
|
|
2180
|
+
1,
|
|
2181
|
+
"W201",
|
|
2182
|
+
(
|
|
2183
|
+
f"{tag}{where} has no x-fixlet-modification-time MIMEField; add "
|
|
2184
|
+
f"`{MODIFICATION_TIME_MARKER}` if intentional"
|
|
2185
|
+
),
|
|
2186
|
+
)
|
|
2187
|
+
)
|
|
2188
|
+
if "W202" not in disabled and element.find("SourceReleaseDate") is None:
|
|
2189
|
+
issues.append(
|
|
2190
|
+
(
|
|
2191
|
+
1,
|
|
2192
|
+
"W202",
|
|
2193
|
+
(
|
|
2194
|
+
f"{tag}{where} has no SourceReleaseDate; add "
|
|
2195
|
+
f"`{SOURCE_RELEASE_DATE_MARKER}` if intentional"
|
|
2196
|
+
),
|
|
2197
|
+
)
|
|
2198
|
+
)
|
|
2199
|
+
if "W203" not in disabled:
|
|
2200
|
+
download = element.find("DownloadSize")
|
|
2201
|
+
raw = _strip_cdata(download.text or "") if download is not None else ""
|
|
2202
|
+
size = int(raw) if DOWNLOAD_SIZE_RE.match(raw) else 0
|
|
2203
|
+
if size > 0 and not DOWNLOAD_KEYWORD_RE.search(_action_scripts_text(element)):
|
|
2204
|
+
issues.append(
|
|
2205
|
+
(
|
|
2206
|
+
1,
|
|
2207
|
+
"W203",
|
|
2208
|
+
(
|
|
2209
|
+
f"{tag}{where} has DownloadSize > 0 but no download/prefetch "
|
|
2210
|
+
f"keyword in any ActionScript; add `{DOWNLOAD_SIZE_MARKER}` "
|
|
2211
|
+
"if intentional"
|
|
2212
|
+
),
|
|
2213
|
+
)
|
|
2214
|
+
)
|
|
2215
|
+
return issues
|
|
2216
|
+
|
|
2217
|
+
|
|
2218
|
+
# (codes, opt-out marker, check function) for the checks that scan element text;
|
|
2219
|
+
# each function scans a single content-object block and returns local linenos.
|
|
2220
|
+
# `codes` is the tuple of check IDs the function can emit: the check is skipped
|
|
2221
|
+
# only when its marker is present or ALL of its codes are disabled, and an
|
|
2222
|
+
# individually-disabled code is dropped from the results afterward.
|
|
2223
|
+
VALUE_CHECKS = (
|
|
2224
|
+
(("E200",), MIMETYPE_MARKER, check_action_mimetypes),
|
|
2225
|
+
(("E201",), SOURCE_RELEASE_DATE_MARKER, check_source_release_date_format),
|
|
2226
|
+
(("E202",), MODIFICATION_TIME_MARKER, check_modification_time_format),
|
|
2227
|
+
(("W219",), SOURCE_RELEASE_DATE_MARKER, check_release_after_modification),
|
|
2228
|
+
(("E216",), FIRST_PROPAGATION_MARKER, check_first_propagation_format),
|
|
2229
|
+
(("E203",), DOWNLOAD_SIZE_MARKER, check_download_size_value),
|
|
2230
|
+
(("W220",), DOWNLOAD_SIZE_MARKER, check_download_size_total),
|
|
2231
|
+
(("W221",), SETTING_VALUE_MARKER, check_setting_value_target),
|
|
2232
|
+
(("E205",), CPE_MARKER, check_cpe23),
|
|
2233
|
+
(("E206",), ACTION_UI_METADATA_MARKER, check_action_ui_metadata),
|
|
2234
|
+
(("E207",), CDATA_MARKER, check_cdata_required),
|
|
2235
|
+
(("E209",), CVE_NAMES_MARKER, check_cve_names),
|
|
2236
|
+
(("E210",), MIMEFIELD_DUP_MARKER, check_duplicate_mimefield_names),
|
|
2237
|
+
(("E220",), ANALYSIS_PROPERTY_MARKER, check_duplicate_analysis_properties),
|
|
2238
|
+
(("E211", "W209", "W214"), TITLE_MARKER, check_title),
|
|
2239
|
+
(("E212", "E213", "W212", "W213"), RELEVANCE_MARKER, check_relevance),
|
|
2240
|
+
(("E215",), CDATA_CLOSE_MARKER, check_cdata_close),
|
|
2241
|
+
(("E217",), SUCCESS_CRITERIA_MARKER, check_success_criteria),
|
|
2242
|
+
(("E218",), ACTION_ID_MARKER, check_duplicate_action_ids),
|
|
2243
|
+
(("E219",), EVALUATION_PERIOD_MARKER, check_evaluation_period),
|
|
2244
|
+
(("W216",), SEVERITY_MARKER, check_source_severity),
|
|
2245
|
+
(("W204",), CDATA_MARKER, check_actionscript_cdata),
|
|
2246
|
+
(("W205",), ACTION_BLANK_LINES_MARKER, check_actionscript_blank_lines),
|
|
2247
|
+
(("W206",), PREFETCH_MARKER, check_prefetch_lines),
|
|
2248
|
+
(("W207",), PREFETCH_HTTPS_MARKER, check_prefetch_https),
|
|
2249
|
+
(("W208",), ACTIONSCRIPT_EMPTY_MARKER, check_empty_actionscript),
|
|
2250
|
+
(("W211",), DOWNLOAD_MARKER, check_dynamic_download),
|
|
2251
|
+
(("W218",), LINK_TEXT_MARKER, check_link_text),
|
|
2252
|
+
(("E221",), SCRIPT_COMMENT_MARKER, check_script_comments),
|
|
2253
|
+
)
|
|
2254
|
+
|
|
2255
|
+
# (presence code, opt-out marker) -- markers scoped like the value checks
|
|
2256
|
+
PRESENCE_MARKERS = (
|
|
2257
|
+
("W201", MODIFICATION_TIME_MARKER),
|
|
2258
|
+
("W202", SOURCE_RELEASE_DATE_MARKER),
|
|
2259
|
+
("W203", DOWNLOAD_SIZE_MARKER),
|
|
2260
|
+
("E204", DESCRIPTION_MARKER),
|
|
2261
|
+
("W215", DESCRIPTION_MARKER),
|
|
2262
|
+
)
|
|
2263
|
+
|
|
2264
|
+
|
|
2265
|
+
def _value_checks(severities=None, prefetch_url_schemes=None):
|
|
2266
|
+
"""Return VALUE_CHECKS with the configurable checks bound to their options.
|
|
2267
|
+
|
|
2268
|
+
`severities` (check_source_severity, see --severity-values) and
|
|
2269
|
+
`prefetch_url_schemes` (check_prefetch_lines, see --prefetch-url-schemes)
|
|
2270
|
+
are None for the built-in default; every other entry is passed through
|
|
2271
|
+
unchanged.
|
|
2272
|
+
"""
|
|
2273
|
+
bound = {}
|
|
2274
|
+
if severities is not None:
|
|
2275
|
+
bound[check_source_severity] = {"allowed": severities}
|
|
2276
|
+
if prefetch_url_schemes is not None:
|
|
2277
|
+
bound[check_prefetch_lines] = {"schemes": prefetch_url_schemes}
|
|
2278
|
+
if not bound:
|
|
2279
|
+
return VALUE_CHECKS
|
|
2280
|
+
return tuple(
|
|
2281
|
+
(
|
|
2282
|
+
(codes, marker, functools.partial(check, **bound[check]))
|
|
2283
|
+
if check in bound
|
|
2284
|
+
else (codes, marker, check)
|
|
2285
|
+
)
|
|
2286
|
+
for codes, marker, check in VALUE_CHECKS
|
|
2287
|
+
)
|
|
2288
|
+
|
|
2289
|
+
|
|
2290
|
+
def _run_checks(src, root, disabled, severities=None, prefetch_url_schemes=None):
|
|
2291
|
+
"""Run every check on each content object independently; return sorted issues.
|
|
2292
|
+
|
|
2293
|
+
Each block's checks see only that block's text, and a marker governs a block
|
|
2294
|
+
only if it sits inside it or outside all objects (file-level). Local line
|
|
2295
|
+
numbers from the per-block scans are offset back to the file's line numbers.
|
|
2296
|
+
`severities` overrides the W216 vocabulary (see --severity-values), and
|
|
2297
|
+
`prefetch_url_schemes` the W206 URL schemes (see --prefetch-url-schemes).
|
|
2298
|
+
"""
|
|
2299
|
+
blocks = _content_object_blocks(root, src)
|
|
2300
|
+
outside = _outside_text(src, blocks)
|
|
2301
|
+
value_checks = _value_checks(severities, prefetch_url_schemes)
|
|
2302
|
+
issues = []
|
|
2303
|
+
for start, _end, element in blocks:
|
|
2304
|
+
block = src[start:_end]
|
|
2305
|
+
start_line = _lineno(src, start)
|
|
2306
|
+
marker_text = block + outside
|
|
2307
|
+
for codes, marker, check in value_checks:
|
|
2308
|
+
if marker in marker_text or all(code in disabled for code in codes):
|
|
2309
|
+
continue
|
|
2310
|
+
for lineno, found_code, message in check(block):
|
|
2311
|
+
if found_code in disabled:
|
|
2312
|
+
continue
|
|
2313
|
+
issues.append((start_line + lineno - 1, found_code, message))
|
|
2314
|
+
presence_disabled = set(disabled)
|
|
2315
|
+
for code, marker in PRESENCE_MARKERS:
|
|
2316
|
+
if marker in marker_text:
|
|
2317
|
+
presence_disabled.add(code)
|
|
2318
|
+
tag = element.tag if element is not None else None
|
|
2319
|
+
for lineno, found_code, message in _check_element(
|
|
2320
|
+
tag, element, presence_disabled
|
|
2321
|
+
):
|
|
2322
|
+
issues.append((start_line + lineno - 1, found_code, message))
|
|
2323
|
+
return sorted(issues)
|
|
2324
|
+
|
|
2325
|
+
|
|
2326
|
+
def _fix_block(block, marker_text, disabled, strict, now, severities=None):
|
|
2327
|
+
"""Apply the auto-fixers to one content-object block; return (new, fixed).
|
|
2328
|
+
|
|
2329
|
+
Marker gating is the same per-block scoping used by the checks.
|
|
2330
|
+
`severities` overrides the W216 vocabulary (see --severity-values).
|
|
2331
|
+
"""
|
|
2332
|
+
fixed = []
|
|
2333
|
+
if "W216" not in disabled and SEVERITY_MARKER not in marker_text:
|
|
2334
|
+
block, got = fix_source_severity(
|
|
2335
|
+
block, CANONICAL_SEVERITIES if severities is None else severities
|
|
2336
|
+
)
|
|
2337
|
+
fixed += got
|
|
2338
|
+
if "E203" not in disabled and DOWNLOAD_SIZE_MARKER not in marker_text:
|
|
2339
|
+
block, got = fix_download_size(block)
|
|
2340
|
+
fixed += got
|
|
2341
|
+
# after E203, so an invalid DownloadSize reset to 0 still gets the real total
|
|
2342
|
+
if "W220" not in disabled and DOWNLOAD_SIZE_MARKER not in marker_text:
|
|
2343
|
+
block, got = fix_download_size_total(block)
|
|
2344
|
+
fixed += got
|
|
2345
|
+
if "E200" not in disabled and MIMETYPE_MARKER not in marker_text:
|
|
2346
|
+
block, got = fix_action_mimetypes(block)
|
|
2347
|
+
fixed += got
|
|
2348
|
+
if "E201" not in disabled and SOURCE_RELEASE_DATE_MARKER not in marker_text:
|
|
2349
|
+
block, got = fix_source_release_date_format(block)
|
|
2350
|
+
fixed += got
|
|
2351
|
+
# before fix_missing_dates, which dates an inserted SourceReleaseDate from
|
|
2352
|
+
# a valid modification time
|
|
2353
|
+
if "E202" not in disabled and MODIFICATION_TIME_MARKER not in marker_text:
|
|
2354
|
+
block, got = fix_timestamp_format(
|
|
2355
|
+
block, MODTIME_VALUE_RE, "E202", MODIFICATION_TIME_NAME
|
|
2356
|
+
)
|
|
2357
|
+
fixed += got
|
|
2358
|
+
if "E216" not in disabled and FIRST_PROPAGATION_MARKER not in marker_text:
|
|
2359
|
+
block, got = fix_timestamp_format(
|
|
2360
|
+
block, FIRST_PROP_VALUE_RE, "E216", FIRST_PROPAGATION_NAME
|
|
2361
|
+
)
|
|
2362
|
+
fixed += got
|
|
2363
|
+
if "E209" not in disabled and CVE_NAMES_MARKER not in marker_text:
|
|
2364
|
+
block, got = fix_cve_names(block)
|
|
2365
|
+
fixed += got
|
|
2366
|
+
if "E219" not in disabled and EVALUATION_PERIOD_MARKER not in marker_text:
|
|
2367
|
+
block, got = fix_evaluation_period(block)
|
|
2368
|
+
fixed += got
|
|
2369
|
+
if "W209" not in disabled and TITLE_MARKER not in marker_text:
|
|
2370
|
+
block, got = fix_title(block)
|
|
2371
|
+
fixed += got
|
|
2372
|
+
if "W213" not in disabled and RELEVANCE_MARKER not in marker_text:
|
|
2373
|
+
block, got = fix_relevance_whitespace(block)
|
|
2374
|
+
fixed += got
|
|
2375
|
+
if "W205" not in disabled and ACTION_BLANK_LINES_MARKER not in marker_text:
|
|
2376
|
+
block, got = fix_blank_lines(block)
|
|
2377
|
+
fixed += got
|
|
2378
|
+
if "E207" not in disabled and CDATA_MARKER not in marker_text:
|
|
2379
|
+
block, got = fix_cdata_required(block)
|
|
2380
|
+
fixed += got
|
|
2381
|
+
# runs after E207 so an entity-escaped Description is unescaped and
|
|
2382
|
+
# CDATA-wrapped first, revealing its literal <script> body to rewrite
|
|
2383
|
+
if "E221" not in disabled and SCRIPT_COMMENT_MARKER not in marker_text:
|
|
2384
|
+
block, got = fix_script_comments(block)
|
|
2385
|
+
fixed += got
|
|
2386
|
+
# runs after the W205 collapse, which preserves the terminator's indentation
|
|
2387
|
+
if "E215" not in disabled and CDATA_CLOSE_MARKER not in marker_text:
|
|
2388
|
+
block, got = fix_cdata_close(block)
|
|
2389
|
+
fixed += got
|
|
2390
|
+
if strict and "W204" not in disabled and CDATA_MARKER not in marker_text:
|
|
2391
|
+
block, got = fix_actionscript_cdata(block)
|
|
2392
|
+
fixed += got
|
|
2393
|
+
fix_srd = "W202" not in disabled and SOURCE_RELEASE_DATE_MARKER not in marker_text
|
|
2394
|
+
fix_modtime = "W201" not in disabled and MODIFICATION_TIME_MARKER not in marker_text
|
|
2395
|
+
if fix_srd or fix_modtime:
|
|
2396
|
+
block, got = fix_missing_dates(
|
|
2397
|
+
block, now, fix_srd=fix_srd, fix_modtime=fix_modtime
|
|
2398
|
+
)
|
|
2399
|
+
fixed += got
|
|
2400
|
+
return block, fixed
|
|
2401
|
+
|
|
2402
|
+
|
|
2403
|
+
def _autofix(src, root, disabled, strict, now, severities=None):
|
|
2404
|
+
"""Rewrite each content object independently; return (new_src, fixed).
|
|
2405
|
+
|
|
2406
|
+
Text outside the content objects is left untouched. Fixed line numbers are
|
|
2407
|
+
offset back to the file's line numbers.
|
|
2408
|
+
"""
|
|
2409
|
+
blocks = _content_object_blocks(root, src)
|
|
2410
|
+
outside = _outside_text(src, blocks)
|
|
2411
|
+
result = []
|
|
2412
|
+
fixed = []
|
|
2413
|
+
cursor = 0
|
|
2414
|
+
for start, end, _element in blocks:
|
|
2415
|
+
result.append(src[cursor:start])
|
|
2416
|
+
block = src[start:end]
|
|
2417
|
+
start_line = _lineno(src, start)
|
|
2418
|
+
new_block, block_fixed = _fix_block(
|
|
2419
|
+
block, block + outside, disabled, strict, now, severities
|
|
2420
|
+
)
|
|
2421
|
+
fixed += [
|
|
2422
|
+
(start_line + lineno - 1, code, message)
|
|
2423
|
+
for lineno, code, message in block_fixed
|
|
2424
|
+
]
|
|
2425
|
+
result.append(new_block)
|
|
2426
|
+
cursor = end
|
|
2427
|
+
result.append(src[cursor:])
|
|
2428
|
+
return "".join(result), fixed
|
|
2429
|
+
|
|
2430
|
+
|
|
2431
|
+
# --------------------------------------------------------------------------
|
|
2432
|
+
# auto-fixers (each mutates `src` text and returns (new_src, fixed_list))
|
|
2433
|
+
# --------------------------------------------------------------------------
|
|
2434
|
+
|
|
2435
|
+
|
|
2436
|
+
def fix_download_size(src):
|
|
2437
|
+
"""E203: rewrite an empty/invalid <DownloadSize> to 0."""
|
|
2438
|
+
fixed = []
|
|
2439
|
+
|
|
2440
|
+
def repl(match):
|
|
2441
|
+
value = _strip_cdata(match.group(1))
|
|
2442
|
+
if value == "" or not DOWNLOAD_SIZE_RE.match(value):
|
|
2443
|
+
fixed.append(
|
|
2444
|
+
(
|
|
2445
|
+
_lineno(src, match.start()),
|
|
2446
|
+
"E203",
|
|
2447
|
+
f'DownloadSize "{value}" set to 0',
|
|
2448
|
+
)
|
|
2449
|
+
)
|
|
2450
|
+
return "<DownloadSize>0</DownloadSize>"
|
|
2451
|
+
return match.group(0)
|
|
2452
|
+
|
|
2453
|
+
return DOWNLOAD_SIZE_TAG_RE.sub(repl, src), fixed
|
|
2454
|
+
|
|
2455
|
+
|
|
2456
|
+
def fix_source_severity(src, allowed=CANONICAL_SEVERITIES):
|
|
2457
|
+
"""W216: rewrite an `Unspecified` <SourceSeverity> to an empty one, and a
|
|
2458
|
+
case-only mismatch (`HIGH`, `critical`) to its allowed spelling.
|
|
2459
|
+
|
|
2460
|
+
Only the `Unspecified` spellings check_source_severity reports as
|
|
2461
|
+
auto-fixable are cleared; a value listed in `allowed` (a repo that opts
|
|
2462
|
+
`Unspecified` back in via --severity-values) is kept. A value that matches
|
|
2463
|
+
exactly one allowed spelling ignoring case is rewritten to it; any other
|
|
2464
|
+
out-of-vocabulary value (`Medium`, `N/A`) is left for a human.
|
|
2465
|
+
"""
|
|
2466
|
+
fixed = []
|
|
2467
|
+
by_lower = {}
|
|
2468
|
+
for item in allowed:
|
|
2469
|
+
by_lower.setdefault(item.lower(), []).append(item)
|
|
2470
|
+
|
|
2471
|
+
def repl(match):
|
|
2472
|
+
value = _strip_cdata(match.group(1)).strip()
|
|
2473
|
+
if value in allowed:
|
|
2474
|
+
return match.group(0)
|
|
2475
|
+
if not _is_unspecified_severity(match.group(1)):
|
|
2476
|
+
# a case-only mismatch (`HIGH`) has exactly one intended spelling
|
|
2477
|
+
spellings = by_lower.get(value.lower(), [])
|
|
2478
|
+
if "<![CDATA[" in match.group(1) or len(spellings) != 1:
|
|
2479
|
+
return match.group(0)
|
|
2480
|
+
fixed.append(
|
|
2481
|
+
(
|
|
2482
|
+
_lineno(src, match.start()),
|
|
2483
|
+
"W216",
|
|
2484
|
+
f'SourceSeverity "{value}" set to "{spellings[0]}"',
|
|
2485
|
+
)
|
|
2486
|
+
)
|
|
2487
|
+
return f"<SourceSeverity>{spellings[0]}</SourceSeverity>"
|
|
2488
|
+
fixed.append(
|
|
2489
|
+
(
|
|
2490
|
+
_lineno(src, match.start()),
|
|
2491
|
+
"W216",
|
|
2492
|
+
f'cleared SourceSeverity "{value}" to empty',
|
|
2493
|
+
)
|
|
2494
|
+
)
|
|
2495
|
+
return "<SourceSeverity></SourceSeverity>"
|
|
2496
|
+
|
|
2497
|
+
return SOURCE_SEVERITY_RE.sub(repl, src), fixed
|
|
2498
|
+
|
|
2499
|
+
|
|
2500
|
+
# E200: the MIMEType the ActionScript hooks already assume when none is given
|
|
2501
|
+
DEFAULT_MIMETYPE = "application/x-Fixlet-Windows-Shell"
|
|
2502
|
+
_MIMETYPES_BY_LOWER = {item.lower(): item for item in ALLOWED_MIMETYPES}
|
|
2503
|
+
CDATA_SECTION_RE = re.compile(r"<!\[CDATA\[.*?\]\]>", re.DOTALL)
|
|
2504
|
+
|
|
2505
|
+
|
|
2506
|
+
def fix_action_mimetypes(src):
|
|
2507
|
+
"""E200: add a missing MIMEType, or fix the case of an allowed one.
|
|
2508
|
+
|
|
2509
|
+
A missing MIMEType becomes DEFAULT_MIMETYPE, which is what every
|
|
2510
|
+
ActionScript hook in this repo already treats a missing one as. A value
|
|
2511
|
+
that differs from an allowed MIMEType only in case is rewritten to the
|
|
2512
|
+
allowed spelling. Any other value is left for a human.
|
|
2513
|
+
"""
|
|
2514
|
+
fixed = []
|
|
2515
|
+
# an `<ActionScript ...>` inside a CDATA section is text (e.g. relevance
|
|
2516
|
+
# that generates BES XML), not an element, and must never be edited
|
|
2517
|
+
cdata_spans = [m.span() for m in CDATA_SECTION_RE.finditer(src)]
|
|
2518
|
+
|
|
2519
|
+
def repl(match):
|
|
2520
|
+
if any(start < match.start() < end for start, end in cdata_spans):
|
|
2521
|
+
return match.group(0)
|
|
2522
|
+
attrs = match.group(1)
|
|
2523
|
+
mime_match = MIMETYPE_ATTR_RE.search(attrs)
|
|
2524
|
+
if mime_match is None:
|
|
2525
|
+
new_attrs = f' MIMEType="{DEFAULT_MIMETYPE}"' + attrs
|
|
2526
|
+
message = f"added missing ActionScript MIMEType {DEFAULT_MIMETYPE}"
|
|
2527
|
+
else:
|
|
2528
|
+
value = mime_match.group(1)
|
|
2529
|
+
canonical = _MIMETYPES_BY_LOWER.get(value.lower())
|
|
2530
|
+
if canonical is None or canonical == value:
|
|
2531
|
+
return match.group(0)
|
|
2532
|
+
new_attrs = (
|
|
2533
|
+
attrs[: mime_match.start(1)] + canonical + attrs[mime_match.end(1) :]
|
|
2534
|
+
)
|
|
2535
|
+
message = f'ActionScript MIMEType "{value}" set to "{canonical}"'
|
|
2536
|
+
fixed.append((_lineno(src, match.start()), "E200", message))
|
|
2537
|
+
return f"<ActionScript{new_attrs}>"
|
|
2538
|
+
|
|
2539
|
+
return ACTIONSCRIPT_OPEN_RE.sub(repl, src), fixed
|
|
2540
|
+
|
|
2541
|
+
|
|
2542
|
+
# E201: a year-first date whose parts are unambiguous, with `-`, `/`, or `.`
|
|
2543
|
+
# separators, unpadded parts, and an optional time part to drop. A
|
|
2544
|
+
# month/day-first date (07/14/2026 or 14/07/2026) is never guessed at.
|
|
2545
|
+
LOOSE_SRD_RE = re.compile(
|
|
2546
|
+
r"^(\d{4})[-/.](\d{1,2})[-/.](\d{1,2})(?:[T ]\d{1,2}:\d{2}(?::\d{2})?\S*)?$"
|
|
2547
|
+
)
|
|
2548
|
+
|
|
2549
|
+
|
|
2550
|
+
def fix_source_release_date_format(src):
|
|
2551
|
+
"""E201: reformat an unambiguous year-first SourceReleaseDate as YYYY-MM-DD."""
|
|
2552
|
+
fixed = []
|
|
2553
|
+
|
|
2554
|
+
def repl(match):
|
|
2555
|
+
inner = match.group(1)
|
|
2556
|
+
if "<![CDATA[" in inner:
|
|
2557
|
+
return match.group(0)
|
|
2558
|
+
value = inner.strip()
|
|
2559
|
+
loose = LOOSE_SRD_RE.match(value)
|
|
2560
|
+
if not value or _valid_source_release_date(value) or not loose:
|
|
2561
|
+
return match.group(0)
|
|
2562
|
+
year, month, day = (int(part) for part in loose.groups())
|
|
2563
|
+
try:
|
|
2564
|
+
new_value = date(year, month, day).isoformat()
|
|
2565
|
+
except ValueError:
|
|
2566
|
+
return match.group(0)
|
|
2567
|
+
fixed.append(
|
|
2568
|
+
(
|
|
2569
|
+
_lineno(src, match.start()),
|
|
2570
|
+
"E201",
|
|
2571
|
+
f'SourceReleaseDate "{value}" set to {new_value}',
|
|
2572
|
+
)
|
|
2573
|
+
)
|
|
2574
|
+
return f"<SourceReleaseDate>{new_value}</SourceReleaseDate>"
|
|
2575
|
+
|
|
2576
|
+
return SRD_RE.sub(repl, src), fixed
|
|
2577
|
+
|
|
2578
|
+
|
|
2579
|
+
# E202/E216: an RFC 5322-ish date-time with fixable formatting -- any case,
|
|
2580
|
+
# full or abbreviated day/month names, unpadded day/hour, a UTC zone name, or
|
|
2581
|
+
# a `+HH:MM` offset. Only these spellings are rewritten; anything else (a
|
|
2582
|
+
# missing field, EST, ISO 8601) is left for a human.
|
|
2583
|
+
LOOSE_TIMESTAMP_RE = re.compile(
|
|
2584
|
+
r"^(?:(?P<dow>[A-Za-z]{3,9}),?\s+)?"
|
|
2585
|
+
r"(?P<day>\d{1,2})\s+(?P<mon>[A-Za-z]{3,9})\s+(?P<year>\d{4})\s+"
|
|
2586
|
+
r"(?P<hour>\d{1,2}):(?P<minute>\d{2}):(?P<second>\d{2})\s*"
|
|
2587
|
+
r"(?P<zone>[+-]\d{2}:?\d{2}|GMT|UTC|UT|Z)$",
|
|
2588
|
+
re.IGNORECASE,
|
|
2589
|
+
)
|
|
2590
|
+
_FULL_WEEKDAYS = (
|
|
2591
|
+
"monday",
|
|
2592
|
+
"tuesday",
|
|
2593
|
+
"wednesday",
|
|
2594
|
+
"thursday",
|
|
2595
|
+
"friday",
|
|
2596
|
+
"saturday",
|
|
2597
|
+
"sunday",
|
|
2598
|
+
)
|
|
2599
|
+
_FULL_MONTHS = (
|
|
2600
|
+
"january",
|
|
2601
|
+
"february",
|
|
2602
|
+
"march",
|
|
2603
|
+
"april",
|
|
2604
|
+
"may",
|
|
2605
|
+
"june",
|
|
2606
|
+
"july",
|
|
2607
|
+
"august",
|
|
2608
|
+
"september",
|
|
2609
|
+
"october",
|
|
2610
|
+
"november",
|
|
2611
|
+
"december",
|
|
2612
|
+
)
|
|
2613
|
+
_UTC_ZONE_NAMES = frozenset(["gmt", "utc", "ut", "z"])
|
|
2614
|
+
|
|
2615
|
+
|
|
2616
|
+
def _abbreviation(word, full_names):
|
|
2617
|
+
"""`word` as a title-case 3-letter abbreviation, or None if it is not one.
|
|
2618
|
+
|
|
2619
|
+
Accepts the abbreviation or the full English name in any case.
|
|
2620
|
+
"""
|
|
2621
|
+
lowered = word.lower()
|
|
2622
|
+
if len(lowered) == 3 and any(name[:3] == lowered for name in full_names):
|
|
2623
|
+
return lowered.title()
|
|
2624
|
+
if lowered in full_names:
|
|
2625
|
+
return lowered[:3].title()
|
|
2626
|
+
return None
|
|
2627
|
+
|
|
2628
|
+
|
|
2629
|
+
def _normalize_timestamp(value):
|
|
2630
|
+
"""Return the valid RFC 5322 spelling of a loosely-formatted `value`, or None.
|
|
2631
|
+
|
|
2632
|
+
A day-of-week that is not the date's real one is dropped rather than
|
|
2633
|
+
recomputed: RFC 5322 makes it optional, and the date is what the author
|
|
2634
|
+
wrote down deliberately.
|
|
2635
|
+
"""
|
|
2636
|
+
match = LOOSE_TIMESTAMP_RE.match(value)
|
|
2637
|
+
if not match:
|
|
2638
|
+
return None
|
|
2639
|
+
month = _abbreviation(match.group("mon"), _FULL_MONTHS)
|
|
2640
|
+
if month is None:
|
|
2641
|
+
return None
|
|
2642
|
+
zone = match.group("zone")
|
|
2643
|
+
zone = "+0000" if zone.lower() in _UTC_ZONE_NAMES else zone.replace(":", "")
|
|
2644
|
+
rest = (
|
|
2645
|
+
f"{int(match.group('day')):02d} {month} {match.group('year')} "
|
|
2646
|
+
f"{int(match.group('hour')):02d}:{match.group('minute')}:"
|
|
2647
|
+
f"{match.group('second')} {zone}"
|
|
2648
|
+
)
|
|
2649
|
+
parsed = _parse_timestamp(rest)
|
|
2650
|
+
if parsed is None:
|
|
2651
|
+
return None
|
|
2652
|
+
dow = match.group("dow")
|
|
2653
|
+
if dow is not None:
|
|
2654
|
+
abbreviation = _abbreviation(dow, _FULL_WEEKDAYS)
|
|
2655
|
+
if abbreviation is None:
|
|
2656
|
+
return None # not a weekday at all: something else is wrong
|
|
2657
|
+
if abbreviation == WEEKDAY_ABBREVS[parsed.weekday()]:
|
|
2658
|
+
return f"{abbreviation}, {rest}"
|
|
2659
|
+
return rest
|
|
2660
|
+
|
|
2661
|
+
|
|
2662
|
+
def fix_timestamp_format(src, value_re, code, field):
|
|
2663
|
+
"""E202/E216: normalize the formatting of a loosely-written timestamp.
|
|
2664
|
+
|
|
2665
|
+
`value_re` is MODTIME_VALUE_RE or FIRST_PROP_VALUE_RE and `code` the code
|
|
2666
|
+
its check reports. A CDATA-wrapped value is left untouched.
|
|
2667
|
+
"""
|
|
2668
|
+
fixed = []
|
|
2669
|
+
|
|
2670
|
+
def repl(match):
|
|
2671
|
+
inner = match.group(1)
|
|
2672
|
+
if "<![CDATA[" in inner:
|
|
2673
|
+
return match.group(0)
|
|
2674
|
+
value = inner.strip()
|
|
2675
|
+
if _valid_timestamp(value):
|
|
2676
|
+
return match.group(0)
|
|
2677
|
+
new_value = _normalize_timestamp(value)
|
|
2678
|
+
if new_value is None:
|
|
2679
|
+
return match.group(0)
|
|
2680
|
+
fixed.append(
|
|
2681
|
+
(
|
|
2682
|
+
_lineno(src, match.start()),
|
|
2683
|
+
code,
|
|
2684
|
+
f'{field} "{value}" set to "{new_value}"',
|
|
2685
|
+
)
|
|
2686
|
+
)
|
|
2687
|
+
return _sub_capture(match, new_value)
|
|
2688
|
+
|
|
2689
|
+
return value_re.sub(repl, src), fixed
|
|
2690
|
+
|
|
2691
|
+
|
|
2692
|
+
def fix_cve_names(src):
|
|
2693
|
+
"""E209: uppercase a lowercase/mixed-case `cve-` prefix in <CVENames>.
|
|
2694
|
+
|
|
2695
|
+
Only that prefix is touched; several <CVENames> elements are NOT merged
|
|
2696
|
+
into one, and any other malformed value is left for a human.
|
|
2697
|
+
"""
|
|
2698
|
+
fixed = []
|
|
2699
|
+
|
|
2700
|
+
def repl(match):
|
|
2701
|
+
inner = match.group(1)
|
|
2702
|
+
new_inner = re.sub(
|
|
2703
|
+
r"(?<![\w-])cve-(?=\d{4}-\d{4,}(?![\w-]))",
|
|
2704
|
+
"CVE-",
|
|
2705
|
+
inner,
|
|
2706
|
+
flags=re.IGNORECASE,
|
|
2707
|
+
)
|
|
2708
|
+
if new_inner == inner:
|
|
2709
|
+
return match.group(0)
|
|
2710
|
+
fixed.append(
|
|
2711
|
+
(
|
|
2712
|
+
_lineno(src, match.start()),
|
|
2713
|
+
"E209",
|
|
2714
|
+
"uppercased the CVE- prefix of a CVENames value",
|
|
2715
|
+
)
|
|
2716
|
+
)
|
|
2717
|
+
return f"<CVENames>{new_inner}</CVENames>"
|
|
2718
|
+
|
|
2719
|
+
return CVENAMES_TAG_RE.sub(repl, src), fixed
|
|
2720
|
+
|
|
2721
|
+
|
|
2722
|
+
# E219: an HH:MM:SS duration missing zero-padding (6:5:0)
|
|
2723
|
+
LOOSE_EVALUATION_PERIOD_RE = re.compile(r"^(\d{1,}):(\d{1,2}):(\d{1,2})$")
|
|
2724
|
+
|
|
2725
|
+
|
|
2726
|
+
def fix_evaluation_period(src):
|
|
2727
|
+
"""E219: zero-pad an x-relevance-evaluation-period (6:00:00 -> 06:00:00).
|
|
2728
|
+
|
|
2729
|
+
Only when the minutes and seconds are already below 60; an out-of-range
|
|
2730
|
+
part is left for a human.
|
|
2731
|
+
"""
|
|
2732
|
+
fixed = []
|
|
2733
|
+
|
|
2734
|
+
def repl(match):
|
|
2735
|
+
if match.group(1).strip().lower() != "x-relevance-evaluation-period":
|
|
2736
|
+
return match.group(0)
|
|
2737
|
+
inner = match.group(2)
|
|
2738
|
+
if "<![CDATA[" in inner:
|
|
2739
|
+
return match.group(0)
|
|
2740
|
+
value = inner.strip()
|
|
2741
|
+
loose = LOOSE_EVALUATION_PERIOD_RE.match(value)
|
|
2742
|
+
if EVALUATION_PERIOD_RE.match(value) or not loose:
|
|
2743
|
+
return match.group(0)
|
|
2744
|
+
hours, minutes, seconds = (int(part) for part in loose.groups())
|
|
2745
|
+
if minutes > 59 or seconds > 59:
|
|
2746
|
+
return match.group(0)
|
|
2747
|
+
new_value = f"{hours:02d}:{minutes:02d}:{seconds:02d}"
|
|
2748
|
+
fixed.append(
|
|
2749
|
+
(
|
|
2750
|
+
_lineno(src, match.start()),
|
|
2751
|
+
"E219",
|
|
2752
|
+
f'x-relevance-evaluation-period "{value}" set to "{new_value}"',
|
|
2753
|
+
)
|
|
2754
|
+
)
|
|
2755
|
+
full = match.group(0)
|
|
2756
|
+
start = match.start(2) - match.start(0)
|
|
2757
|
+
end = match.end(2) - match.start(0)
|
|
2758
|
+
return full[:start] + new_value + full[end:]
|
|
2759
|
+
|
|
2760
|
+
return NAMED_MIMEFIELD_RE.sub(repl, src), fixed
|
|
2761
|
+
|
|
2762
|
+
|
|
2763
|
+
def fix_download_size_total(src):
|
|
2764
|
+
"""W220: set <DownloadSize> to the prefetches' total, unzip.exe left out.
|
|
2765
|
+
|
|
2766
|
+
Rewrites only what check_download_size_total reports (same helper), so an
|
|
2767
|
+
unknowable total -- a conditional prefetch, a runtime size, a dynamic
|
|
2768
|
+
download -- is never touched. The unzip utility is left out of the total
|
|
2769
|
+
the way the console computes it.
|
|
2770
|
+
"""
|
|
2771
|
+
mismatch = _download_size_mismatch(src)
|
|
2772
|
+
if mismatch is None:
|
|
2773
|
+
return src, []
|
|
2774
|
+
size_match, declared, total, unzip = mismatch
|
|
2775
|
+
new_size = total - unzip
|
|
2776
|
+
fixed = [
|
|
2777
|
+
(
|
|
2778
|
+
_lineno(src, size_match.start()),
|
|
2779
|
+
"W220",
|
|
2780
|
+
f"DownloadSize {declared} set to the prefetches' total, {new_size}",
|
|
2781
|
+
)
|
|
2782
|
+
]
|
|
2783
|
+
new_src = (
|
|
2784
|
+
src[: size_match.start()]
|
|
2785
|
+
+ f"<DownloadSize>{new_size}</DownloadSize>"
|
|
2786
|
+
+ src[size_match.end() :]
|
|
2787
|
+
)
|
|
2788
|
+
return new_src, fixed
|
|
2789
|
+
|
|
2790
|
+
|
|
2791
|
+
def fix_title(src):
|
|
2792
|
+
"""W209: trim a <Title>, detab it, and collapse internal space runs.
|
|
2793
|
+
|
|
2794
|
+
A CDATA-wrapped title is left untouched (its content is opaque here).
|
|
2795
|
+
"""
|
|
2796
|
+
fixed = []
|
|
2797
|
+
|
|
2798
|
+
def repl(match):
|
|
2799
|
+
inner = match.group(1)
|
|
2800
|
+
if "<![CDATA[" in inner:
|
|
2801
|
+
return match.group(0)
|
|
2802
|
+
new_inner = DOUBLE_SPACE_RE.sub(" ", inner.replace("\t", " ").strip())
|
|
2803
|
+
if new_inner == inner:
|
|
2804
|
+
return match.group(0)
|
|
2805
|
+
fixed.append(
|
|
2806
|
+
(
|
|
2807
|
+
_lineno(src, match.start()),
|
|
2808
|
+
"W209",
|
|
2809
|
+
"normalized Title whitespace (trim, tabs, repeated spaces)",
|
|
2810
|
+
)
|
|
2811
|
+
)
|
|
2812
|
+
return f"<Title>{new_inner}</Title>"
|
|
2813
|
+
|
|
2814
|
+
return TITLE_TAG_RE.sub(repl, src), fixed
|
|
2815
|
+
|
|
2816
|
+
|
|
2817
|
+
def fix_relevance_whitespace(src):
|
|
2818
|
+
"""W213: trim leading/trailing whitespace from a <Relevance> body.
|
|
2819
|
+
|
|
2820
|
+
A CDATA-wrapped Relevance is left untouched (its content is opaque here),
|
|
2821
|
+
mirroring fix_title's treatment of a CDATA-wrapped Title.
|
|
2822
|
+
"""
|
|
2823
|
+
fixed = []
|
|
2824
|
+
|
|
2825
|
+
def repl(match):
|
|
2826
|
+
inner = match.group(1)
|
|
2827
|
+
if "<![CDATA[" in inner:
|
|
2828
|
+
return match.group(0)
|
|
2829
|
+
new_inner = inner.strip()
|
|
2830
|
+
if new_inner == inner:
|
|
2831
|
+
return match.group(0)
|
|
2832
|
+
fixed.append(
|
|
2833
|
+
(
|
|
2834
|
+
_lineno(src, match.start()),
|
|
2835
|
+
"W213",
|
|
2836
|
+
"trimmed Relevance whitespace",
|
|
2837
|
+
)
|
|
2838
|
+
)
|
|
2839
|
+
return f"<Relevance>{new_inner}</Relevance>"
|
|
2840
|
+
|
|
2841
|
+
return RELEVANCE_TAG_RE.sub(repl, src), fixed
|
|
2842
|
+
|
|
2843
|
+
|
|
2844
|
+
def fix_blank_lines(src):
|
|
2845
|
+
"""W205: collapse 2+ blank lines before </ActionScript> to one."""
|
|
2846
|
+
fixed = []
|
|
2847
|
+
|
|
2848
|
+
def repl(match):
|
|
2849
|
+
fixed.append(
|
|
2850
|
+
(
|
|
2851
|
+
_lineno(src, match.start()),
|
|
2852
|
+
"W205",
|
|
2853
|
+
"collapsed blank lines before </ActionScript>",
|
|
2854
|
+
)
|
|
2855
|
+
)
|
|
2856
|
+
return match.group(1) + "\n" + match.group(2)
|
|
2857
|
+
|
|
2858
|
+
return BLANK_BEFORE_CLOSE_RE.sub(repl, src), fixed
|
|
2859
|
+
|
|
2860
|
+
|
|
2861
|
+
def fix_cdata_close(src):
|
|
2862
|
+
"""E215: strip the whitespace before an ActionScript close tag.
|
|
2863
|
+
|
|
2864
|
+
Applies to both the CDATA terminator (`]]></ActionScript>`) and, for a
|
|
2865
|
+
non-CDATA body, the bare close (`</ActionScript>`).
|
|
2866
|
+
"""
|
|
2867
|
+
fixed = []
|
|
2868
|
+
|
|
2869
|
+
def repl(match):
|
|
2870
|
+
canonical = (
|
|
2871
|
+
CDATA_CLOSE_CANONICAL
|
|
2872
|
+
if "]]>" in match.group(0)
|
|
2873
|
+
else CDATA_CLOSE_CANONICAL_PLAIN
|
|
2874
|
+
)
|
|
2875
|
+
if match.group(0) == canonical:
|
|
2876
|
+
return match.group(0)
|
|
2877
|
+
fixed.append(
|
|
2878
|
+
(
|
|
2879
|
+
_lineno(src, match.start()),
|
|
2880
|
+
"E215",
|
|
2881
|
+
"stripped whitespace before the ActionScript close tag",
|
|
2882
|
+
)
|
|
2883
|
+
)
|
|
2884
|
+
return canonical
|
|
2885
|
+
|
|
2886
|
+
return CDATA_CLOSE_RE.sub(repl, src), fixed
|
|
2887
|
+
|
|
2888
|
+
|
|
2889
|
+
def fix_cdata_required(src):
|
|
2890
|
+
"""E207: unescape a Description/Relevance/ActionScript body and CDATA-wrap it.
|
|
2891
|
+
|
|
2892
|
+
Skips a body whose unescaped text would contain `]]>`, which cannot sit
|
|
2893
|
+
inside a single CDATA section -- that one is left to error.
|
|
2894
|
+
"""
|
|
2895
|
+
fixed = []
|
|
2896
|
+
|
|
2897
|
+
def repl(match):
|
|
2898
|
+
tag, inner = match.group(1), match.group(2)
|
|
2899
|
+
if "<![CDATA[" in inner or CHILD_ELEMENT_RE.search(inner):
|
|
2900
|
+
return match.group(0)
|
|
2901
|
+
if not SPECIAL_ENTITY_RE.search(inner):
|
|
2902
|
+
return match.group(0)
|
|
2903
|
+
decoded = _xml_unescape(inner)
|
|
2904
|
+
if "]]>" in decoded:
|
|
2905
|
+
return match.group(0)
|
|
2906
|
+
fixed.append(
|
|
2907
|
+
(
|
|
2908
|
+
_lineno(src, match.start()),
|
|
2909
|
+
"E207",
|
|
2910
|
+
f"wrapped {tag} body in <![CDATA[ ... ]]>",
|
|
2911
|
+
)
|
|
2912
|
+
)
|
|
2913
|
+
open_tag = match.group(0)[: match.start(2) - match.start()]
|
|
2914
|
+
return f"{open_tag}<![CDATA[{decoded}]]></{tag}>"
|
|
2915
|
+
|
|
2916
|
+
return CDATA_ELEMENT_RE.sub(repl, src), fixed
|
|
2917
|
+
|
|
2918
|
+
|
|
2919
|
+
def _sub_capture(match, new_inner):
|
|
2920
|
+
"""Return match.group(0) with its capture group 1 replaced by `new_inner`."""
|
|
2921
|
+
full = match.group(0)
|
|
2922
|
+
start = match.start(1) - match.start(0)
|
|
2923
|
+
end = match.end(1) - match.start(0)
|
|
2924
|
+
return full[:start] + new_inner + full[end:]
|
|
2925
|
+
|
|
2926
|
+
|
|
2927
|
+
def fix_script_comments(src):
|
|
2928
|
+
"""E221: rewrite a `//` line comment inside a Description <script> block
|
|
2929
|
+
as `/* ... */`.
|
|
2930
|
+
|
|
2931
|
+
Operates on the literal <script> markup already in `src` -- run this
|
|
2932
|
+
after fix_cdata_required (E207) in the same pass so an entity-escaped
|
|
2933
|
+
Description is unescaped and CDATA-wrapped first, revealing its literal
|
|
2934
|
+
<script> body for this fixer to rewrite. A comment whose text contains
|
|
2935
|
+
`*/` cannot be safely wrapped in a single block comment and is left alone
|
|
2936
|
+
(still reported by check_script_comments).
|
|
2937
|
+
"""
|
|
2938
|
+
fixed = []
|
|
2939
|
+
|
|
2940
|
+
def rewrite_body(body_text):
|
|
2941
|
+
pieces = []
|
|
2942
|
+
cursor = 0
|
|
2943
|
+
hits = []
|
|
2944
|
+
for start, end, comment_body in _js_line_comments(body_text):
|
|
2945
|
+
if "*/" in comment_body:
|
|
2946
|
+
continue
|
|
2947
|
+
pieces.append(body_text[cursor:start])
|
|
2948
|
+
pieces.append("/*" + comment_body.rstrip(" \t") + " */")
|
|
2949
|
+
cursor = end
|
|
2950
|
+
hits.append(start)
|
|
2951
|
+
pieces.append(body_text[cursor:])
|
|
2952
|
+
return "".join(pieces), hits
|
|
2953
|
+
|
|
2954
|
+
def repl_description(desc_match):
|
|
2955
|
+
body = desc_match.group(1)
|
|
2956
|
+
body_abs_start = desc_match.start(1)
|
|
2957
|
+
if not SCRIPT_OPEN_RE.search(body):
|
|
2958
|
+
return desc_match.group(0)
|
|
2959
|
+
|
|
2960
|
+
def repl_script(script_match):
|
|
2961
|
+
script_body = script_match.group(1)
|
|
2962
|
+
script_abs_start = body_abs_start + script_match.start(1)
|
|
2963
|
+
new_script_body, hit_offsets = rewrite_body(script_body)
|
|
2964
|
+
for offset in hit_offsets:
|
|
2965
|
+
fixed.append(
|
|
2966
|
+
(
|
|
2967
|
+
_lineno(src, script_abs_start + offset),
|
|
2968
|
+
"E221",
|
|
2969
|
+
(
|
|
2970
|
+
"rewrote a `//` comment as `/* ... */` in a Description "
|
|
2971
|
+
"<script> block"
|
|
2972
|
+
),
|
|
2973
|
+
)
|
|
2974
|
+
)
|
|
2975
|
+
return _sub_capture(script_match, new_script_body)
|
|
2976
|
+
|
|
2977
|
+
new_body = SCRIPT_BLOCK_RE.sub(repl_script, body)
|
|
2978
|
+
return _sub_capture(desc_match, new_body)
|
|
2979
|
+
|
|
2980
|
+
new_src = DESCRIPTION_ELEMENT_RE.sub(repl_description, src)
|
|
2981
|
+
return new_src, fixed
|
|
2982
|
+
|
|
2983
|
+
|
|
2984
|
+
def fix_actionscript_cdata(src):
|
|
2985
|
+
"""W204: wrap an un-wrapped ActionScript body in <![CDATA[ ... ]]>.
|
|
2986
|
+
|
|
2987
|
+
The body is XML-unescaped before wrapping: inside CDATA an entity is
|
|
2988
|
+
literal text, so wrapping `"x"` as-is would hand the agent
|
|
2989
|
+
`"x"` instead of `"x"`. Skips a body whose unescaped text
|
|
2990
|
+
contains a `]]>` sequence, which cannot be placed inside a CDATA section
|
|
2991
|
+
without splitting -- that one is left to warn.
|
|
2992
|
+
"""
|
|
2993
|
+
fixed = []
|
|
2994
|
+
|
|
2995
|
+
def repl(match):
|
|
2996
|
+
attrs, body = match.group(1), match.group(2)
|
|
2997
|
+
if "<![CDATA[" in body:
|
|
2998
|
+
return match.group(0)
|
|
2999
|
+
body = _xml_unescape(body)
|
|
3000
|
+
if "]]>" in body:
|
|
3001
|
+
return match.group(0)
|
|
3002
|
+
fixed.append(
|
|
3003
|
+
(
|
|
3004
|
+
_lineno(src, match.start()),
|
|
3005
|
+
"W204",
|
|
3006
|
+
"wrapped ActionScript body in <![CDATA[ ... ]]>",
|
|
3007
|
+
)
|
|
3008
|
+
)
|
|
3009
|
+
return f"<ActionScript{attrs}><![CDATA[{body}]]></ActionScript>"
|
|
3010
|
+
|
|
3011
|
+
return ACTIONSCRIPT_FULL_RE.sub(repl, src), fixed
|
|
3012
|
+
|
|
3013
|
+
|
|
3014
|
+
def _detect_indent(inner):
|
|
3015
|
+
"""Return the indentation of the first element inside a content block."""
|
|
3016
|
+
match = re.search(r"\n([ \t]+)<\w", inner)
|
|
3017
|
+
return match.group(1) if match else "\t\t"
|
|
3018
|
+
|
|
3019
|
+
|
|
3020
|
+
def _insert_ordered(inner, new_text, anchors):
|
|
3021
|
+
"""Insert `new_text` (a full indented line) before the first anchor found."""
|
|
3022
|
+
for anchor in anchors:
|
|
3023
|
+
pos = inner.find(anchor)
|
|
3024
|
+
if pos != -1:
|
|
3025
|
+
line_start = inner.rfind("\n", 0, pos) + 1
|
|
3026
|
+
return inner[:line_start] + new_text + inner[line_start:]
|
|
3027
|
+
stripped = inner.rstrip()
|
|
3028
|
+
trailing = inner[len(stripped) :]
|
|
3029
|
+
return stripped + "\n" + new_text + (trailing or "\n")
|
|
3030
|
+
|
|
3031
|
+
|
|
3032
|
+
def fix_missing_dates(src, now=None, fix_srd=True, fix_modtime=True):
|
|
3033
|
+
"""W201/W202: insert a missing SourceReleaseDate / modification time.
|
|
3034
|
+
|
|
3035
|
+
Values use the moment the linter ran -- except that a SourceReleaseDate
|
|
3036
|
+
inserted into content that already carries a valid modification time
|
|
3037
|
+
takes that timestamp's date, so the fix never makes the release date
|
|
3038
|
+
later than the last modification (W219). Insertion positions keep the BES.xsd
|
|
3039
|
+
element ordering (SourceReleaseDate in the metadata block; the modification
|
|
3040
|
+
time MIMEField before <Domain>). `fix_srd` / `fix_modtime` gate each insert
|
|
3041
|
+
independently so a single per-field opt-out marker only suppresses its own.
|
|
3042
|
+
"""
|
|
3043
|
+
fixed = []
|
|
3044
|
+
date_now = _today_str(now)
|
|
3045
|
+
modtime = _modtime_str(now)
|
|
3046
|
+
|
|
3047
|
+
def repl(match):
|
|
3048
|
+
open_tag, _tag, inner, close = match.groups()
|
|
3049
|
+
indent = _detect_indent(inner)
|
|
3050
|
+
lineno = _lineno(src, match.start())
|
|
3051
|
+
if fix_srd and "<SourceReleaseDate" not in inner:
|
|
3052
|
+
release = date_now
|
|
3053
|
+
existing = MODTIME_VALUE_RE.search(inner)
|
|
3054
|
+
modified = existing and _parse_timestamp(_strip_cdata(existing.group(1)))
|
|
3055
|
+
if modified:
|
|
3056
|
+
release = modified.date().isoformat()
|
|
3057
|
+
inner = _insert_ordered(
|
|
3058
|
+
inner,
|
|
3059
|
+
f"{indent}<SourceReleaseDate>{release}</SourceReleaseDate>\n",
|
|
3060
|
+
SRD_ANCHORS,
|
|
3061
|
+
)
|
|
3062
|
+
fixed.append((lineno, "W202", f"inserted SourceReleaseDate {release}"))
|
|
3063
|
+
if fix_modtime and MODIFICATION_TIME_NAME not in inner:
|
|
3064
|
+
block = (
|
|
3065
|
+
f"{indent}<MIMEField>\n"
|
|
3066
|
+
f"{indent}\t<Name>{MODIFICATION_TIME_NAME}</Name>\n"
|
|
3067
|
+
f"{indent}\t<Value>{modtime}</Value>\n"
|
|
3068
|
+
f"{indent}</MIMEField>\n"
|
|
3069
|
+
)
|
|
3070
|
+
inner = _insert_ordered(inner, block, MODTIME_ANCHORS)
|
|
3071
|
+
fixed.append((lineno, "W201", "inserted x-fixlet-modification-time"))
|
|
3072
|
+
return open_tag + inner + close
|
|
3073
|
+
|
|
3074
|
+
return CONTENT_BLOCK_RE.sub(repl, src), fixed
|
|
3075
|
+
|
|
3076
|
+
|
|
3077
|
+
# --------------------------------------------------------------------------
|
|
3078
|
+
# file-level checks / fixers (E214 XML declaration, W210 trailing whitespace)
|
|
3079
|
+
# --------------------------------------------------------------------------
|
|
3080
|
+
|
|
3081
|
+
|
|
3082
|
+
def _xml_declaration_encoding(src):
|
|
3083
|
+
"""Return (has_decl, encoding_or_None) for the leading XML declaration."""
|
|
3084
|
+
match = XML_DECL_RE.match(src)
|
|
3085
|
+
if match is None:
|
|
3086
|
+
return False, None
|
|
3087
|
+
enc = XML_DECL_ENCODING_RE.search(match.group(1))
|
|
3088
|
+
return True, (enc.group(1) if enc else None)
|
|
3089
|
+
|
|
3090
|
+
|
|
3091
|
+
def check_xml_declaration(src):
|
|
3092
|
+
"""E214: the file must open with an <?xml ... encoding="UTF-8"?> declaration."""
|
|
3093
|
+
has_decl, encoding = _xml_declaration_encoding(src)
|
|
3094
|
+
if not has_decl:
|
|
3095
|
+
message = "file has no XML declaration"
|
|
3096
|
+
elif encoding is None:
|
|
3097
|
+
message = 'XML declaration has no encoding (expected encoding="UTF-8")'
|
|
3098
|
+
elif encoding.lower() != "utf-8":
|
|
3099
|
+
message = f'XML declaration encoding "{encoding}" is not UTF-8'
|
|
3100
|
+
else:
|
|
3101
|
+
return []
|
|
3102
|
+
return [(1, "E214", f"{message}; add `{XML_DECL_MARKER}` if intentional")]
|
|
3103
|
+
|
|
3104
|
+
|
|
3105
|
+
def check_trailing_whitespace(src):
|
|
3106
|
+
"""W210: no line may have trailing spaces or tabs."""
|
|
3107
|
+
issues = []
|
|
3108
|
+
for lineno, line in enumerate(src.split("\n"), start=1):
|
|
3109
|
+
if line != line.rstrip(" \t"):
|
|
3110
|
+
issues.append(
|
|
3111
|
+
(
|
|
3112
|
+
lineno,
|
|
3113
|
+
"W210",
|
|
3114
|
+
(
|
|
3115
|
+
"line has trailing whitespace; add "
|
|
3116
|
+
f"`{TRAILING_WS_MARKER}` if intentional"
|
|
3117
|
+
),
|
|
3118
|
+
)
|
|
3119
|
+
)
|
|
3120
|
+
return issues
|
|
3121
|
+
|
|
3122
|
+
|
|
3123
|
+
def fix_xml_declaration(src):
|
|
3124
|
+
"""E214: insert a UTF-8 XML declaration, or set encoding="UTF-8" on it."""
|
|
3125
|
+
has_decl, encoding = _xml_declaration_encoding(src)
|
|
3126
|
+
if has_decl and encoding is not None and encoding.lower() == "utf-8":
|
|
3127
|
+
return src, []
|
|
3128
|
+
if not has_decl:
|
|
3129
|
+
# preserve a leading BOM if one is present
|
|
3130
|
+
bom = "\ufeff" if src.startswith("\ufeff") else ""
|
|
3131
|
+
body = src[len(bom) :]
|
|
3132
|
+
new_src = f'{bom}<?xml version="1.0" encoding="UTF-8"?>\n{body}'
|
|
3133
|
+
return new_src, [(1, "E214", "inserted XML declaration")]
|
|
3134
|
+
match = XML_DECL_RE.match(src)
|
|
3135
|
+
version = XML_DECL_VERSION_RE.search(match.group(1))
|
|
3136
|
+
version_value = version.group(1) if version else "1.0"
|
|
3137
|
+
new_decl = f'<?xml version="{version_value}" encoding="UTF-8"?>'
|
|
3138
|
+
new_src = src[: match.start()] + new_decl + src[match.end() :]
|
|
3139
|
+
return new_src, [(1, "E214", 'set XML declaration encoding to "UTF-8"')]
|
|
3140
|
+
|
|
3141
|
+
|
|
3142
|
+
def fix_trailing_whitespace(src):
|
|
3143
|
+
"""W210: strip trailing spaces/tabs from every line."""
|
|
3144
|
+
new_src = TRAILING_WS_RE.sub("", src)
|
|
3145
|
+
if new_src == src:
|
|
3146
|
+
return src, []
|
|
3147
|
+
fixed = [
|
|
3148
|
+
(lineno, "W210", "stripped trailing whitespace")
|
|
3149
|
+
for lineno, line in enumerate(src.split("\n"), start=1)
|
|
3150
|
+
if line != line.rstrip(" \t")
|
|
3151
|
+
]
|
|
3152
|
+
return new_src, fixed
|
|
3153
|
+
|
|
3154
|
+
|
|
3155
|
+
# --------------------------------------------------------------------------
|
|
3156
|
+
# driver
|
|
3157
|
+
# --------------------------------------------------------------------------
|
|
3158
|
+
|
|
3159
|
+
|
|
3160
|
+
def check_filename_matches_title(path, src):
|
|
3161
|
+
"""W217: the file's basename must match its first content object's Title.
|
|
3162
|
+
|
|
3163
|
+
The Title is sanitized for filename-illegal characters the same way an
|
|
3164
|
+
author would when saving the file (/ \\ : * ? " < > | -> _) before it is
|
|
3165
|
+
compared against the basename (extension stripped). Only checked when
|
|
3166
|
+
--check-filename is passed.
|
|
3167
|
+
"""
|
|
3168
|
+
match = TITLE_TAG_RE.search(src)
|
|
3169
|
+
if match is None:
|
|
3170
|
+
return []
|
|
3171
|
+
title = _strip_cdata(match.group(1)).strip()
|
|
3172
|
+
if title == "":
|
|
3173
|
+
return []
|
|
3174
|
+
expected = FILENAME_ILLEGAL_RE.sub("_", title)
|
|
3175
|
+
basename = os.path.basename(path)
|
|
3176
|
+
stem, _ext = os.path.splitext(basename)
|
|
3177
|
+
if stem == expected:
|
|
3178
|
+
return []
|
|
3179
|
+
return [
|
|
3180
|
+
(
|
|
3181
|
+
1,
|
|
3182
|
+
"W217",
|
|
3183
|
+
(
|
|
3184
|
+
f'filename "{basename}" does not match Title "{title}" (expected '
|
|
3185
|
+
f'stem "{expected}"); add `{FILENAME_MARKER}` if intentional'
|
|
3186
|
+
),
|
|
3187
|
+
)
|
|
3188
|
+
]
|
|
3189
|
+
|
|
3190
|
+
|
|
3191
|
+
def check_file(
|
|
3192
|
+
path,
|
|
3193
|
+
disabled=frozenset(),
|
|
3194
|
+
strict=False,
|
|
3195
|
+
auto_fix=False,
|
|
3196
|
+
now=None,
|
|
3197
|
+
check_filename=False,
|
|
3198
|
+
severities=None,
|
|
3199
|
+
prefetch_url_schemes=None,
|
|
3200
|
+
):
|
|
3201
|
+
"""Check one BES file; return (issues, fixed).
|
|
3202
|
+
|
|
3203
|
+
Each of `issues` and `fixed` is a list of (lineno, code, message). A file is
|
|
3204
|
+
skipped (returns [], []) when it carries the file-level skip marker or looks
|
|
3205
|
+
like a mustache template. A file that will not parse as XML yields a single
|
|
3206
|
+
advisory W200 (bes-schema-validate owns file validity) and no other checks.
|
|
3207
|
+
|
|
3208
|
+
When `auto_fix` is set, the fixable conventions are rewritten in place and
|
|
3209
|
+
reported under `fixed`; the CDATA wrap (W204) is applied only when `strict`
|
|
3210
|
+
is also set, and CRLF normalization (E208) is applied last so the written
|
|
3211
|
+
file is entirely CRLF. Read-only, a file that is not all-CRLF is an E208
|
|
3212
|
+
error. The file is read as raw bytes and normalized to LF in memory so the
|
|
3213
|
+
checks are line-ending agnostic. `check_filename` enables W217 (off by
|
|
3214
|
+
default, matching --check-filename). `severities`, if given, overrides
|
|
3215
|
+
W216's allowed SourceSeverity vocabulary (see --severity-values), and
|
|
3216
|
+
`prefetch_url_schemes` W206's allowed URL schemes (see
|
|
3217
|
+
--prefetch-url-schemes).
|
|
3218
|
+
"""
|
|
3219
|
+
if not os.path.isfile(path):
|
|
3220
|
+
return [(1, "W200", "file not found; skipping")], []
|
|
3221
|
+
|
|
3222
|
+
# normalize to LF in memory so the checks/fixers are line-ending agnostic;
|
|
3223
|
+
# `raw` is kept to inspect (and, on auto-fix, rewrite) the real endings.
|
|
3224
|
+
raw, src, _was_crlf = read_source(path)
|
|
3225
|
+
|
|
3226
|
+
if SKIP_MARKER in src:
|
|
3227
|
+
return [], []
|
|
3228
|
+
if MUSTACHE_RE.search(src):
|
|
3229
|
+
return [], []
|
|
3230
|
+
|
|
3231
|
+
try:
|
|
3232
|
+
root = ElementTree.fromstring(src)
|
|
3233
|
+
except ElementTree.ParseError as err:
|
|
3234
|
+
return [(1, "W200", f"not parseable BES XML ({err}); skipping")], []
|
|
3235
|
+
|
|
3236
|
+
check_e208 = "E208" not in disabled
|
|
3237
|
+
crlf_ok = _is_all_crlf(raw)
|
|
3238
|
+
|
|
3239
|
+
fixed = []
|
|
3240
|
+
refusals = []
|
|
3241
|
+
if auto_fix:
|
|
3242
|
+
# each fix stage is kept only if it leaves a schema-valid file valid
|
|
3243
|
+
# (E222); the per-block fixers in _autofix are one stage
|
|
3244
|
+
guard = bes_common.SchemaGuard(
|
|
3245
|
+
src, "E222", path, validate="E222" not in disabled
|
|
3246
|
+
)
|
|
3247
|
+
guard.apply(*_autofix(src, root, disabled, strict, now, severities))
|
|
3248
|
+
# file-level fixers run on the whole document (after the per-block ones):
|
|
3249
|
+
# strip trailing whitespace, then ensure the XML declaration.
|
|
3250
|
+
if "W210" not in disabled and TRAILING_WS_MARKER not in src:
|
|
3251
|
+
guard.apply(*fix_trailing_whitespace(guard.src))
|
|
3252
|
+
if "E214" not in disabled and XML_DECL_MARKER not in src:
|
|
3253
|
+
guard.apply(*fix_xml_declaration(guard.src))
|
|
3254
|
+
new_src, fixed, refusals = guard.src, guard.fixed, guard.refused
|
|
3255
|
+
# CRLF normalization runs LAST: BES files must be entirely CRLF, so any
|
|
3256
|
+
# auto-fix leaves the whole file CRLF (rather than preserving endings).
|
|
3257
|
+
# If the CRLF rule is disabled, write whatever endings resulted (LF).
|
|
3258
|
+
if check_e208:
|
|
3259
|
+
final_bytes = _to_crlf(new_src).encode("utf-8")
|
|
3260
|
+
if not crlf_ok:
|
|
3261
|
+
fixed.append((1, "E208", "normalized line endings to CRLF"))
|
|
3262
|
+
else:
|
|
3263
|
+
final_bytes = new_src.encode("utf-8")
|
|
3264
|
+
if final_bytes != raw:
|
|
3265
|
+
with open(path, "wb") as handle:
|
|
3266
|
+
handle.write(final_bytes)
|
|
3267
|
+
src = new_src
|
|
3268
|
+
try:
|
|
3269
|
+
root = ElementTree.fromstring(src)
|
|
3270
|
+
except ElementTree.ParseError as err:
|
|
3271
|
+
return [(1, "W200", f"not parseable BES XML after fixes ({err})")], fixed
|
|
3272
|
+
|
|
3273
|
+
issues = _run_checks(
|
|
3274
|
+
src,
|
|
3275
|
+
root,
|
|
3276
|
+
disabled,
|
|
3277
|
+
severities=severities,
|
|
3278
|
+
prefetch_url_schemes=prefetch_url_schemes,
|
|
3279
|
+
)
|
|
3280
|
+
# file-level checks on the final src (after any fixes); in auto-fix mode
|
|
3281
|
+
# these come back clean unless the specific fix was disabled.
|
|
3282
|
+
if "E214" not in disabled and XML_DECL_MARKER not in src:
|
|
3283
|
+
issues += check_xml_declaration(src)
|
|
3284
|
+
if "W210" not in disabled and TRAILING_WS_MARKER not in src:
|
|
3285
|
+
issues += check_trailing_whitespace(src)
|
|
3286
|
+
if check_filename and "W217" not in disabled and FILENAME_MARKER not in src:
|
|
3287
|
+
issues += check_filename_matches_title(path, src)
|
|
3288
|
+
issues += refusals
|
|
3289
|
+
if not auto_fix and check_e208 and not crlf_ok:
|
|
3290
|
+
lone_lf = raw.count(b"\n") - raw.count(b"\r\n")
|
|
3291
|
+
lone_cr = raw.count(b"\r") - raw.count(b"\r\n")
|
|
3292
|
+
issues.append(
|
|
3293
|
+
(
|
|
3294
|
+
1,
|
|
3295
|
+
"E208",
|
|
3296
|
+
(
|
|
3297
|
+
f"BES file must use CRLF line endings throughout (found {lone_lf} "
|
|
3298
|
+
f"lone LF, {lone_cr} lone CR); enable --auto-fix to normalize"
|
|
3299
|
+
),
|
|
3300
|
+
)
|
|
3301
|
+
)
|
|
3302
|
+
return sorted(issues), fixed
|
|
3303
|
+
|
|
3304
|
+
|
|
3305
|
+
def is_bes_file(path):
|
|
3306
|
+
"""True if `path` has a recognized BES extension."""
|
|
3307
|
+
return path.endswith(BES_EXTENSIONS)
|
|
3308
|
+
|
|
3309
|
+
|
|
3310
|
+
def check_files(
|
|
3311
|
+
paths,
|
|
3312
|
+
disabled=frozenset(),
|
|
3313
|
+
strict=False,
|
|
3314
|
+
auto_fix=False,
|
|
3315
|
+
check_filename=False,
|
|
3316
|
+
severities=None,
|
|
3317
|
+
prefetch_url_schemes=None,
|
|
3318
|
+
):
|
|
3319
|
+
"""Check several BES files; return a list of (path, issues, fixed) tuples.
|
|
3320
|
+
|
|
3321
|
+
Non-BES paths are skipped. Disabled codes are filtered from the results.
|
|
3322
|
+
This is the programmatic entry point: it does no printing.
|
|
3323
|
+
"""
|
|
3324
|
+
results = []
|
|
3325
|
+
for path in paths:
|
|
3326
|
+
if not is_bes_file(path):
|
|
3327
|
+
continue
|
|
3328
|
+
issues, fixed = check_file(
|
|
3329
|
+
path,
|
|
3330
|
+
disabled=disabled,
|
|
3331
|
+
strict=strict,
|
|
3332
|
+
auto_fix=auto_fix,
|
|
3333
|
+
check_filename=check_filename,
|
|
3334
|
+
severities=severities,
|
|
3335
|
+
prefetch_url_schemes=prefetch_url_schemes,
|
|
3336
|
+
)
|
|
3337
|
+
issues = [item for item in issues if item[1] not in disabled]
|
|
3338
|
+
fixed = [item for item in fixed if item[1] not in disabled]
|
|
3339
|
+
results.append((path, issues, fixed))
|
|
3340
|
+
return results
|
|
3341
|
+
|
|
3342
|
+
|
|
3343
|
+
def discover_bes_files(root="."):
|
|
3344
|
+
"""Return all BES files under `root`, pruning hidden and noise directories."""
|
|
3345
|
+
return bes_common.discover_bes_files(root, BES_EXTENSIONS)
|
|
3346
|
+
|
|
3347
|
+
|
|
3348
|
+
def main(argv=None):
|
|
3349
|
+
"""Execution starts here.
|
|
3350
|
+
|
|
3351
|
+
argv defaults to None so this works both as a console_scripts entry point
|
|
3352
|
+
(pre-commit calls it with no arguments; argparse then reads sys.argv) and
|
|
3353
|
+
when called directly as `main(sys.argv[1:])`.
|
|
3354
|
+
"""
|
|
3355
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
3356
|
+
parser.add_argument(
|
|
3357
|
+
"--strict",
|
|
3358
|
+
action="store_true",
|
|
3359
|
+
help=(
|
|
3360
|
+
"treat warnings as failures (non-zero exit) and enable the CDATA "
|
|
3361
|
+
"auto-fix (W204); default: advisory"
|
|
3362
|
+
),
|
|
3363
|
+
)
|
|
3364
|
+
parser.add_argument(
|
|
3365
|
+
"--errors-only",
|
|
3366
|
+
action="store_true",
|
|
3367
|
+
help=(
|
|
3368
|
+
"report only E-codes: warnings are left out of the output and cannot "
|
|
3369
|
+
"fail the run (even with --strict), but every check and W-code "
|
|
3370
|
+
"auto-fix still runs -- unlike --disable"
|
|
3371
|
+
),
|
|
3372
|
+
)
|
|
3373
|
+
parser.add_argument(
|
|
3374
|
+
"--auto-fix",
|
|
3375
|
+
choices=["yes", "no"],
|
|
3376
|
+
default=None,
|
|
3377
|
+
help=(
|
|
3378
|
+
"rewrite fixable conventions in place (default: yes when files are "
|
|
3379
|
+
"given, no when auto-discovering)"
|
|
3380
|
+
),
|
|
3381
|
+
)
|
|
3382
|
+
parser.add_argument(
|
|
3383
|
+
"--disable",
|
|
3384
|
+
default="",
|
|
3385
|
+
metavar="CODES",
|
|
3386
|
+
help="comma-separated check IDs to skip entirely, e.g. --disable W204",
|
|
3387
|
+
)
|
|
3388
|
+
parser.add_argument(
|
|
3389
|
+
"--check-filename",
|
|
3390
|
+
action="store_true",
|
|
3391
|
+
help=(
|
|
3392
|
+
"enable W217: a file's basename must match its first content "
|
|
3393
|
+
"object's Title, sanitized for filename-illegal characters "
|
|
3394
|
+
"(off by default)"
|
|
3395
|
+
),
|
|
3396
|
+
)
|
|
3397
|
+
parser.add_argument(
|
|
3398
|
+
"--severity-values",
|
|
3399
|
+
default=None,
|
|
3400
|
+
metavar="VALUES",
|
|
3401
|
+
help=(
|
|
3402
|
+
"comma-separated SourceSeverity values W216 accepts (exact case), "
|
|
3403
|
+
"overriding the default vocabulary: "
|
|
3404
|
+
f"{','.join(sorted(CANONICAL_SEVERITIES))}"
|
|
3405
|
+
),
|
|
3406
|
+
)
|
|
3407
|
+
parser.add_argument(
|
|
3408
|
+
"--prefetch-url-schemes",
|
|
3409
|
+
default=None,
|
|
3410
|
+
metavar="SCHEMES",
|
|
3411
|
+
help=(
|
|
3412
|
+
"comma-separated URL schemes a W206 prefetch line may use "
|
|
3413
|
+
"(case-insensitive), overriding the default: "
|
|
3414
|
+
f"{','.join(sorted(DEFAULT_PREFETCH_URL_SCHEMES, key=str.lower))}"
|
|
3415
|
+
),
|
|
3416
|
+
)
|
|
3417
|
+
parser.add_argument(
|
|
3418
|
+
"files",
|
|
3419
|
+
nargs="*",
|
|
3420
|
+
help=(
|
|
3421
|
+
"BES files to check; if omitted, all *.bes files in the current "
|
|
3422
|
+
"folder and below are checked"
|
|
3423
|
+
),
|
|
3424
|
+
)
|
|
3425
|
+
args = parser.parse_args(argv)
|
|
3426
|
+
|
|
3427
|
+
disabled = {
|
|
3428
|
+
code.strip().upper() for code in args.disable.split(",") if code.strip()
|
|
3429
|
+
}
|
|
3430
|
+
unknown = disabled - KNOWN_CODES
|
|
3431
|
+
if unknown:
|
|
3432
|
+
print(
|
|
3433
|
+
f"warning: ignoring unknown --disable code(s): {', '.join(sorted(unknown))}"
|
|
3434
|
+
)
|
|
3435
|
+
|
|
3436
|
+
severities = None
|
|
3437
|
+
if args.severity_values is not None:
|
|
3438
|
+
severities = frozenset(
|
|
3439
|
+
value.strip() for value in args.severity_values.split(",") if value.strip()
|
|
3440
|
+
)
|
|
3441
|
+
prefetch_url_schemes = None
|
|
3442
|
+
if args.prefetch_url_schemes is not None:
|
|
3443
|
+
prefetch_url_schemes = frozenset(
|
|
3444
|
+
value.strip()
|
|
3445
|
+
for value in args.prefetch_url_schemes.split(",")
|
|
3446
|
+
if value.strip()
|
|
3447
|
+
)
|
|
3448
|
+
|
|
3449
|
+
# auto-fix defaults to yes for explicit files, no when auto-discovering; an
|
|
3450
|
+
# explicit --auto-fix always wins.
|
|
3451
|
+
discovering = not args.files
|
|
3452
|
+
if args.auto_fix is not None:
|
|
3453
|
+
auto_fix = args.auto_fix == "yes"
|
|
3454
|
+
else:
|
|
3455
|
+
auto_fix = not discovering
|
|
3456
|
+
paths = args.files if args.files else discover_bes_files(".")
|
|
3457
|
+
|
|
3458
|
+
# --errors-only drops warnings from the report only; the checks and their
|
|
3459
|
+
# fixers already ran (see --disable for skipping them)
|
|
3460
|
+
issue_count, warning_count, fix_count = bes_common.report(
|
|
3461
|
+
check_files(
|
|
3462
|
+
paths,
|
|
3463
|
+
disabled=disabled,
|
|
3464
|
+
strict=args.strict,
|
|
3465
|
+
auto_fix=auto_fix,
|
|
3466
|
+
check_filename=args.check_filename,
|
|
3467
|
+
severities=severities,
|
|
3468
|
+
prefetch_url_schemes=prefetch_url_schemes,
|
|
3469
|
+
),
|
|
3470
|
+
errors_only=args.errors_only,
|
|
3471
|
+
)
|
|
3472
|
+
|
|
3473
|
+
if fix_count:
|
|
3474
|
+
print(f"\nauto-fixed {fix_count} issue(s); review and re-stage the changes.")
|
|
3475
|
+
if warning_count:
|
|
3476
|
+
print(f"{warning_count} BES-convention warning(s).")
|
|
3477
|
+
if issue_count:
|
|
3478
|
+
print(f"{issue_count} BES-convention issue(s).")
|
|
3479
|
+
# E-codes and any fix always fail; warnings fail only under --strict
|
|
3480
|
+
return 1 if (issue_count or fix_count or (warning_count and args.strict)) else 0
|
|
3481
|
+
|
|
3482
|
+
|
|
3483
|
+
if __name__ == "__main__":
|
|
3484
|
+
sys.exit(main(sys.argv[1:]))
|