fasthtmlparser-pure 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,866 @@
1
+ """The parser: a page is scanned once in fast mode, where tags are matched to their end tags and nothing else.
2
+
3
+ Well-formed markup needs nothing more. Where the HTML5 optional-end-tag rules matter (<li>one<li>two</ul>), tags are
4
+ left open inside an element: when its end tag (or the end of the document) finds them on the stack and one of them is
5
+ a rule target (li, p, td, tr, option, ...), only that part is scanned again in rules mode, from the outermost such tag
6
+ up to that end tag. Everything before it was parsed right and is kept.
7
+
8
+ Port of ParsingCore.cs from z3l.FastHtmlParser; positions are char indexes into the source.
9
+ """
10
+
11
+ import re
12
+
13
+ from ._chars import ASCII_SPACE, SPACE, ascii_lower, lower
14
+ from ._tables import TAG_NAMES
15
+
16
+ NAMES = TAG_NAMES
17
+ KEYS = {n: k for k, n in enumerate(NAMES)}
18
+ COMMENT = KEYS["!--"]
19
+ DOCTYPE = KEYS["!doctype"]
20
+ XML_DECLARATION = KEYS["?xml"]
21
+ P = KEYS["p"]
22
+ BOM = chr(0xFEFF)
23
+
24
+ # Nesting deeper than this is flattened, as browsers do (Blink: 512).
25
+ MAX_DEPTH = 512
26
+ # The parts scanned again with the optional-end-tag rules may add up to this many times the source length.
27
+ REPAIR_BUDGET_FACTOR = 8
28
+
29
+ # "</script", "</style", ... by tag key, for the elements whose content is raw text (HTML5 RAWTEXT / RCDATA).
30
+ RAW_TEXT_ENDS = [None] * len(NAMES)
31
+ for _n in ("script", "style", "textarea", "title", "xmp", "iframe", "noembed", "noframes"):
32
+ RAW_TEXT_ENDS[KEYS[_n]] = "</" + _n
33
+
34
+ _END_PATTERNS = {}
35
+
36
+
37
+ def end_pattern(end_tag):
38
+ """A case-insensitive (ASCII) search for an end tag such as "</script"."""
39
+ p = _END_PATTERNS.get(end_tag)
40
+ if p is None:
41
+ p = re.compile(re.escape(end_tag), re.IGNORECASE | re.ASCII)
42
+ _END_PATTERNS[end_tag] = p
43
+ return p
44
+
45
+
46
+ # HTML5 void elements (incl. obsolete ones the parsing rules also treat as void) and declarations.
47
+ VOID_NAMES = ("area base br col embed hr img input link meta source track wbr basefont bgsound frame keygen param "
48
+ "isindex !doctype ?xml").split()
49
+ # Elements that limit the scope of the rules (HTML5 "has an element in scope").
50
+ SCOPE_STOPS = "html body table caption td th template object applet marquee button".split()
51
+ # <p> is ended by the start of a block element (HTML5 "close a p element").
52
+ CLOSES_P_NAMES = ("address article aside blockquote center details dialog dir div dl fieldset figcaption figure "
53
+ "footer form h1 h2 h3 h4 h5 h6 header hgroup hr listing main menu nav ol p pre search section "
54
+ "summary table ul li dd dt plaintext xmp").split()
55
+
56
+ F_VOID = 1
57
+ F_CLOSES_P = 2
58
+ F_RULE = 4
59
+ F_RAW_TEXT = 8
60
+
61
+
62
+ class Rule:
63
+ """t (targets) and s (stops) as flags by tag key, k: the target keys."""
64
+
65
+ __slots__ = ("t", "s", "k")
66
+
67
+ def __init__(self, targets, *stops):
68
+ t = [False] * len(NAMES)
69
+ s = [False] * len(NAMES)
70
+ for n in targets:
71
+ t[KEYS[n]] = True
72
+ for lst in stops:
73
+ for n in lst:
74
+ s[KEYS[n]] = True
75
+ self.t = t
76
+ self.s = s
77
+ self.k = tuple(KEYS[n] for n in targets)
78
+
79
+
80
+ P_RULE = Rule(["p"], SCOPE_STOPS)
81
+
82
+ RULES = [None] * len(NAMES)
83
+ RULES[KEYS["li"]] = Rule(["li"], SCOPE_STOPS, ["ul", "ol", "menu"])
84
+ RULES[KEYS["dt"]] = RULES[KEYS["dd"]] = Rule(["dt", "dd"], SCOPE_STOPS, ["dl"])
85
+ _table_stops = ["html", "body", "template", "table"]
86
+ RULES[KEYS["td"]] = RULES[KEYS["th"]] = Rule(["td", "th"], _table_stops, ["tr", "tbody", "thead", "tfoot"])
87
+ RULES[KEYS["tr"]] = Rule(["tr", "td", "th"], _table_stops, ["tbody", "thead", "tfoot"])
88
+ _section = Rule(["tbody", "thead", "tfoot", "tr", "td", "th", "caption"], _table_stops)
89
+ RULES[KEYS["tbody"]] = RULES[KEYS["thead"]] = RULES[KEYS["tfoot"]] = _section
90
+ _select_stops = ["html", "body", "template", "select", "datalist"]
91
+ RULES[KEYS["option"]] = Rule(["option"], _select_stops, ["optgroup"])
92
+ RULES[KEYS["optgroup"]] = Rule(["option", "optgroup"], _select_stops)
93
+
94
+ FLAGS = [0] * len(NAMES)
95
+ for _n in VOID_NAMES:
96
+ FLAGS[KEYS[_n]] |= F_VOID
97
+ for _n in CLOSES_P_NAMES:
98
+ FLAGS[KEYS[_n]] |= F_CLOSES_P
99
+ for _k, _e in enumerate(RAW_TEXT_ENDS):
100
+ if _e:
101
+ FLAGS[_k] |= F_RAW_TEXT
102
+ for _k, _r in enumerate(RULES):
103
+ if _r:
104
+ FLAGS[_k] |= F_RULE
105
+
106
+ IS_RULE_TARGET = [False] * len(NAMES)
107
+ for _r in RULES:
108
+ if _r:
109
+ for _k in _r.k:
110
+ IS_RULE_TARGET[_k] = True
111
+
112
+ # Inside a tag only '<', '=' and '>' matter; a quoted value after '=' may contain '<' and '>' (data-x="<p>").
113
+ # Nothing follows it in a match, so the greedy quantifiers never backtrack and act as possessive ones.
114
+ TAG_BODY = re.compile(r"""(?:[^<>=]+|=[\t-\r ]*(?:"[^"]*"|'[^']*')?)*""")
115
+ # A tag name ends at whitespace or '/' (<br/> is "br", as <br /> is). On the way to the end of the tag it is read up
116
+ # to '<', '=' or '>' too (TAG_BODY reads what follows); when it runs into '=' it is read char by char instead.
117
+ NAME_PART = re.compile(r"[^\t-\r /<=>]*")
118
+ # NAME_PART and TAG_BODY in one match (the fast scan)
119
+ OPEN_TAG = re.compile(r"""([^\t-\r /<=>]*)(?:[^<>=]+|=[\t-\r ]*(?:"[^"]*"|'[^']*')?)*""")
120
+
121
+
122
+ def lookup_tag(name):
123
+ """Finds a tag name case-insensitively (ASCII); -1 for a name that is not a known tag."""
124
+ if not name or len(name) > 32:
125
+ return -1
126
+ k = KEYS.get(name)
127
+ if k is None:
128
+ k = KEYS.get(ascii_lower(name), -1)
129
+ return k
130
+
131
+
132
+ def is_raw_text(key):
133
+ """Whether the content of the tag is raw text (script, style, textarea, title, ...)."""
134
+ return key >= 0 and RAW_TEXT_ENDS[key] is not None
135
+
136
+
137
+ def is_markup_start(s, i, limit):
138
+ """As in HTML5, '<' starts markup only before an ASCII letter, '/', '!' or '?'."""
139
+ if i + 1 >= limit:
140
+ return False
141
+ c = s[i + 1]
142
+ return ("a" <= c <= "z") or ("A" <= c <= "Z") or c == "/" or c == "!" or c == "?"
143
+
144
+
145
+ def find_raw_text_end(s, start, end_tag, limit):
146
+ """The position of the '<' of the end tag (e.g. "</script") in s[start:limit], or limit if there is none."""
147
+ n = len(end_tag)
148
+ search = end_pattern(end_tag).search
149
+ while start < limit:
150
+ m = search(s, start)
151
+ if m is None:
152
+ break
153
+ pos = m.start()
154
+ if pos + n > limit:
155
+ break
156
+ after = pos + n
157
+ # "</script>", "</script >", "</script/>" end it; "</scripts" does not.
158
+ if after >= limit:
159
+ return pos
160
+ c = s[after]
161
+ if c == ">" or c == "/" or c in ASCII_SPACE:
162
+ return pos
163
+ start = after
164
+ return limit
165
+
166
+
167
+ def find_comment_end(s, content_start, limit):
168
+ """The position of the '>' that ends a comment whose content starts at content_start (in s[:limit]).
169
+
170
+ As in HTML5: "<!-->" and "<!--->" are empty comments, "-->" or "--!>" end a comment, otherwise it runs to the end.
171
+ """
172
+ if content_start < limit and s[content_start] == ">":
173
+ return content_start
174
+ if content_start + 1 < limit and s[content_start] == "-" and s[content_start + 1] == ">":
175
+ return content_start + 1
176
+ start = content_start
177
+ while start < limit:
178
+ dashes = s.find("--", start)
179
+ if dashes < 0 or dashes + 2 > limit:
180
+ break
181
+ pos = dashes + 2
182
+ if pos < limit and s[pos] == ">":
183
+ return pos
184
+ if pos + 1 < limit and s[pos] == "!" and s[pos + 1] == ">":
185
+ return pos + 1
186
+ start = dashes + 1
187
+ return limit - 1
188
+
189
+
190
+ def is_xml(src):
191
+ """An XML document (RSS, Atom, sitemap, XHTML) is parsed by XML rules: elements end only with an end tag or "/>"."""
192
+ i = 0
193
+ n = len(src)
194
+ while i < n and (src[i] in ASCII_SPACE or src[i] == BOM):
195
+ i += 1
196
+ return n - i >= 5 and src[i] == "<" and src[i + 1] == "?" and ascii_lower(src[i:i + 5]) == "<?xml"
197
+
198
+
199
+ def parse_tags(src):
200
+ """Parses src into the flat tag list: the tags in document order as parallel lists
201
+ (keys, custom names, parents, depths, starts, ends, inner starts, inner ends)."""
202
+ if "<" not in src:
203
+ return [], [], [], [], [], [], [], []
204
+ st = _Parser(src)
205
+ st.scan_fast()
206
+ return st.finish()
207
+
208
+
209
+ class _Parser:
210
+ def __init__(self, src):
211
+ self.src = src
212
+ self.is_xml = is_xml(src)
213
+ self.budget = REPAIR_BUDGET_FACTOR * len(src) # chars that may still be scanned again by repair
214
+ self.keys = [] # tag key, or -1 for a custom name
215
+ self.customs = [] # the lower-case name of a tag that is not a known one, "" for the others
216
+ self.parents = [] # the index of the enclosing tag, -1 at the top level
217
+ self.starts = [] # the opening '<'
218
+ self.ends = [] # the last '>': of the end tag, or of the opening tag
219
+ self.inner_starts = [] # right after the opening tag
220
+ self.inner_ends = [] # the last char before the end tag; 0 when the tag is not closed
221
+ self.depths = []
222
+ self.prev_open = [] # the open tags as a linked list in document order
223
+ self.self_closed = [] # <x/>: stays in the open list (a later </x> ends it), but nests nothing
224
+ self.stack = [] # indexes of the open tags that contain the following ones
225
+ self.open_by_key = [0] * len(NAMES) # per tag, how many are on the stack: kept only in rules mode
226
+ self.open_in_list = [0] * len(NAMES) # per tag, how many are in the open list
227
+ self.custom_open = {} # custom names (lower case): how many such tags are in the open list
228
+ self.count = 0
229
+ self.stack_top = 0
230
+ self.last_open = -1
231
+ self.open_p = 0 # <p> tags on the stack, kept in both modes
232
+ self.tag_started = False # scan results
233
+ self.tag_start = 0
234
+
235
+ def finish(self):
236
+ """The end of the document ends every tag still open (a truncated page keeps its text), right before an
237
+ unfinished tag if the source stops inside one."""
238
+ region_end = self.tag_start - 1 if self.tag_started else len(self.src)
239
+ if not self.is_xml:
240
+ s = self.first_misnested(0, self.stack_top)
241
+ if s >= 0:
242
+ self.repair(s, region_end)
243
+ self.truncate_open_list(0, region_end - 1)
244
+ count = self.count
245
+ out = [self.keys, self.customs, self.parents, self.depths, self.starts, self.ends, self.inner_starts,
246
+ self.inner_ends]
247
+ for a in out:
248
+ del a[count:]
249
+ inner_starts = self.inner_starts
250
+ for k, e in enumerate(self.inner_ends):
251
+ if e == 0:
252
+ inner_starts[k] = 0 # not closed: no body
253
+ return out
254
+
255
+ def scan_fast(self):
256
+ """The first scan of the whole page in fast mode: what scan(False, 0, size) does, with the common cases inline
257
+ and the state in local variables. Whatever the inline code does not handle (comments, custom names, the
258
+ optional-end-tag rules, a misnested end tag, an unfinished tag) is handed to the general code with the state
259
+ written back."""
260
+ src = self.src
261
+ to = len(src)
262
+ keys = self.keys
263
+ customs = self.customs
264
+ parents = self.parents
265
+ depths = self.depths
266
+ starts = self.starts
267
+ ends = self.ends
268
+ inner_starts = self.inner_starts
269
+ inner_ends = self.inner_ends
270
+ prev_open = self.prev_open
271
+ self_closed = self.self_closed
272
+ stack = self.stack
273
+ open_in_list = self.open_in_list
274
+ xml = self.is_xml
275
+ tag_body = TAG_BODY.match
276
+ open_tag = OPEN_TAG.match
277
+ find = src.find
278
+ get_key = KEYS.get
279
+ flags_of = FLAGS
280
+ rules_of = RULES
281
+ raw_ends = RAW_TEXT_ENDS
282
+ count = 0
283
+ stack_top = 0
284
+ last_open = -1
285
+ open_p = 0
286
+ i = 0
287
+ while i < to:
288
+ lt = find("<", i)
289
+ if lt < 0:
290
+ break
291
+ i = lt
292
+ if i + 1 >= to:
293
+ break
294
+ c = src[i + 1]
295
+ tag_start = i + 1
296
+ if c == "/":
297
+ p = tag_body(src, tag_start).end()
298
+ if p < to and src[p] == ">":
299
+ key = get_key(src[tag_start + 1:p])
300
+ j = last_open
301
+ if key is not None and j >= 0 and keys[j] == key and stack_top > 0 and stack[stack_top - 1] == j:
302
+ # the end tag of the innermost open tag
303
+ inner_ends[j] = tag_start - 2
304
+ ends[j] = p
305
+ stack_top -= 1
306
+ if key == P:
307
+ open_p -= 1
308
+ open_in_list[key] -= 1
309
+ last_open = prev_open[j]
310
+ i = p + 1
311
+ continue
312
+ else:
313
+ if not (("a" <= c <= "z") or ("A" <= c <= "Z") or c == "!" or c == "?"):
314
+ i += 1
315
+ continue
316
+ if c != "!":
317
+ # the name and the rest of the tag in one match
318
+ m = open_tag(src, tag_start)
319
+ p = m.end()
320
+ name = m.group(1)
321
+ if src.startswith("=", m.end(1)):
322
+ name = None
323
+ if name is not None and p < to and src[p] == ">" and src[p - 1] < "\x80":
324
+ key = get_key(name)
325
+ if key is None and len(name) <= 32:
326
+ key = get_key(ascii_lower(name))
327
+ if key is not None:
328
+ flags = 0 if xml else flags_of[key]
329
+ if (flags & F_RULE == 0 or stack_top == 0 or keys[stack[stack_top - 1]] < 0
330
+ or not rules_of[key].t[keys[stack[stack_top - 1]]]) \
331
+ and (open_p == 0 or flags & F_CLOSES_P == 0):
332
+ # a known tag that ends no open tag
333
+ if stack_top > 0:
334
+ parent = stack[stack_top - 1]
335
+ depth = depths[parent] + 1
336
+ else:
337
+ parent = -1
338
+ depth = 0
339
+ customs.append("")
340
+ parents.append(parent)
341
+ starts.append(i)
342
+ ends.append(p)
343
+ inner_starts.append(p + 1)
344
+ inner_ends.append(0)
345
+ keys.append(key)
346
+ depths.append(depth)
347
+ if flags & F_VOID:
348
+ prev_open.append(-1)
349
+ self_closed.append(False)
350
+ count += 1
351
+ i = p + 1
352
+ continue
353
+ last = src[p - 1]
354
+ self_closing = last == "/" or (last <= " " and self._self_closing(tag_start, p))
355
+ self_closed.append(self_closing)
356
+ prev_open.append(last_open)
357
+ last_open = count
358
+ open_in_list[key] += 1
359
+ count += 1
360
+ i = p + 1
361
+ if self_closing:
362
+ continue
363
+ if stack_top < MAX_DEPTH:
364
+ if stack_top < len(stack):
365
+ stack[stack_top] = count - 1
366
+ else:
367
+ stack.append(count - 1)
368
+ stack_top += 1
369
+ if key == P:
370
+ open_p += 1
371
+ # <script>, <style>, ...: continue right before the matching end tag
372
+ if flags & F_RAW_TEXT:
373
+ i = find_raw_text_end(src, p + 1, raw_ends[key], to)
374
+ continue
375
+ # anything else: the general code, with the state written back
376
+ self.count = count
377
+ self.stack_top = stack_top
378
+ self.last_open = last_open
379
+ self.open_p = open_p
380
+ stop = self.scan_one(i, to)
381
+ count = self.count
382
+ stack_top = self.stack_top
383
+ last_open = self.last_open
384
+ open_p = self.open_p
385
+ if stop < 0:
386
+ self.tag_started = True
387
+ self.tag_start = i + 1
388
+ return
389
+ i = stop
390
+ self.count = count
391
+ self.stack_top = stack_top
392
+ self.last_open = last_open
393
+ self.open_p = open_p
394
+ self.tag_started = False
395
+
396
+ def scan(self, rules, start, to):
397
+ """Parses src[start:to] (start is outside a tag), adding tags at count. Tags still open at to stay on the
398
+ stack."""
399
+ src = self.src
400
+ tag_started = False
401
+ tag_start = 0
402
+ i = start
403
+ while i < to:
404
+ # Outside a tag only '<' matters: jump to it.
405
+ lt = src.find("<", i, to)
406
+ if lt < 0:
407
+ break
408
+ i = lt
409
+ if i + 1 >= to:
410
+ break
411
+ c = src[i + 1]
412
+ tag_start = i + 1
413
+ if c == "/":
414
+ # an end tag; inside a tag only '<', '=' and '>' matter
415
+ p = TAG_BODY.match(src, tag_start).end()
416
+ if p >= to:
417
+ p = self.scan_tag_slowly(tag_start, to)
418
+ if p < 0:
419
+ tag_started = True
420
+ break
421
+ if src[p] == "<":
422
+ # An unfinished tag: start over at this '<' (it may also start a comment).
423
+ i = p
424
+ continue
425
+ self.end_tag(rules, tag_start, p)
426
+ i = p + 1
427
+ continue
428
+
429
+ # As in HTML5, '<' starts markup only before a letter, '/', '!' or '?': "a < b" is text.
430
+ if not (("a" <= c <= "z") or ("A" <= c <= "Z") or c == "!" or c == "?"):
431
+ i += 1
432
+ continue
433
+ if c == "!" and i + 4 <= to and src[i + 2] == "-" and src[i + 3] == "-":
434
+ # <!-- ... -->: one "!--" tag; its content is not markup, it does not nest anything.
435
+ i = self.comment(i, to) + 1
436
+ continue
437
+
438
+ m = NAME_PART.match(src, tag_start)
439
+ p = m.end()
440
+ name = m.group()
441
+ if p < len(src) and src[p] == "=":
442
+ name = None # the name runs into '=': it is read char by char
443
+ # a tag without attributes ends right after its name
444
+ if p < len(src) and src[p] != ">":
445
+ p = TAG_BODY.match(src, p).end()
446
+ if p >= to:
447
+ p = self.scan_tag_slowly(tag_start, to)
448
+ if p < 0:
449
+ tag_started = True
450
+ break
451
+ if src[p] == "<":
452
+ i = p
453
+ continue
454
+ i = self.start_tag(rules, tag_start, p, to, c, name) + 1
455
+ self.tag_started = tag_started
456
+ self.tag_start = tag_start
457
+
458
+ def scan_one(self, i, to):
459
+ """One step of scan(False, ...) at the '<' at i: the position to go on from, or -1 for a tag that does not
460
+ end."""
461
+ src = self.src
462
+ c = src[i + 1]
463
+ tag_start = i + 1
464
+ if c == "/":
465
+ p = TAG_BODY.match(src, tag_start).end()
466
+ if p >= to:
467
+ p = self.scan_tag_slowly(tag_start, to)
468
+ if p < 0:
469
+ return -1
470
+ if src[p] == "<":
471
+ return p
472
+ self.end_tag(False, tag_start, p)
473
+ return p + 1
474
+ if not (("a" <= c <= "z") or ("A" <= c <= "Z") or c == "!" or c == "?"):
475
+ return i + 1
476
+ if c == "!" and i + 4 <= to and src[i + 2] == "-" and src[i + 3] == "-":
477
+ return self.comment(i, to) + 1
478
+ m = NAME_PART.match(src, tag_start)
479
+ p = m.end()
480
+ name = m.group()
481
+ if p < len(src) and src[p] == "=":
482
+ name = None
483
+ if p < len(src) and src[p] != ">":
484
+ p = TAG_BODY.match(src, p).end()
485
+ if p >= to:
486
+ p = self.scan_tag_slowly(tag_start, to)
487
+ if p < 0:
488
+ return -1
489
+ if src[p] == "<":
490
+ return p
491
+ return self.start_tag(False, tag_start, p, to, c, name) + 1
492
+
493
+ def _self_closing(self, tag_start, gt):
494
+ """Whether the opening tag that ends at gt (with whitespace before '>') ends with "/"."""
495
+ src = self.src
496
+ e = gt
497
+ while e > tag_start and src[e - 1] in ASCII_SPACE:
498
+ e -= 1
499
+ return e > tag_start and src[e - 1] == "/"
500
+
501
+ def scan_tag_slowly(self, i, to):
502
+ """The inside of a tag from i, as the fast scan reads it but within to: the position of the '<' or '>' that
503
+ ends it, or -1 when the tag does not end before to."""
504
+ src = self.src
505
+ while i < to:
506
+ c = src[i]
507
+ if c == "<" or c == ">":
508
+ return i
509
+ if c == "=":
510
+ q = i + 1
511
+ while q < to and src[q] in ASCII_SPACE:
512
+ q += 1
513
+ if q < to:
514
+ quote = src[q]
515
+ if quote == '"' or quote == "'":
516
+ close = src.find(quote, q + 1)
517
+ if 0 <= close < to:
518
+ i = close
519
+ i += 1
520
+ return -1
521
+
522
+ def _add(self, key, custom, parent, start, end, depth, prev_open, self_closed):
523
+ """Appends a tag; the lists always end at count (repair cuts them back)."""
524
+ self.keys.append(key)
525
+ self.customs.append(custom)
526
+ self.parents.append(parent)
527
+ self.starts.append(start)
528
+ self.ends.append(end)
529
+ self.inner_starts.append(end + 1)
530
+ self.inner_ends.append(0)
531
+ self.depths.append(depth)
532
+ self.prev_open.append(prev_open)
533
+ self.self_closed.append(self_closed)
534
+
535
+ def comment(self, i, to):
536
+ """Adds the comment that starts at i; returns the position of its '>'."""
537
+ comment_end = find_comment_end(self.src, i + 4, to)
538
+ parent = -1
539
+ depth = 0
540
+ if self.stack_top > 0:
541
+ parent = self.stack[self.stack_top - 1]
542
+ depth = self.depths[parent] + 1
543
+ k = self.count
544
+ self._add(COMMENT, "", parent, i, comment_end, depth, -1, False)
545
+ self.count = k + 1
546
+ return comment_end
547
+
548
+ def end_tag(self, rules, tag_start, gt):
549
+ """Matches the end tag src[tag_start-1 .. gt] to the innermost open tag with that name."""
550
+ src = self.src
551
+ name = src[tag_start + 1:gt]
552
+ # Nearly always the name as it stands is a known tag name.
553
+ key = KEYS.get(name)
554
+ if key is None:
555
+ name = name.strip(SPACE)
556
+ if not name:
557
+ return
558
+ # Resolve the name once, then compare ints instead of a lookup per candidate.
559
+ key = KEYS.get(name)
560
+ if key is None:
561
+ key = KEYS.get(ascii_lower(name), -1) if len(name) <= 32 else -1
562
+
563
+ # Well-formed markup: the end tag closes the innermost open tag, which is on top of the stack. What
564
+ # match_end and truncate_open_list do for it, without the loops.
565
+ j = self.last_open
566
+ if key >= 0 and j >= 0 and self.keys[j] == key and self.stack_top > 0 and self.stack[self.stack_top - 1] == j:
567
+ self.inner_ends[j] = tag_start - 2
568
+ self.ends[j] = gt
569
+ self.stack_top -= 1
570
+ if key == P:
571
+ self.open_p -= 1
572
+ if rules:
573
+ self.open_by_key[key] -= 1
574
+ self.open_in_list[key] -= 1
575
+ self.last_open = self.prev_open[j]
576
+ return
577
+ # The rules may have ended the matched tag itself (<li><div><li>x</div>: the second <li> ends the first one
578
+ # together with its <div>, as in a browser): the end tag is matched again, to what is open now.
579
+ # Once: the part is not scanned a second time.
580
+ if not self.match_end(rules, key, name, tag_start, gt, True):
581
+ return
582
+ self.match_end(rules, key, name, tag_start, gt, False)
583
+
584
+ def match_end(self, rules, key, name, tag_start, gt, can_repair):
585
+ """Ends the innermost open tag with the end tag's name. Returns True when it scanned a part again with the
586
+ optional-end-tag rules instead, and the end tag is to be matched once more."""
587
+ # A name that no open tag has is a stray end tag: it is skipped without scanning the open tags.
588
+ closing = None
589
+ if key >= 0:
590
+ if self.open_in_list[key] <= 0:
591
+ return False
592
+ else:
593
+ if not self.custom_open:
594
+ return False
595
+ closing = lower(name)
596
+ if self.custom_open.get(closing, 0) <= 0:
597
+ return False
598
+
599
+ keys = self.keys
600
+ prev_open = self.prev_open
601
+ j = self.last_open
602
+ while j >= 0:
603
+ if key >= 0:
604
+ if keys[j] != key:
605
+ j = prev_open[j]
606
+ continue
607
+ elif keys[j] >= 0 or self.customs[j] != closing:
608
+ j = prev_open[j]
609
+ continue
610
+
611
+ inner_end = tag_start - 2
612
+
613
+ # Tags left open inside it: if the optional-end-tag rules apply to one of them,
614
+ # scan its part again with the rules (fast mode only; rules mode has applied them).
615
+ stack = self.stack
616
+ if can_repair and not rules and not self.is_xml and self.stack_top > 0 and stack[self.stack_top - 1] > j:
617
+ above = self.stack_top - 1
618
+ while above > 0 and stack[above - 1] > j:
619
+ above -= 1
620
+ target = self.first_misnested(above, self.stack_top)
621
+ if target >= 0:
622
+ self.repair(target, tag_start - 1)
623
+ return True
624
+
625
+ self.inner_ends[j] = inner_end
626
+ self.ends[j] = gt
627
+
628
+ while self.stack_top > 0 and stack[self.stack_top - 1] >= j:
629
+ self.stack_top -= 1
630
+ popped = keys[stack[self.stack_top]]
631
+ if popped == P:
632
+ self.open_p -= 1
633
+ if rules and popped >= 0:
634
+ self.open_by_key[popped] -= 1
635
+
636
+ # It leaves the open list with everything opened after it; the tags opened inside it and
637
+ # never closed end right before this end tag, as in a browser.
638
+ self.truncate_open_list(j, inner_end)
639
+ return False
640
+ return False
641
+
642
+ def start_tag(self, rules, tag_start, gt, to, first, name):
643
+ """Adds the tag src[tag_start-1 .. gt]; returns the position to continue from (gt, or before a raw-text end
644
+ tag). name: the name as the scan read it, None when it runs into '='."""
645
+ src = self.src
646
+ # The name in the inside of the opening tag (trimmed; "!--" for a comment). It starts at tag_start: the first
647
+ # char (first) is a letter, '!' or '?'.
648
+ last = src[gt - 1]
649
+ if last >= "\x80" or first == "!" or name is None:
650
+ end = gt
651
+ if last >= "\x80":
652
+ while end > tag_start and src[end - 1] in SPACE:
653
+ end -= 1
654
+ if first == "!" and tag_start + 3 <= end and src[tag_start + 1] == "-" and src[tag_start + 2] == "-":
655
+ length = 3
656
+ elif name is None:
657
+ # The name ends at whitespace or '/': <br/> is "br", as <br /> is.
658
+ e = tag_start
659
+ while e < end and src[e] not in ASCII_SPACE and src[e] != "/":
660
+ e += 1
661
+ length = e - tag_start
662
+ else:
663
+ length = len(name)
664
+ if tag_start + length > end:
665
+ length = end - tag_start
666
+ name = src[tag_start:tag_start + length]
667
+ else:
668
+ length = len(name)
669
+ key = -1
670
+ custom_name = ""
671
+ if length > 0:
672
+ key = KEYS.get(name)
673
+ if key is None:
674
+ key = KEYS.get(ascii_lower(name), -1) if length <= 32 else -1
675
+ if key < 0:
676
+ # Custom names (svg elements, web components, ...): the lower-case name.
677
+ custom_name = lower(name)
678
+ self.custom_open.setdefault(custom_name, 0)
679
+ flags = FLAGS[key] if key >= 0 and not self.is_xml else 0
680
+
681
+ if flags:
682
+ # Optional end tags (HTML5): <li> ends an open <li>, <td> an open <td>, ...
683
+ if flags & F_RULE:
684
+ rule = RULES[key]
685
+ if rules:
686
+ # a target is open (in well-formed markup it is not)
687
+ if any(self.open_by_key[t] > 0 for t in rule.k):
688
+ self.close_in_scope(rule, tag_start - 2, True)
689
+ elif self.stack_top > 0:
690
+ # The common case of a left-out end tag is caught right here: the innermost open tag is a target.
691
+ top = self.keys[self.stack[self.stack_top - 1]]
692
+ if top >= 0 and rule.t[top]:
693
+ self.apply_rule_in_fast_mode(rule, tag_start)
694
+
695
+ # A block element ends an open <p>. Usually no <p> is open and this is one compare.
696
+ if self.open_p > 0 and flags & F_CLOSES_P:
697
+ if rules:
698
+ self.close_in_scope(P_RULE, tag_start - 2, True)
699
+ else:
700
+ self.apply_rule_in_fast_mode(P_RULE, tag_start)
701
+
702
+ # the innermost open tag is the parent
703
+ parent = -1
704
+ depth = 0
705
+ if self.stack_top > 0:
706
+ parent = self.stack[self.stack_top - 1]
707
+ depth = self.depths[parent] + 1
708
+
709
+ index = self.count
710
+ # Void elements (<br>, <img>, <meta>, ...) and declarations never contain anything.
711
+ is_void = flags & F_VOID != 0
712
+ self_closing = is_void
713
+ if not self_closing:
714
+ if last == "/":
715
+ self_closing = True
716
+ elif last <= " ":
717
+ self_closing = self._self_closing(tag_start, gt)
718
+
719
+ if is_void:
720
+ self._add(key, custom_name, parent, tag_start - 1, gt, depth, -1, False)
721
+ else:
722
+ self._add(key, custom_name, parent, tag_start - 1, gt, depth, self.last_open, self_closing)
723
+ self.last_open = index
724
+ if key >= 0:
725
+ self.open_in_list[key] += 1
726
+ elif custom_name:
727
+ self.custom_open[custom_name] += 1
728
+ self.count = index + 1
729
+
730
+ if not self_closing:
731
+ # Deeper than MAX_DEPTH the tag is not nested: it stays in the open list for its end tag.
732
+ if self.stack_top < MAX_DEPTH:
733
+ if self.stack_top < len(self.stack):
734
+ self.stack[self.stack_top] = index
735
+ else:
736
+ self.stack.append(index)
737
+ self.stack_top += 1
738
+ if key == P:
739
+ self.open_p += 1
740
+ if rules and key >= 0:
741
+ self.open_by_key[key] += 1
742
+
743
+ # <script>, <style>, <textarea>, <title>, ...: the content is text, not markup;
744
+ # continue right before the matching end tag.
745
+ if flags & F_RAW_TEXT:
746
+ return find_raw_text_end(src, gt + 1, RAW_TEXT_ENDS[key], to) - 1
747
+ return gt
748
+
749
+ # ----- implied end tags -----
750
+
751
+ def first_misnested(self, start, to):
752
+ """Among the tags left open at stack positions [start, to), the outermost one that a rule should have ended,
753
+ or -1."""
754
+ keys = self.keys
755
+ stack = self.stack
756
+ s = start
757
+ while s < to - 1:
758
+ key = keys[stack[s]]
759
+ if key >= 0 and IS_RULE_TARGET[key]:
760
+ inner = s + 1
761
+ while inner < to:
762
+ inner_key = keys[stack[inner]]
763
+ if inner_key >= 0:
764
+ r = RULES[inner_key]
765
+ if r is not None and r.t[key]:
766
+ return s
767
+ inner += 1
768
+ s += 1
769
+ return -1
770
+
771
+ def repair(self, stack_pos, region_end):
772
+ """Scans again with the optional-end-tag rules from the tag at stack position stack_pos up to region_end:
773
+ the tags from it on are replaced, the ones before it are kept."""
774
+ first = self.stack[stack_pos]
775
+ start = self.starts[first]
776
+
777
+ # Past the budget the part is left as it is, so that parsing stays linear on any markup.
778
+ cost = region_end - start
779
+ if cost > self.budget:
780
+ return
781
+ self.budget -= cost
782
+
783
+ self.truncate_open_list(first, -1)
784
+ self.stack_top = stack_pos
785
+ self.count = first
786
+ for a in (self.keys, self.customs, self.parents, self.starts, self.ends, self.inner_starts, self.inner_ends,
787
+ self.depths, self.prev_open, self.self_closed):
788
+ del a[first:]
789
+
790
+ obk = self.open_by_key
791
+ for k in range(len(obk)):
792
+ obk[k] = 0
793
+ self.open_p = 0
794
+ for s in range(stack_pos):
795
+ key = self.keys[self.stack[s]]
796
+ if key >= 0:
797
+ obk[key] += 1
798
+ if key == P:
799
+ self.open_p += 1
800
+
801
+ self.scan(True, start, region_end)
802
+
803
+ def truncate_open_list(self, first, end):
804
+ """Removes the tags from index first on from the open list (they are its tail).
805
+ With end >= 0 the ones that have no end tag of their own end there."""
806
+ k = self.last_open
807
+ keys = self.keys
808
+ while k >= first:
809
+ key = keys[k]
810
+ if key >= 0:
811
+ self.open_in_list[key] -= 1
812
+ else:
813
+ custom = self.customs[k]
814
+ if custom and custom in self.custom_open:
815
+ self.custom_open[custom] -= 1
816
+ if end >= 0 and not self.self_closed[k] and self.inner_ends[k] == 0:
817
+ self.inner_ends[k] = end
818
+ self.ends[k] = end
819
+ k = self.prev_open[k]
820
+ self.last_open = k
821
+
822
+ def scope_start(self, rule):
823
+ """The stack position of the outermost target of the rule below the first stop tag, or -1."""
824
+ start = -1
825
+ s = self.stack_top - 1
826
+ keys = self.keys
827
+ stack = self.stack
828
+ while s >= 0:
829
+ k = keys[stack[s]]
830
+ if k >= 0:
831
+ if rule.t[k]:
832
+ start = s
833
+ elif rule.s[k]:
834
+ break
835
+ s -= 1
836
+ return start
837
+
838
+ def apply_rule_in_fast_mode(self, rule, tag_start):
839
+ """Applies the rule for the tag that starts at tag_start. If a tag it ends was itself misnested, that part
840
+ is scanned again with the rules first."""
841
+ start = self.scope_start(rule)
842
+ if start < 0:
843
+ return
844
+ m = self.first_misnested(start, self.stack_top)
845
+ if m >= 0:
846
+ self.repair(m, tag_start - 1)
847
+ start = self.scope_start(rule) # the part scanned again changed the stack
848
+ self.close_from(start, tag_start - 2, False)
849
+
850
+ def close_in_scope(self, rule, end, by_key):
851
+ """Ends the outermost target found below the first stop tag together with everything opened inside it."""
852
+ self.close_from(self.scope_start(rule), end, by_key)
853
+
854
+ def close_from(self, start, end, by_key):
855
+ """Ends the tag at stack position start (-1: none) together with everything opened inside it."""
856
+ if start < 0:
857
+ return
858
+ first = self.stack[start]
859
+ while self.stack_top > start:
860
+ self.stack_top -= 1
861
+ key = self.keys[self.stack[self.stack_top]]
862
+ if key == P:
863
+ self.open_p -= 1
864
+ if by_key and key >= 0:
865
+ self.open_by_key[key] -= 1
866
+ self.truncate_open_list(first, end)