fasthtmlparser-pure 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fasthtmlparser_pure/__init__.py +41 -0
- fasthtmlparser_pure/_api.py +636 -0
- fasthtmlparser_pure/_api.pyi +342 -0
- fasthtmlparser_pure/_attrs.py +270 -0
- fasthtmlparser_pure/_chars.py +124 -0
- fasthtmlparser_pure/_css.py +861 -0
- fasthtmlparser_pure/_decode.py +144 -0
- fasthtmlparser_pure/_links.py +483 -0
- fasthtmlparser_pure/_parser.py +866 -0
- fasthtmlparser_pure/_query.py +128 -0
- fasthtmlparser_pure/_tables.py +2494 -0
- fasthtmlparser_pure/_text.py +257 -0
- fasthtmlparser_pure/_xpath.py +1594 -0
- fasthtmlparser_pure/py.typed +0 -0
- fasthtmlparser_pure-0.1.0.dist-info/METADATA +166 -0
- fasthtmlparser_pure-0.1.0.dist-info/RECORD +19 -0
- fasthtmlparser_pure-0.1.0.dist-info/WHEEL +5 -0
- fasthtmlparser_pure-0.1.0.dist-info/licenses/LICENSE.md +79 -0
- fasthtmlparser_pure-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
"""Visible text: tags, comments and invisible content removed, whitespace collapsed, entities decoded; and the text
|
|
2
|
+
with the visible line structure kept (Node.text_line_breaks)."""
|
|
3
|
+
|
|
4
|
+
import re
|
|
5
|
+
|
|
6
|
+
from . import _decode as dec
|
|
7
|
+
from ._chars import ASCII_SPACE, HAS_SPACE, SPACE_RUNS, ascii_lower
|
|
8
|
+
from ._parser import KEYS, NAMES, find_comment_end, find_raw_text_end, lookup_tag
|
|
9
|
+
|
|
10
|
+
INVISIBLE = {
|
|
11
|
+
"style": "</style", "script": "</script", "noembed": "</noembed", "noscript": "</noscript",
|
|
12
|
+
"template": "</template", "noframes": "</noframes",
|
|
13
|
+
}
|
|
14
|
+
# '<' that starts markup: before an ASCII letter, '/', '!' or '?'
|
|
15
|
+
MARKUP = re.compile(r"<[a-zA-Z/!?]")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def skip_markup(s, i):
|
|
19
|
+
"""s[i] is '<'. Returns the index of the last char of what is not text there: a comment, a tag, or a tag together
|
|
20
|
+
with invisible content (script, style, ...) up to its end tag. -1: an unterminated tag."""
|
|
21
|
+
n = len(s)
|
|
22
|
+
if i + 4 <= n and s[i + 1] == "!" and s[i + 2] == "-" and s[i + 3] == "-":
|
|
23
|
+
return find_comment_end(s, i + 4, n)
|
|
24
|
+
tag_end = s.find(">", i + 1)
|
|
25
|
+
if tag_end < 0:
|
|
26
|
+
return -1
|
|
27
|
+
end_tag = invisible_content_end(s, i + 1, tag_end)
|
|
28
|
+
if end_tag is not None:
|
|
29
|
+
return find_raw_text_end(s, tag_end + 1, end_tag, n) - 1
|
|
30
|
+
return tag_end
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def invisible_content_end(s, start, to):
|
|
34
|
+
"""For the inside of an opening tag s[start:to], the end tag ("</script") when the element's content is never
|
|
35
|
+
visible text (script, style, noscript, template, noframes, noembed), otherwise None."""
|
|
36
|
+
if start >= to:
|
|
37
|
+
return None
|
|
38
|
+
first = s[start]
|
|
39
|
+
if first not in "sStTnN":
|
|
40
|
+
return None
|
|
41
|
+
n = 1
|
|
42
|
+
while start + n < to and n <= 8:
|
|
43
|
+
c = s[start + n]
|
|
44
|
+
if not (("a" <= c <= "z") or ("A" <= c <= "Z")):
|
|
45
|
+
break
|
|
46
|
+
n += 1
|
|
47
|
+
if start + n < to:
|
|
48
|
+
c = s[start + n]
|
|
49
|
+
if c != "/" and c not in ASCII_SPACE:
|
|
50
|
+
return None
|
|
51
|
+
end = INVISIBLE.get(ascii_lower(s[start:start + n]))
|
|
52
|
+
if end is None:
|
|
53
|
+
return None
|
|
54
|
+
e = to
|
|
55
|
+
while e > start and s[e - 1] in ASCII_SPACE:
|
|
56
|
+
e -= 1
|
|
57
|
+
if s[e - 1] == "/":
|
|
58
|
+
return None # <script src="x" />: no content
|
|
59
|
+
return end
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def text_runs(s):
|
|
63
|
+
"""The text runs of s with tags, comments and invisible content removed, whitespace not yet collapsed."""
|
|
64
|
+
search = MARKUP.search
|
|
65
|
+
m = search(s)
|
|
66
|
+
if m is None:
|
|
67
|
+
return s
|
|
68
|
+
out = []
|
|
69
|
+
start = 0
|
|
70
|
+
while m is not None:
|
|
71
|
+
lt = m.start()
|
|
72
|
+
if lt > start:
|
|
73
|
+
out.append(s[start:lt])
|
|
74
|
+
last = skip_markup(s, lt)
|
|
75
|
+
if last < 0:
|
|
76
|
+
return "".join(out) # an unterminated tag: the rest is markup
|
|
77
|
+
start = last + 1
|
|
78
|
+
m = search(s, start)
|
|
79
|
+
if start < len(s):
|
|
80
|
+
out.append(s[start:])
|
|
81
|
+
return "".join(out)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def collapse(s):
|
|
85
|
+
"""Whitespace runs as one ' ', trimmed."""
|
|
86
|
+
if HAS_SPACE.search(s) is None:
|
|
87
|
+
return s
|
|
88
|
+
t = SPACE_RUNS.sub(" ", s)
|
|
89
|
+
if not t:
|
|
90
|
+
return t
|
|
91
|
+
a = 1 if t[0] == " " else 0
|
|
92
|
+
z = len(t) - (1 if t[-1] == " " else 0)
|
|
93
|
+
return t if a == 0 and z == len(t) else t[a:z]
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def clear_tags(s):
|
|
97
|
+
"""The visible text of an HTML fragment."""
|
|
98
|
+
if not s:
|
|
99
|
+
return ""
|
|
100
|
+
text = collapse(text_runs(s))
|
|
101
|
+
return dec.decode(text, False) if "&" in text else text
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def text_equals(value, expected, d, i):
|
|
105
|
+
"""Whether the visible text of the body (value) of the tag i equals expected."""
|
|
106
|
+
if not value or not expected:
|
|
107
|
+
return False
|
|
108
|
+
# whitespace is collapsed even where text keeps it (a tag without children), as the .NET version compares
|
|
109
|
+
text = collapse(text_runs(value))
|
|
110
|
+
amp = text.find("&")
|
|
111
|
+
if amp < 0:
|
|
112
|
+
return text == expected
|
|
113
|
+
# Text with entities must be decoded before comparing: the full text, when the part before it matches.
|
|
114
|
+
head = text[:amp].rstrip(" ")
|
|
115
|
+
return expected.startswith(head) and d.text_of(i) == expected
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
# ----- text with line breaks -----
|
|
119
|
+
|
|
120
|
+
BREAK_SPACE = 1 # table cell: word separator
|
|
121
|
+
BREAK_LINE = 2 # <br> and block elements (div, li, tr, ...): new line
|
|
122
|
+
BREAK_PARAGRAPH = 3 # p, h1-h6: new paragraph (blank line)
|
|
123
|
+
BREAK_SKIP_CONTENT = 4 # script, style, noscript: contents are not text
|
|
124
|
+
# inline tags (b, span, a, ...) are removed, words are not split
|
|
125
|
+
|
|
126
|
+
BR = KEYS["br"]
|
|
127
|
+
KINDS = [0] * len(NAMES)
|
|
128
|
+
for _t in "p h1 h2 h3 h4 h5 h6".split():
|
|
129
|
+
KINDS[KEYS[_t]] = BREAK_PARAGRAPH
|
|
130
|
+
for _t in ("br div li ul ol dl dt dd table caption tr thead tbody tfoot section article aside nav header footer main "
|
|
131
|
+
"address blockquote pre hr figure figcaption form fieldset legend details summary dialog option html head "
|
|
132
|
+
"body title hgroup search menu center dir listing xmp plaintext multicol frameset marquee").split():
|
|
133
|
+
KINDS[KEYS[_t]] = BREAK_LINE
|
|
134
|
+
KINDS[KEYS["td"]] = BREAK_SPACE
|
|
135
|
+
KINDS[KEYS["th"]] = BREAK_SPACE
|
|
136
|
+
for _t in "script style noscript template noframes noembed".split():
|
|
137
|
+
KINDS[KEYS[_t]] = BREAK_SKIP_CONTENT
|
|
138
|
+
|
|
139
|
+
PLAIN = re.compile(r"[^&<]*")
|
|
140
|
+
TAG_NAME_END = re.compile(r"[\t\r\n\f /]")
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
class _Lines:
|
|
144
|
+
__slots__ = ("out", "pending_breaks", "pending_space", "started")
|
|
145
|
+
|
|
146
|
+
def __init__(self):
|
|
147
|
+
self.out = []
|
|
148
|
+
self.pending_breaks = 0 # new lines requested since the last text char
|
|
149
|
+
self.pending_space = False
|
|
150
|
+
self.started = False
|
|
151
|
+
|
|
152
|
+
def chars(self, s):
|
|
153
|
+
"""Writes text, collapsing whitespace and turning requested breaks into trimmed lines."""
|
|
154
|
+
if HAS_SPACE.search(s) is None:
|
|
155
|
+
self.text(s)
|
|
156
|
+
return
|
|
157
|
+
pos = 0
|
|
158
|
+
n = len(s)
|
|
159
|
+
for m in SPACE_RUNS.finditer(s):
|
|
160
|
+
i = m.start()
|
|
161
|
+
if i > pos:
|
|
162
|
+
self.text(s[pos:i])
|
|
163
|
+
self.pending_space = True
|
|
164
|
+
pos = m.end()
|
|
165
|
+
if pos < n:
|
|
166
|
+
self.text(s[pos:])
|
|
167
|
+
|
|
168
|
+
def text(self, s):
|
|
169
|
+
if self.started:
|
|
170
|
+
# nothing before the first text char: leading breaks and spaces are trimmed
|
|
171
|
+
if self.pending_breaks > 0:
|
|
172
|
+
self.out.append("\n\n" if self.pending_breaks >= 2 else "\n") # at most one blank line
|
|
173
|
+
elif self.pending_space:
|
|
174
|
+
self.out.append(" ")
|
|
175
|
+
self.pending_breaks = 0
|
|
176
|
+
self.pending_space = False
|
|
177
|
+
if s:
|
|
178
|
+
self.out.append(s)
|
|
179
|
+
self.started = True
|
|
180
|
+
|
|
181
|
+
def brk(self, kind, known):
|
|
182
|
+
if kind == BREAK_LINE:
|
|
183
|
+
if known == BR:
|
|
184
|
+
self.pending_breaks += 1 # every <br> counts
|
|
185
|
+
elif self.pending_breaks < 1:
|
|
186
|
+
self.pending_breaks = 1
|
|
187
|
+
elif self.pending_breaks < 2:
|
|
188
|
+
self.pending_breaks = 2
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def clear_tags_line_breaks(value):
|
|
192
|
+
"""The text of an HTML fragment with the visible line structure kept. Lines are joined with "\\n"."""
|
|
193
|
+
if not value:
|
|
194
|
+
return ""
|
|
195
|
+
value = dec.normalize_newlines(value) # values are read as a browser reads the page
|
|
196
|
+
w = _Lines()
|
|
197
|
+
n = len(value)
|
|
198
|
+
plain = PLAIN.match
|
|
199
|
+
i = 0
|
|
200
|
+
while i < n:
|
|
201
|
+
c = value[i]
|
|
202
|
+
if c == "&":
|
|
203
|
+
# '&': decoded chars go through the same whitespace/text logic ( collapses, < is plain text)
|
|
204
|
+
decoded, consumed = dec.decode_reference(value, i + 1)
|
|
205
|
+
if decoded is not None:
|
|
206
|
+
w.chars(decoded)
|
|
207
|
+
i += 1 + consumed
|
|
208
|
+
else:
|
|
209
|
+
w.chars("&")
|
|
210
|
+
i += 1
|
|
211
|
+
continue
|
|
212
|
+
|
|
213
|
+
if c != "<" or not (i + 1 < n and (value[i + 1].isascii() and (value[i + 1].isalpha() or value[i + 1] in "/!?"))):
|
|
214
|
+
# plain text up to the next '&' or '<'
|
|
215
|
+
j = plain(value, i + 1).end()
|
|
216
|
+
w.chars(value[i:j])
|
|
217
|
+
i = j
|
|
218
|
+
continue
|
|
219
|
+
|
|
220
|
+
if i + 4 <= n and value[i + 1] == "!" and value[i + 2] == "-" and value[i + 3] == "-":
|
|
221
|
+
e = value.find("-->", i + 4)
|
|
222
|
+
if e < 0:
|
|
223
|
+
break
|
|
224
|
+
i = e + 3
|
|
225
|
+
continue
|
|
226
|
+
|
|
227
|
+
close_tag = value.find(">", i)
|
|
228
|
+
if close_tag < 0:
|
|
229
|
+
break # unterminated tag: the rest is markup
|
|
230
|
+
tag = value[i + 1:close_tag]
|
|
231
|
+
i = close_tag + 1
|
|
232
|
+
|
|
233
|
+
closing = tag[:1] == "/"
|
|
234
|
+
name = tag[1:] if closing else tag
|
|
235
|
+
m = TAG_NAME_END.search(name)
|
|
236
|
+
if m is not None:
|
|
237
|
+
name = name[:m.start()]
|
|
238
|
+
known = lookup_tag(name)
|
|
239
|
+
if known < 0:
|
|
240
|
+
continue # unknown tag: just removed
|
|
241
|
+
|
|
242
|
+
kind = KINDS[known]
|
|
243
|
+
if kind == BREAK_LINE or kind == BREAK_PARAGRAPH:
|
|
244
|
+
w.brk(kind, known)
|
|
245
|
+
elif kind == BREAK_SPACE:
|
|
246
|
+
w.chars(" ")
|
|
247
|
+
elif kind == BREAK_SKIP_CONTENT:
|
|
248
|
+
if not closing and not tag.rstrip(ASCII_SPACE).endswith("/"):
|
|
249
|
+
# jump to the closing tag; it is removed on the next iteration
|
|
250
|
+
e = _index_fold(value, "</" + NAMES[known], i)
|
|
251
|
+
i = n if e < 0 else e
|
|
252
|
+
return "".join(w.out)
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def _index_fold(s, sub, start):
|
|
256
|
+
m = re.compile(re.escape(sub), re.IGNORECASE | re.ASCII).search(s, start)
|
|
257
|
+
return -1 if m is None else m.start()
|