fasthtmlparser-pure 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,257 @@
1
+ """Visible text: tags, comments and invisible content removed, whitespace collapsed, entities decoded; and the text
2
+ with the visible line structure kept (Node.text_line_breaks)."""
3
+
4
+ import re
5
+
6
+ from . import _decode as dec
7
+ from ._chars import ASCII_SPACE, HAS_SPACE, SPACE_RUNS, ascii_lower
8
+ from ._parser import KEYS, NAMES, find_comment_end, find_raw_text_end, lookup_tag
9
+
10
+ INVISIBLE = {
11
+ "style": "</style", "script": "</script", "noembed": "</noembed", "noscript": "</noscript",
12
+ "template": "</template", "noframes": "</noframes",
13
+ }
14
+ # '<' that starts markup: before an ASCII letter, '/', '!' or '?'
15
+ MARKUP = re.compile(r"<[a-zA-Z/!?]")
16
+
17
+
18
+ def skip_markup(s, i):
19
+ """s[i] is '<'. Returns the index of the last char of what is not text there: a comment, a tag, or a tag together
20
+ with invisible content (script, style, ...) up to its end tag. -1: an unterminated tag."""
21
+ n = len(s)
22
+ if i + 4 <= n and s[i + 1] == "!" and s[i + 2] == "-" and s[i + 3] == "-":
23
+ return find_comment_end(s, i + 4, n)
24
+ tag_end = s.find(">", i + 1)
25
+ if tag_end < 0:
26
+ return -1
27
+ end_tag = invisible_content_end(s, i + 1, tag_end)
28
+ if end_tag is not None:
29
+ return find_raw_text_end(s, tag_end + 1, end_tag, n) - 1
30
+ return tag_end
31
+
32
+
33
+ def invisible_content_end(s, start, to):
34
+ """For the inside of an opening tag s[start:to], the end tag ("</script") when the element's content is never
35
+ visible text (script, style, noscript, template, noframes, noembed), otherwise None."""
36
+ if start >= to:
37
+ return None
38
+ first = s[start]
39
+ if first not in "sStTnN":
40
+ return None
41
+ n = 1
42
+ while start + n < to and n <= 8:
43
+ c = s[start + n]
44
+ if not (("a" <= c <= "z") or ("A" <= c <= "Z")):
45
+ break
46
+ n += 1
47
+ if start + n < to:
48
+ c = s[start + n]
49
+ if c != "/" and c not in ASCII_SPACE:
50
+ return None
51
+ end = INVISIBLE.get(ascii_lower(s[start:start + n]))
52
+ if end is None:
53
+ return None
54
+ e = to
55
+ while e > start and s[e - 1] in ASCII_SPACE:
56
+ e -= 1
57
+ if s[e - 1] == "/":
58
+ return None # <script src="x" />: no content
59
+ return end
60
+
61
+
62
+ def text_runs(s):
63
+ """The text runs of s with tags, comments and invisible content removed, whitespace not yet collapsed."""
64
+ search = MARKUP.search
65
+ m = search(s)
66
+ if m is None:
67
+ return s
68
+ out = []
69
+ start = 0
70
+ while m is not None:
71
+ lt = m.start()
72
+ if lt > start:
73
+ out.append(s[start:lt])
74
+ last = skip_markup(s, lt)
75
+ if last < 0:
76
+ return "".join(out) # an unterminated tag: the rest is markup
77
+ start = last + 1
78
+ m = search(s, start)
79
+ if start < len(s):
80
+ out.append(s[start:])
81
+ return "".join(out)
82
+
83
+
84
+ def collapse(s):
85
+ """Whitespace runs as one ' ', trimmed."""
86
+ if HAS_SPACE.search(s) is None:
87
+ return s
88
+ t = SPACE_RUNS.sub(" ", s)
89
+ if not t:
90
+ return t
91
+ a = 1 if t[0] == " " else 0
92
+ z = len(t) - (1 if t[-1] == " " else 0)
93
+ return t if a == 0 and z == len(t) else t[a:z]
94
+
95
+
96
+ def clear_tags(s):
97
+ """The visible text of an HTML fragment."""
98
+ if not s:
99
+ return ""
100
+ text = collapse(text_runs(s))
101
+ return dec.decode(text, False) if "&" in text else text
102
+
103
+
104
+ def text_equals(value, expected, d, i):
105
+ """Whether the visible text of the body (value) of the tag i equals expected."""
106
+ if not value or not expected:
107
+ return False
108
+ # whitespace is collapsed even where text keeps it (a tag without children), as the .NET version compares
109
+ text = collapse(text_runs(value))
110
+ amp = text.find("&")
111
+ if amp < 0:
112
+ return text == expected
113
+ # Text with entities must be decoded before comparing: the full text, when the part before it matches.
114
+ head = text[:amp].rstrip(" ")
115
+ return expected.startswith(head) and d.text_of(i) == expected
116
+
117
+
118
+ # ----- text with line breaks -----
119
+
120
+ BREAK_SPACE = 1 # table cell: word separator
121
+ BREAK_LINE = 2 # <br> and block elements (div, li, tr, ...): new line
122
+ BREAK_PARAGRAPH = 3 # p, h1-h6: new paragraph (blank line)
123
+ BREAK_SKIP_CONTENT = 4 # script, style, noscript: contents are not text
124
+ # inline tags (b, span, a, ...) are removed, words are not split
125
+
126
+ BR = KEYS["br"]
127
+ KINDS = [0] * len(NAMES)
128
+ for _t in "p h1 h2 h3 h4 h5 h6".split():
129
+ KINDS[KEYS[_t]] = BREAK_PARAGRAPH
130
+ for _t in ("br div li ul ol dl dt dd table caption tr thead tbody tfoot section article aside nav header footer main "
131
+ "address blockquote pre hr figure figcaption form fieldset legend details summary dialog option html head "
132
+ "body title hgroup search menu center dir listing xmp plaintext multicol frameset marquee").split():
133
+ KINDS[KEYS[_t]] = BREAK_LINE
134
+ KINDS[KEYS["td"]] = BREAK_SPACE
135
+ KINDS[KEYS["th"]] = BREAK_SPACE
136
+ for _t in "script style noscript template noframes noembed".split():
137
+ KINDS[KEYS[_t]] = BREAK_SKIP_CONTENT
138
+
139
+ PLAIN = re.compile(r"[^&<]*")
140
+ TAG_NAME_END = re.compile(r"[\t\r\n\f /]")
141
+
142
+
143
+ class _Lines:
144
+ __slots__ = ("out", "pending_breaks", "pending_space", "started")
145
+
146
+ def __init__(self):
147
+ self.out = []
148
+ self.pending_breaks = 0 # new lines requested since the last text char
149
+ self.pending_space = False
150
+ self.started = False
151
+
152
+ def chars(self, s):
153
+ """Writes text, collapsing whitespace and turning requested breaks into trimmed lines."""
154
+ if HAS_SPACE.search(s) is None:
155
+ self.text(s)
156
+ return
157
+ pos = 0
158
+ n = len(s)
159
+ for m in SPACE_RUNS.finditer(s):
160
+ i = m.start()
161
+ if i > pos:
162
+ self.text(s[pos:i])
163
+ self.pending_space = True
164
+ pos = m.end()
165
+ if pos < n:
166
+ self.text(s[pos:])
167
+
168
+ def text(self, s):
169
+ if self.started:
170
+ # nothing before the first text char: leading breaks and spaces are trimmed
171
+ if self.pending_breaks > 0:
172
+ self.out.append("\n\n" if self.pending_breaks >= 2 else "\n") # at most one blank line
173
+ elif self.pending_space:
174
+ self.out.append(" ")
175
+ self.pending_breaks = 0
176
+ self.pending_space = False
177
+ if s:
178
+ self.out.append(s)
179
+ self.started = True
180
+
181
+ def brk(self, kind, known):
182
+ if kind == BREAK_LINE:
183
+ if known == BR:
184
+ self.pending_breaks += 1 # every <br> counts
185
+ elif self.pending_breaks < 1:
186
+ self.pending_breaks = 1
187
+ elif self.pending_breaks < 2:
188
+ self.pending_breaks = 2
189
+
190
+
191
+ def clear_tags_line_breaks(value):
192
+ """The text of an HTML fragment with the visible line structure kept. Lines are joined with "\\n"."""
193
+ if not value:
194
+ return ""
195
+ value = dec.normalize_newlines(value) # values are read as a browser reads the page
196
+ w = _Lines()
197
+ n = len(value)
198
+ plain = PLAIN.match
199
+ i = 0
200
+ while i < n:
201
+ c = value[i]
202
+ if c == "&":
203
+ # '&': decoded chars go through the same whitespace/text logic (&nbsp; collapses, &lt; is plain text)
204
+ decoded, consumed = dec.decode_reference(value, i + 1)
205
+ if decoded is not None:
206
+ w.chars(decoded)
207
+ i += 1 + consumed
208
+ else:
209
+ w.chars("&")
210
+ i += 1
211
+ continue
212
+
213
+ if c != "<" or not (i + 1 < n and (value[i + 1].isascii() and (value[i + 1].isalpha() or value[i + 1] in "/!?"))):
214
+ # plain text up to the next '&' or '<'
215
+ j = plain(value, i + 1).end()
216
+ w.chars(value[i:j])
217
+ i = j
218
+ continue
219
+
220
+ if i + 4 <= n and value[i + 1] == "!" and value[i + 2] == "-" and value[i + 3] == "-":
221
+ e = value.find("-->", i + 4)
222
+ if e < 0:
223
+ break
224
+ i = e + 3
225
+ continue
226
+
227
+ close_tag = value.find(">", i)
228
+ if close_tag < 0:
229
+ break # unterminated tag: the rest is markup
230
+ tag = value[i + 1:close_tag]
231
+ i = close_tag + 1
232
+
233
+ closing = tag[:1] == "/"
234
+ name = tag[1:] if closing else tag
235
+ m = TAG_NAME_END.search(name)
236
+ if m is not None:
237
+ name = name[:m.start()]
238
+ known = lookup_tag(name)
239
+ if known < 0:
240
+ continue # unknown tag: just removed
241
+
242
+ kind = KINDS[known]
243
+ if kind == BREAK_LINE or kind == BREAK_PARAGRAPH:
244
+ w.brk(kind, known)
245
+ elif kind == BREAK_SPACE:
246
+ w.chars(" ")
247
+ elif kind == BREAK_SKIP_CONTENT:
248
+ if not closing and not tag.rstrip(ASCII_SPACE).endswith("/"):
249
+ # jump to the closing tag; it is removed on the next iteration
250
+ e = _index_fold(value, "</" + NAMES[known], i)
251
+ i = n if e < 0 else e
252
+ return "".join(w.out)
253
+
254
+
255
+ def _index_fold(s, sub, start):
256
+ m = re.compile(re.escape(sub), re.IGNORECASE | re.ASCII).search(s, start)
257
+ return -1 if m is None else m.start()