fasthtmlparser-pure 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,144 @@
1
+ """HTML5 character reference decoder (WHATWG "character reference state"): all 2125 named references, the legacy
2
+ names without ';', numeric references with or without ';', and the spec replacements (0, surrogates, > U+10FFFF ->
3
+ U+FFFD; 0x80-0x9F -> Windows-1252)."""
4
+
5
+ from ._tables import ENTITIES, LEGACY, MAX_LEGACY_NAME, MAX_NAME
6
+
7
+ # Replacements for numeric references 0x80..0x9F; 0 = keep the code point as is.
8
+ WINDOWS_1252 = (
9
+ 0x20AC, 0, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0, 0x017D, 0,
10
+ 0, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0, 0x017E,
11
+ 0x0178,
12
+ )
13
+
14
+ _ALNUM = frozenset("0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
15
+ _HEX = {c: int(c, 16) for c in "0123456789abcdefABCDEF"}
16
+
17
+
18
+ def needs_decoding(s):
19
+ """Whether a value read from the page differs from its source: it has a character reference or a CR."""
20
+ return "&" in s or "\r" in s
21
+
22
+
23
+ def normalize_newlines(s):
24
+ """HTML input stream preprocessing: CR LF and a lone CR become LF."""
25
+ if "\r" not in s:
26
+ return s
27
+ return s.replace("\r\n", "\n").replace("\r", "\n")
28
+
29
+
30
+ def decode(s, in_attribute):
31
+ """Decodes the character references in s; CR LF and a lone CR become LF. With in_attribute, a legacy name
32
+ without ';' followed by '=' or an alphanumeric is left as is (so "?a=1&copy=2" stays intact)."""
33
+ # Line breaks first, as a browser reads the page; a CR written as "
" is decoded after that and stays.
34
+ if "\r" in s:
35
+ s = s.replace("\r\n", "\n").replace("\r", "\n")
36
+ amp = s.find("&")
37
+ if amp < 0:
38
+ return s
39
+ n = len(s)
40
+ out = []
41
+ pos = 0
42
+ while amp >= 0:
43
+ if amp > pos:
44
+ out.append(s[pos:amp])
45
+ text = None
46
+ consumed = 0
47
+ if amp + 1 < n:
48
+ text, consumed = reference(s, amp + 1, in_attribute)
49
+ if text is None:
50
+ out.append("&") # not a reference: keep '&'
51
+ pos = amp + 1
52
+ else:
53
+ out.append(text)
54
+ pos = amp + 1 + consumed
55
+ amp = s.find("&", pos) if pos < n else -1
56
+ if pos < n:
57
+ out.append(s[pos:])
58
+ return "".join(out)
59
+
60
+
61
+ def decode_reference(s, at):
62
+ """Decodes one character reference; s[at] is the char right after '&'. Returns (text, chars consumed after '&'),
63
+ or (None, 0) when there is no reference."""
64
+ if at >= len(s):
65
+ return None, 0
66
+ return reference(s, at, False)
67
+
68
+
69
+ def reference(s, at, in_attribute):
70
+ if s[at] == "#":
71
+ cp, consumed = match_numeric(s, at)
72
+ if cp < 0:
73
+ return None, 0
74
+ return chr(cp), consumed
75
+ return match_named(s, at, in_attribute)
76
+
77
+
78
+ def match_numeric(s, at):
79
+ """s[at] is '#'. Returns (code point (-1: not a reference), chars consumed)."""
80
+ n = len(s)
81
+ i = at + 1
82
+ c = s[i] if i < n else ""
83
+ is_hex = c == "x" or c == "X"
84
+ if is_hex:
85
+ i += 1
86
+ digits = i
87
+ value = 0
88
+ if is_hex:
89
+ while i < n:
90
+ d = _HEX.get(s[i])
91
+ if d is None:
92
+ break
93
+ value = value * 16 + d
94
+ if value > 0x10FFFF:
95
+ value = 0x110000 # saturate on long digit runs
96
+ i += 1
97
+ else:
98
+ while i < n:
99
+ c = s[i]
100
+ if c < "0" or c > "9":
101
+ break
102
+ value = value * 10 + ord(c) - 48
103
+ if value > 0x10FFFF:
104
+ value = 0x110000
105
+ i += 1
106
+ if i == digits:
107
+ return -1, 0 # "&#" / "&#x" without digits is plain text
108
+ if i < n and s[i] == ";":
109
+ i += 1
110
+ if value == 0 or value > 0x10FFFF or 0xD800 <= value <= 0xDFFF:
111
+ value = 0xFFFD
112
+ elif 0x80 <= value <= 0x9F:
113
+ mapped = WINDOWS_1252[value - 0x80]
114
+ if mapped:
115
+ value = mapped
116
+ return value, i - at
117
+
118
+
119
+ def match_named(s, at, in_attribute):
120
+ """s[at] is the char right after '&'. Returns (decoded text (None: not a reference), chars consumed)."""
121
+ length = len(s)
122
+ n = 0
123
+ limit = MAX_NAME + 1
124
+ while n < limit and at + n < length and s[at + n] in _ALNUM:
125
+ n += 1
126
+ if n < 2:
127
+ return None, 0
128
+ # Full match: "&name;".
129
+ if at + n < length and s[at + n] == ";":
130
+ v = ENTITIES.get(s[at:at + n])
131
+ if v is not None:
132
+ return v, n + 1
133
+ # Longest legacy prefix without ';': "&copy2024" -> "(c)2024", "&notit;" -> "(not)it;".
134
+ k = n if n < MAX_LEGACY_NAME else MAX_LEGACY_NAME
135
+ while k >= 2:
136
+ name = s[at:at + k]
137
+ if name in LEGACY:
138
+ if in_attribute and at + k < length:
139
+ c = s[at + k]
140
+ if c == "=" or c in _ALNUM:
141
+ break
142
+ return ENTITIES[name], k
143
+ k -= 1
144
+ return None, 0
@@ -0,0 +1,483 @@
1
+ """Links: href / src made absolute against the page URL, written as .NET's System.Uri writes them.
2
+
3
+ Resolution follows RFC 3986 the way the Go port does it (net/url: Parse + ResolveReference), byte for byte: the URL
4
+ parts here are UTF-8 bytes.
5
+ """
6
+
7
+ import re
8
+
9
+ from . import _decode as dec
10
+ from ._chars import ascii_lower, trim_space
11
+
12
+
13
+ class UrlError(Exception):
14
+ pass
15
+
16
+
17
+ class Url:
18
+ __slots__ = ("scheme", "opaque", "user", "host", "path", "raw_path", "force_query", "raw_query", "fragment",
19
+ "raw_fragment")
20
+
21
+ def __init__(self):
22
+ self.scheme = b""
23
+ self.opaque = b""
24
+ self.user = None # the userinfo as written back (escaped), or None
25
+ self.host = b""
26
+ self.path = b"" # decoded
27
+ self.raw_path = b"" # the path as written, when it differs from the default escaping of path
28
+ self.force_query = False
29
+ self.raw_query = b""
30
+ self.fragment = b"" # decoded
31
+ self.raw_fragment = b""
32
+
33
+ def copy(self):
34
+ u = Url()
35
+ for k in Url.__slots__:
36
+ setattr(u, k, getattr(self, k))
37
+ return u
38
+
39
+
40
+ MODE_PATH = 0
41
+ MODE_HOST = 1
42
+ MODE_USER_PASSWORD = 2
43
+ MODE_FRAGMENT = 3
44
+
45
+ DEFAULT_PORTS = {b"http": b"80", b"https": b"443", b"ftp": b"21", b"ws": b"80", b"wss": b"443"}
46
+ RESERVED_FOR_DISPLAY = frozenset(b"%/?#[]@:!$&'()*+,;=")
47
+ UPPER_HEX = b"0123456789ABCDEF"
48
+ HOST_ALLOWED = frozenset(b"!$&'()*+,;=:[]<>\"")
49
+ VALID_ENCODED = frozenset(b"!$&'()*+,;=:@[]%")
50
+ USERINFO = re.compile(rb"\A[a-zA-Z0-9\-._:~!$&'()*+,;=%@]*\Z")
51
+ CONTROL = re.compile(rb"[\x00-\x1f\x7f]")
52
+ HEX = frozenset(b"0123456789abcdefABCDEF")
53
+
54
+
55
+ def _should_escape(c, mode):
56
+ if 0x61 <= c <= 0x7A or 0x41 <= c <= 0x5A or 0x30 <= c <= 0x39:
57
+ return False
58
+ if mode == MODE_HOST and c in HOST_ALLOWED:
59
+ return False
60
+ if c in b"-_.~":
61
+ return False
62
+ if c in b"$&+,/:;=?@" and mode != MODE_HOST:
63
+ if mode == MODE_PATH:
64
+ return c == 0x3F # '?'
65
+ if mode == MODE_USER_PASSWORD:
66
+ return c in b"@/?:"
67
+ return False
68
+ if mode == MODE_FRAGMENT and c in b"!()*":
69
+ return False
70
+ return True
71
+
72
+
73
+ # escape decisions by byte, per mode
74
+ SHOULD_ESCAPE = tuple(tuple(_should_escape(c, m) for c in range(256)) for m in range(4))
75
+
76
+
77
+ def _unhex(c):
78
+ return c - 48 if c <= 57 else (c | 0x20) - 87
79
+
80
+
81
+ def unescape(s, mode):
82
+ """net/url unescape; raises UrlError on a malformed URL."""
83
+ count = 0
84
+ n = len(s)
85
+ table = SHOULD_ESCAPE[mode]
86
+ i = 0
87
+ while i < n:
88
+ c = s[i]
89
+ if c == 37:
90
+ count += 1
91
+ if i + 2 >= n or s[i + 1] not in HEX or s[i + 2] not in HEX:
92
+ raise UrlError("invalid URL escape")
93
+ # in the host, %-encoding is only for non-ASCII bytes (and %25)
94
+ if mode == MODE_HOST and _unhex(s[i + 1]) < 8 and s[i:i + 3] != b"%25":
95
+ raise UrlError("invalid URL escape")
96
+ i += 3
97
+ else:
98
+ if mode == MODE_HOST and c < 0x80 and table[c]:
99
+ raise UrlError("invalid character in host name")
100
+ i += 1
101
+ if count == 0:
102
+ return s
103
+ out = bytearray()
104
+ i = 0
105
+ while i < n:
106
+ c = s[i]
107
+ if c == 37:
108
+ out.append((_unhex(s[i + 1]) << 4) | _unhex(s[i + 2]))
109
+ i += 3
110
+ else:
111
+ out.append(c)
112
+ i += 1
113
+ return bytes(out)
114
+
115
+
116
+ def escape(s, mode):
117
+ table = SHOULD_ESCAPE[mode]
118
+ out = None
119
+ start = 0
120
+ for i, c in enumerate(s):
121
+ if table[c]:
122
+ if out is None:
123
+ out = bytearray()
124
+ out += s[start:i]
125
+ out.append(37)
126
+ out.append(UPPER_HEX[c >> 4])
127
+ out.append(UPPER_HEX[c & 15])
128
+ start = i + 1
129
+ if out is None:
130
+ return s
131
+ out += s[start:]
132
+ return bytes(out)
133
+
134
+
135
+ def _valid_encoded(s, mode):
136
+ table = SHOULD_ESCAPE[mode]
137
+ for c in s:
138
+ if c in VALID_ENCODED:
139
+ continue
140
+ if table[c]:
141
+ return False
142
+ return True
143
+
144
+
145
+ # ----- parsing -----
146
+
147
+ def _get_scheme(raw):
148
+ for i, c in enumerate(raw):
149
+ if 0x61 <= (c | 0x20) <= 0x7A:
150
+ continue
151
+ if 0x30 <= c <= 0x39 or c == 43 or c == 45 or c == 46:
152
+ if i == 0:
153
+ return b"", raw
154
+ continue
155
+ if c == 58:
156
+ if i == 0:
157
+ raise UrlError("missing protocol scheme")
158
+ return raw[:i], raw[i + 1:]
159
+ return b"", raw
160
+ return b"", raw
161
+
162
+
163
+ def _set_path(u, p):
164
+ u.path = unescape(p, MODE_PATH)
165
+ u.raw_path = b"" if escape(u.path, MODE_PATH) == p else p
166
+
167
+
168
+ def _escaped_path(u):
169
+ if u.raw_path and _valid_encoded(u.raw_path, MODE_PATH):
170
+ try:
171
+ if unescape(u.raw_path, MODE_PATH) == u.path:
172
+ return u.raw_path
173
+ except UrlError:
174
+ pass
175
+ if u.path == b"*":
176
+ return b"*"
177
+ return escape(u.path, MODE_PATH)
178
+
179
+
180
+ def _escaped_fragment(u):
181
+ if u.raw_fragment and _valid_encoded(u.raw_fragment, MODE_FRAGMENT):
182
+ try:
183
+ if unescape(u.raw_fragment, MODE_FRAGMENT) == u.fragment:
184
+ return u.raw_fragment
185
+ except UrlError:
186
+ pass
187
+ return escape(u.fragment, MODE_FRAGMENT)
188
+
189
+
190
+ def _valid_optional_port(port):
191
+ if not port:
192
+ return True
193
+ if port[0] != 58:
194
+ return False
195
+ return all(48 <= c <= 57 for c in port[1:])
196
+
197
+
198
+ def _parse_host(host):
199
+ if host.startswith(b"["):
200
+ i = host.rfind(b"]")
201
+ if i < 0:
202
+ raise UrlError("missing ']' in host")
203
+ if not _valid_optional_port(host[i + 1:]):
204
+ raise UrlError("invalid port")
205
+ else:
206
+ i = host.rfind(b":")
207
+ if i >= 0 and not _valid_optional_port(host[i:]):
208
+ raise UrlError("invalid port")
209
+ return unescape(host, MODE_HOST)
210
+
211
+
212
+ def _parse_authority(u, authority):
213
+ i = authority.rfind(b"@")
214
+ u.host = _parse_host(authority if i < 0 else authority[i + 1:])
215
+ if i < 0:
216
+ return
217
+ userinfo = authority[:i]
218
+ if not USERINFO.match(userinfo):
219
+ raise UrlError("invalid userinfo")
220
+ colon = userinfo.find(b":")
221
+ if colon < 0:
222
+ u.user = escape(unescape(userinfo, MODE_USER_PASSWORD), MODE_USER_PASSWORD)
223
+ else:
224
+ name = unescape(userinfo[:colon], MODE_USER_PASSWORD)
225
+ password = unescape(userinfo[colon + 1:], MODE_USER_PASSWORD)
226
+ u.user = escape(name, MODE_USER_PASSWORD) + b":" + escape(password, MODE_USER_PASSWORD)
227
+
228
+
229
+ def parse_url(raw_url):
230
+ """net/url Parse."""
231
+ h = raw_url.find(b"#")
232
+ u = _parse_no_fragment(raw_url if h < 0 else raw_url[:h])
233
+ if h >= 0:
234
+ frag = raw_url[h + 1:]
235
+ u.fragment = unescape(frag, MODE_FRAGMENT)
236
+ u.raw_fragment = b"" if escape(u.fragment, MODE_FRAGMENT) == frag else frag
237
+ return u
238
+
239
+
240
+ def _parse_no_fragment(raw):
241
+ if CONTROL.search(raw):
242
+ raise UrlError("invalid control character in URL")
243
+ u = Url()
244
+ if raw == b"*":
245
+ u.path = b"*"
246
+ return u
247
+ scheme, rest = _get_scheme(raw)
248
+ u.scheme = scheme.lower()
249
+
250
+ q = rest.find(b"?")
251
+ if q >= 0 and q == len(rest) - 1:
252
+ u.force_query = True
253
+ rest = rest[:q]
254
+ elif q >= 0:
255
+ u.raw_query = rest[q + 1:]
256
+ rest = rest[:q]
257
+
258
+ if not rest.startswith(b"/"):
259
+ if u.scheme:
260
+ # rootless paths are opaque (RFC 3986)
261
+ u.opaque = rest
262
+ return u
263
+ # a relative-path reference: the first segment cannot contain ':'
264
+ slash = rest.find(b"/")
265
+ first = rest if slash < 0 else rest[:slash]
266
+ if b":" in first:
267
+ raise UrlError("first path segment in URL cannot contain colon")
268
+
269
+ if (u.scheme or not rest.startswith(b"///")) and rest.startswith(b"//"):
270
+ authority = rest[2:]
271
+ rest = b""
272
+ slash = authority.find(b"/")
273
+ if slash >= 0:
274
+ rest = authority[slash:]
275
+ authority = authority[:slash]
276
+ _parse_authority(u, authority)
277
+ _set_path(u, rest)
278
+ return u
279
+
280
+
281
+ def _resolve_path(base, ref):
282
+ if not ref:
283
+ full = base
284
+ elif ref[0] != 47:
285
+ slash = base.rfind(b"/")
286
+ full = (b"" if slash < 0 else base[:slash + 1]) + ref
287
+ else:
288
+ full = ref
289
+ if not full:
290
+ return b""
291
+ dst = bytearray(b"/")
292
+ first = True
293
+ remaining = full
294
+ found = True
295
+ elem = b""
296
+ while found:
297
+ slash = remaining.find(b"/")
298
+ found = slash >= 0
299
+ elem = remaining[:slash] if found else remaining
300
+ remaining = remaining[slash + 1:] if found else b""
301
+ if elem == b".":
302
+ first = False
303
+ continue
304
+ if elem == b"..":
305
+ s = bytes(dst[1:])
306
+ index = s.rfind(b"/")
307
+ dst = bytearray(b"/")
308
+ if index < 0:
309
+ first = True
310
+ else:
311
+ dst += s[:index]
312
+ else:
313
+ if not first:
314
+ dst.append(47)
315
+ dst += elem
316
+ first = False
317
+ if elem == b"." or elem == b"..":
318
+ dst.append(47)
319
+ if len(dst) > 1 and dst[1] == 47:
320
+ dst = dst[1:]
321
+ return bytes(dst)
322
+
323
+
324
+ def resolve_reference(base, ref):
325
+ """net/url ResolveReference."""
326
+ url = ref.copy()
327
+ if not ref.scheme:
328
+ url.scheme = base.scheme
329
+ if ref.scheme or ref.host or ref.user is not None:
330
+ _set_path(url, _resolve_path(_escaped_path(ref), b""))
331
+ return url
332
+ if ref.opaque:
333
+ url.user = None
334
+ url.host = b""
335
+ url.path = b""
336
+ return url
337
+ if not ref.path and not ref.force_query and not ref.raw_query:
338
+ url.raw_query = base.raw_query
339
+ if not ref.fragment:
340
+ url.fragment = base.fragment
341
+ url.raw_fragment = base.raw_fragment
342
+ if not ref.path and base.opaque:
343
+ url.opaque = base.opaque
344
+ url.user = None
345
+ url.host = b""
346
+ url.path = b""
347
+ return url
348
+ url.host = base.host
349
+ url.user = base.user
350
+ _set_path(url, _resolve_path(_escaped_path(base), _escaped_path(ref)))
351
+ return url
352
+
353
+
354
+ def _split_host_port(host_port):
355
+ host = host_port
356
+ port = b""
357
+ colon = host.rfind(b":")
358
+ if colon >= 0 and _valid_optional_port(host[colon:]):
359
+ port = host[colon + 1:]
360
+ host = host[:colon]
361
+ if len(host) >= 2 and host[0] == 91 and host[-1] == 93:
362
+ host = host[1:-1]
363
+ return host, port
364
+
365
+
366
+ def _valid_utf8(b):
367
+ try:
368
+ b.decode("utf-8")
369
+ return True
370
+ except UnicodeDecodeError:
371
+ return False
372
+
373
+
374
+ def _unescape_for_display(s):
375
+ """Decodes %XX escapes except those of reserved and control characters; escaped bytes of non-ASCII text are
376
+ decoded when they form valid UTF-8."""
377
+ if b"%" not in s:
378
+ return s
379
+ out = bytearray()
380
+ n = len(s)
381
+ i = 0
382
+ while i < n:
383
+ if s[i] != 37 or i + 2 >= n or s[i + 1] not in HEX or s[i + 2] not in HEX:
384
+ out.append(s[i])
385
+ i += 1
386
+ continue
387
+ c = (_unhex(s[i + 1]) << 4) | _unhex(s[i + 2])
388
+ if c < 0x80:
389
+ if c < 0x20 or c == 0x7F or c in RESERVED_FOR_DISPLAY:
390
+ out += s[i:i + 3]
391
+ else:
392
+ out.append(c)
393
+ i += 3
394
+ continue
395
+ # a run of escaped bytes >= 0x80: decoded together when it is valid UTF-8
396
+ j = i
397
+ run = bytearray()
398
+ while j + 2 < n and s[j] == 37 and s[j + 1] in HEX and s[j + 2] in HEX:
399
+ v = (_unhex(s[j + 1]) << 4) | _unhex(s[j + 2])
400
+ if v < 0x80:
401
+ break
402
+ run.append(v)
403
+ j += 3
404
+ out += run if _valid_utf8(bytes(run)) else s[i:j]
405
+ i = j
406
+ return bytes(out)
407
+
408
+
409
+ def display_string(u):
410
+ """Writes a resolved URL as the .NET version does (System.Uri.ToString): the scheme and host in lower case, no
411
+ default port, an empty path as "/", and percent-escapes decoded except those of the characters that delimit URL
412
+ parts ("%2F" stays) and of control characters; non-ASCII text is written as is."""
413
+ scheme = u.scheme.lower()
414
+ b = bytearray(scheme + b":")
415
+ if not u.opaque:
416
+ if u.host or u.user is not None:
417
+ b += b"//"
418
+ if u.user is not None:
419
+ b += u.user + b"@"
420
+ host, port = _split_host_port(u.host)
421
+ b += host.decode("utf-8", "replace").lower().encode("utf-8")
422
+ if port and port != DEFAULT_PORTS.get(scheme):
423
+ b += b":" + port
424
+ path = _escaped_path(u)
425
+ if not path and u.host:
426
+ path = b"/"
427
+ b += _unescape_for_display(path)
428
+ else:
429
+ b += u.opaque
430
+ if u.raw_query or u.force_query:
431
+ b += b"?" + _unescape_for_display(u.raw_query)
432
+ if u.fragment:
433
+ b += b"#" + _unescape_for_display(_escaped_fragment(u))
434
+ return bytes(b).decode("utf-8", "replace")
435
+
436
+
437
+ def _b(s):
438
+ return s.encode("utf-8", "surrogatepass")
439
+
440
+
441
+ def absolute_url(s):
442
+ """Parses a page URL with its own scheme and host; None for anything else."""
443
+ if not s:
444
+ return None
445
+ try:
446
+ u = parse_url(_b(s))
447
+ except UrlError:
448
+ return None
449
+ if not u.scheme or (not u.host and not u.opaque and u.scheme != b"file"):
450
+ return None
451
+ return u
452
+
453
+
454
+ def _absolute_link(link):
455
+ """A link with its own scheme ("https://...", "mailto:...", "file:..."). "/path" and "//host/path" are
456
+ relative."""
457
+ try:
458
+ return bool(parse_url(_b(trim_space(link))).scheme)
459
+ except UrlError:
460
+ return False
461
+
462
+
463
+ def correct_link(raw, page_url, base):
464
+ """The raw attribute value with its character references decoded and made absolute against the page.
465
+ Percent-escapes are part of the URL and are kept. An empty link gives the page URL."""
466
+ if not raw:
467
+ return page_url
468
+ link = dec.decode(raw, True) if "&" in raw or "\r" in raw else raw
469
+ if base is None or _absolute_link(link):
470
+ return link
471
+ try:
472
+ return display_string(resolve_reference(base, parse_url(_b(trim_space(link)))))
473
+ except UrlError:
474
+ return link
475
+
476
+
477
+ def after_scheme(raw_href, scheme):
478
+ """Everything after the first ':' of a link with the scheme (mailto, tel), or ""."""
479
+ href = trim_space(dec.decode(raw_href, True))
480
+ if len(href) < len(scheme) or ascii_lower(href[:len(scheme)]) != scheme:
481
+ return ""
482
+ i = href.find(":")
483
+ return "" if i < 0 else href[i + 1:]