fasthtmlparser-pure 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fasthtmlparser_pure/__init__.py +41 -0
- fasthtmlparser_pure/_api.py +636 -0
- fasthtmlparser_pure/_api.pyi +342 -0
- fasthtmlparser_pure/_attrs.py +270 -0
- fasthtmlparser_pure/_chars.py +124 -0
- fasthtmlparser_pure/_css.py +861 -0
- fasthtmlparser_pure/_decode.py +144 -0
- fasthtmlparser_pure/_links.py +483 -0
- fasthtmlparser_pure/_parser.py +866 -0
- fasthtmlparser_pure/_query.py +128 -0
- fasthtmlparser_pure/_tables.py +2494 -0
- fasthtmlparser_pure/_text.py +257 -0
- fasthtmlparser_pure/_xpath.py +1594 -0
- fasthtmlparser_pure/py.typed +0 -0
- fasthtmlparser_pure-0.1.0.dist-info/METADATA +166 -0
- fasthtmlparser_pure-0.1.0.dist-info/RECORD +19 -0
- fasthtmlparser_pure-0.1.0.dist-info/WHEEL +5 -0
- fasthtmlparser_pure-0.1.0.dist-info/licenses/LICENSE.md +79 -0
- fasthtmlparser_pure-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
"""HTML5 character reference decoder (WHATWG "character reference state"): all 2125 named references, the legacy
|
|
2
|
+
names without ';', numeric references with or without ';', and the spec replacements (0, surrogates, > U+10FFFF ->
|
|
3
|
+
U+FFFD; 0x80-0x9F -> Windows-1252)."""
|
|
4
|
+
|
|
5
|
+
from ._tables import ENTITIES, LEGACY, MAX_LEGACY_NAME, MAX_NAME
|
|
6
|
+
|
|
7
|
+
# Replacements for numeric references 0x80..0x9F; 0 = keep the code point as is.
|
|
8
|
+
WINDOWS_1252 = (
|
|
9
|
+
0x20AC, 0, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0, 0x017D, 0,
|
|
10
|
+
0, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0, 0x017E,
|
|
11
|
+
0x0178,
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
_ALNUM = frozenset("0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
|
|
15
|
+
_HEX = {c: int(c, 16) for c in "0123456789abcdefABCDEF"}
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def needs_decoding(s):
|
|
19
|
+
"""Whether a value read from the page differs from its source: it has a character reference or a CR."""
|
|
20
|
+
return "&" in s or "\r" in s
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def normalize_newlines(s):
|
|
24
|
+
"""HTML input stream preprocessing: CR LF and a lone CR become LF."""
|
|
25
|
+
if "\r" not in s:
|
|
26
|
+
return s
|
|
27
|
+
return s.replace("\r\n", "\n").replace("\r", "\n")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def decode(s, in_attribute):
|
|
31
|
+
"""Decodes the character references in s; CR LF and a lone CR become LF. With in_attribute, a legacy name
|
|
32
|
+
without ';' followed by '=' or an alphanumeric is left as is (so "?a=1©=2" stays intact)."""
|
|
33
|
+
# Line breaks first, as a browser reads the page; a CR written as " " is decoded after that and stays.
|
|
34
|
+
if "\r" in s:
|
|
35
|
+
s = s.replace("\r\n", "\n").replace("\r", "\n")
|
|
36
|
+
amp = s.find("&")
|
|
37
|
+
if amp < 0:
|
|
38
|
+
return s
|
|
39
|
+
n = len(s)
|
|
40
|
+
out = []
|
|
41
|
+
pos = 0
|
|
42
|
+
while amp >= 0:
|
|
43
|
+
if amp > pos:
|
|
44
|
+
out.append(s[pos:amp])
|
|
45
|
+
text = None
|
|
46
|
+
consumed = 0
|
|
47
|
+
if amp + 1 < n:
|
|
48
|
+
text, consumed = reference(s, amp + 1, in_attribute)
|
|
49
|
+
if text is None:
|
|
50
|
+
out.append("&") # not a reference: keep '&'
|
|
51
|
+
pos = amp + 1
|
|
52
|
+
else:
|
|
53
|
+
out.append(text)
|
|
54
|
+
pos = amp + 1 + consumed
|
|
55
|
+
amp = s.find("&", pos) if pos < n else -1
|
|
56
|
+
if pos < n:
|
|
57
|
+
out.append(s[pos:])
|
|
58
|
+
return "".join(out)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def decode_reference(s, at):
|
|
62
|
+
"""Decodes one character reference; s[at] is the char right after '&'. Returns (text, chars consumed after '&'),
|
|
63
|
+
or (None, 0) when there is no reference."""
|
|
64
|
+
if at >= len(s):
|
|
65
|
+
return None, 0
|
|
66
|
+
return reference(s, at, False)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def reference(s, at, in_attribute):
|
|
70
|
+
if s[at] == "#":
|
|
71
|
+
cp, consumed = match_numeric(s, at)
|
|
72
|
+
if cp < 0:
|
|
73
|
+
return None, 0
|
|
74
|
+
return chr(cp), consumed
|
|
75
|
+
return match_named(s, at, in_attribute)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def match_numeric(s, at):
|
|
79
|
+
"""s[at] is '#'. Returns (code point (-1: not a reference), chars consumed)."""
|
|
80
|
+
n = len(s)
|
|
81
|
+
i = at + 1
|
|
82
|
+
c = s[i] if i < n else ""
|
|
83
|
+
is_hex = c == "x" or c == "X"
|
|
84
|
+
if is_hex:
|
|
85
|
+
i += 1
|
|
86
|
+
digits = i
|
|
87
|
+
value = 0
|
|
88
|
+
if is_hex:
|
|
89
|
+
while i < n:
|
|
90
|
+
d = _HEX.get(s[i])
|
|
91
|
+
if d is None:
|
|
92
|
+
break
|
|
93
|
+
value = value * 16 + d
|
|
94
|
+
if value > 0x10FFFF:
|
|
95
|
+
value = 0x110000 # saturate on long digit runs
|
|
96
|
+
i += 1
|
|
97
|
+
else:
|
|
98
|
+
while i < n:
|
|
99
|
+
c = s[i]
|
|
100
|
+
if c < "0" or c > "9":
|
|
101
|
+
break
|
|
102
|
+
value = value * 10 + ord(c) - 48
|
|
103
|
+
if value > 0x10FFFF:
|
|
104
|
+
value = 0x110000
|
|
105
|
+
i += 1
|
|
106
|
+
if i == digits:
|
|
107
|
+
return -1, 0 # "&#" / "&#x" without digits is plain text
|
|
108
|
+
if i < n and s[i] == ";":
|
|
109
|
+
i += 1
|
|
110
|
+
if value == 0 or value > 0x10FFFF or 0xD800 <= value <= 0xDFFF:
|
|
111
|
+
value = 0xFFFD
|
|
112
|
+
elif 0x80 <= value <= 0x9F:
|
|
113
|
+
mapped = WINDOWS_1252[value - 0x80]
|
|
114
|
+
if mapped:
|
|
115
|
+
value = mapped
|
|
116
|
+
return value, i - at
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def match_named(s, at, in_attribute):
|
|
120
|
+
"""s[at] is the char right after '&'. Returns (decoded text (None: not a reference), chars consumed)."""
|
|
121
|
+
length = len(s)
|
|
122
|
+
n = 0
|
|
123
|
+
limit = MAX_NAME + 1
|
|
124
|
+
while n < limit and at + n < length and s[at + n] in _ALNUM:
|
|
125
|
+
n += 1
|
|
126
|
+
if n < 2:
|
|
127
|
+
return None, 0
|
|
128
|
+
# Full match: "&name;".
|
|
129
|
+
if at + n < length and s[at + n] == ";":
|
|
130
|
+
v = ENTITIES.get(s[at:at + n])
|
|
131
|
+
if v is not None:
|
|
132
|
+
return v, n + 1
|
|
133
|
+
# Longest legacy prefix without ';': "©2024" -> "(c)2024", "¬it;" -> "(not)it;".
|
|
134
|
+
k = n if n < MAX_LEGACY_NAME else MAX_LEGACY_NAME
|
|
135
|
+
while k >= 2:
|
|
136
|
+
name = s[at:at + k]
|
|
137
|
+
if name in LEGACY:
|
|
138
|
+
if in_attribute and at + k < length:
|
|
139
|
+
c = s[at + k]
|
|
140
|
+
if c == "=" or c in _ALNUM:
|
|
141
|
+
break
|
|
142
|
+
return ENTITIES[name], k
|
|
143
|
+
k -= 1
|
|
144
|
+
return None, 0
|
|
@@ -0,0 +1,483 @@
|
|
|
1
|
+
"""Links: href / src made absolute against the page URL, written as .NET's System.Uri writes them.
|
|
2
|
+
|
|
3
|
+
Resolution follows RFC 3986 the way the Go port does it (net/url: Parse + ResolveReference), byte for byte: the URL
|
|
4
|
+
parts here are UTF-8 bytes.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import re
|
|
8
|
+
|
|
9
|
+
from . import _decode as dec
|
|
10
|
+
from ._chars import ascii_lower, trim_space
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class UrlError(Exception):
|
|
14
|
+
pass
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class Url:
|
|
18
|
+
__slots__ = ("scheme", "opaque", "user", "host", "path", "raw_path", "force_query", "raw_query", "fragment",
|
|
19
|
+
"raw_fragment")
|
|
20
|
+
|
|
21
|
+
def __init__(self):
|
|
22
|
+
self.scheme = b""
|
|
23
|
+
self.opaque = b""
|
|
24
|
+
self.user = None # the userinfo as written back (escaped), or None
|
|
25
|
+
self.host = b""
|
|
26
|
+
self.path = b"" # decoded
|
|
27
|
+
self.raw_path = b"" # the path as written, when it differs from the default escaping of path
|
|
28
|
+
self.force_query = False
|
|
29
|
+
self.raw_query = b""
|
|
30
|
+
self.fragment = b"" # decoded
|
|
31
|
+
self.raw_fragment = b""
|
|
32
|
+
|
|
33
|
+
def copy(self):
|
|
34
|
+
u = Url()
|
|
35
|
+
for k in Url.__slots__:
|
|
36
|
+
setattr(u, k, getattr(self, k))
|
|
37
|
+
return u
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
MODE_PATH = 0
|
|
41
|
+
MODE_HOST = 1
|
|
42
|
+
MODE_USER_PASSWORD = 2
|
|
43
|
+
MODE_FRAGMENT = 3
|
|
44
|
+
|
|
45
|
+
DEFAULT_PORTS = {b"http": b"80", b"https": b"443", b"ftp": b"21", b"ws": b"80", b"wss": b"443"}
|
|
46
|
+
RESERVED_FOR_DISPLAY = frozenset(b"%/?#[]@:!$&'()*+,;=")
|
|
47
|
+
UPPER_HEX = b"0123456789ABCDEF"
|
|
48
|
+
HOST_ALLOWED = frozenset(b"!$&'()*+,;=:[]<>\"")
|
|
49
|
+
VALID_ENCODED = frozenset(b"!$&'()*+,;=:@[]%")
|
|
50
|
+
USERINFO = re.compile(rb"\A[a-zA-Z0-9\-._:~!$&'()*+,;=%@]*\Z")
|
|
51
|
+
CONTROL = re.compile(rb"[\x00-\x1f\x7f]")
|
|
52
|
+
HEX = frozenset(b"0123456789abcdefABCDEF")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _should_escape(c, mode):
|
|
56
|
+
if 0x61 <= c <= 0x7A or 0x41 <= c <= 0x5A or 0x30 <= c <= 0x39:
|
|
57
|
+
return False
|
|
58
|
+
if mode == MODE_HOST and c in HOST_ALLOWED:
|
|
59
|
+
return False
|
|
60
|
+
if c in b"-_.~":
|
|
61
|
+
return False
|
|
62
|
+
if c in b"$&+,/:;=?@" and mode != MODE_HOST:
|
|
63
|
+
if mode == MODE_PATH:
|
|
64
|
+
return c == 0x3F # '?'
|
|
65
|
+
if mode == MODE_USER_PASSWORD:
|
|
66
|
+
return c in b"@/?:"
|
|
67
|
+
return False
|
|
68
|
+
if mode == MODE_FRAGMENT and c in b"!()*":
|
|
69
|
+
return False
|
|
70
|
+
return True
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
# escape decisions by byte, per mode
|
|
74
|
+
SHOULD_ESCAPE = tuple(tuple(_should_escape(c, m) for c in range(256)) for m in range(4))
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _unhex(c):
|
|
78
|
+
return c - 48 if c <= 57 else (c | 0x20) - 87
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def unescape(s, mode):
|
|
82
|
+
"""net/url unescape; raises UrlError on a malformed URL."""
|
|
83
|
+
count = 0
|
|
84
|
+
n = len(s)
|
|
85
|
+
table = SHOULD_ESCAPE[mode]
|
|
86
|
+
i = 0
|
|
87
|
+
while i < n:
|
|
88
|
+
c = s[i]
|
|
89
|
+
if c == 37:
|
|
90
|
+
count += 1
|
|
91
|
+
if i + 2 >= n or s[i + 1] not in HEX or s[i + 2] not in HEX:
|
|
92
|
+
raise UrlError("invalid URL escape")
|
|
93
|
+
# in the host, %-encoding is only for non-ASCII bytes (and %25)
|
|
94
|
+
if mode == MODE_HOST and _unhex(s[i + 1]) < 8 and s[i:i + 3] != b"%25":
|
|
95
|
+
raise UrlError("invalid URL escape")
|
|
96
|
+
i += 3
|
|
97
|
+
else:
|
|
98
|
+
if mode == MODE_HOST and c < 0x80 and table[c]:
|
|
99
|
+
raise UrlError("invalid character in host name")
|
|
100
|
+
i += 1
|
|
101
|
+
if count == 0:
|
|
102
|
+
return s
|
|
103
|
+
out = bytearray()
|
|
104
|
+
i = 0
|
|
105
|
+
while i < n:
|
|
106
|
+
c = s[i]
|
|
107
|
+
if c == 37:
|
|
108
|
+
out.append((_unhex(s[i + 1]) << 4) | _unhex(s[i + 2]))
|
|
109
|
+
i += 3
|
|
110
|
+
else:
|
|
111
|
+
out.append(c)
|
|
112
|
+
i += 1
|
|
113
|
+
return bytes(out)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def escape(s, mode):
|
|
117
|
+
table = SHOULD_ESCAPE[mode]
|
|
118
|
+
out = None
|
|
119
|
+
start = 0
|
|
120
|
+
for i, c in enumerate(s):
|
|
121
|
+
if table[c]:
|
|
122
|
+
if out is None:
|
|
123
|
+
out = bytearray()
|
|
124
|
+
out += s[start:i]
|
|
125
|
+
out.append(37)
|
|
126
|
+
out.append(UPPER_HEX[c >> 4])
|
|
127
|
+
out.append(UPPER_HEX[c & 15])
|
|
128
|
+
start = i + 1
|
|
129
|
+
if out is None:
|
|
130
|
+
return s
|
|
131
|
+
out += s[start:]
|
|
132
|
+
return bytes(out)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _valid_encoded(s, mode):
|
|
136
|
+
table = SHOULD_ESCAPE[mode]
|
|
137
|
+
for c in s:
|
|
138
|
+
if c in VALID_ENCODED:
|
|
139
|
+
continue
|
|
140
|
+
if table[c]:
|
|
141
|
+
return False
|
|
142
|
+
return True
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
# ----- parsing -----
|
|
146
|
+
|
|
147
|
+
def _get_scheme(raw):
|
|
148
|
+
for i, c in enumerate(raw):
|
|
149
|
+
if 0x61 <= (c | 0x20) <= 0x7A:
|
|
150
|
+
continue
|
|
151
|
+
if 0x30 <= c <= 0x39 or c == 43 or c == 45 or c == 46:
|
|
152
|
+
if i == 0:
|
|
153
|
+
return b"", raw
|
|
154
|
+
continue
|
|
155
|
+
if c == 58:
|
|
156
|
+
if i == 0:
|
|
157
|
+
raise UrlError("missing protocol scheme")
|
|
158
|
+
return raw[:i], raw[i + 1:]
|
|
159
|
+
return b"", raw
|
|
160
|
+
return b"", raw
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _set_path(u, p):
|
|
164
|
+
u.path = unescape(p, MODE_PATH)
|
|
165
|
+
u.raw_path = b"" if escape(u.path, MODE_PATH) == p else p
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _escaped_path(u):
|
|
169
|
+
if u.raw_path and _valid_encoded(u.raw_path, MODE_PATH):
|
|
170
|
+
try:
|
|
171
|
+
if unescape(u.raw_path, MODE_PATH) == u.path:
|
|
172
|
+
return u.raw_path
|
|
173
|
+
except UrlError:
|
|
174
|
+
pass
|
|
175
|
+
if u.path == b"*":
|
|
176
|
+
return b"*"
|
|
177
|
+
return escape(u.path, MODE_PATH)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _escaped_fragment(u):
|
|
181
|
+
if u.raw_fragment and _valid_encoded(u.raw_fragment, MODE_FRAGMENT):
|
|
182
|
+
try:
|
|
183
|
+
if unescape(u.raw_fragment, MODE_FRAGMENT) == u.fragment:
|
|
184
|
+
return u.raw_fragment
|
|
185
|
+
except UrlError:
|
|
186
|
+
pass
|
|
187
|
+
return escape(u.fragment, MODE_FRAGMENT)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _valid_optional_port(port):
|
|
191
|
+
if not port:
|
|
192
|
+
return True
|
|
193
|
+
if port[0] != 58:
|
|
194
|
+
return False
|
|
195
|
+
return all(48 <= c <= 57 for c in port[1:])
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _parse_host(host):
|
|
199
|
+
if host.startswith(b"["):
|
|
200
|
+
i = host.rfind(b"]")
|
|
201
|
+
if i < 0:
|
|
202
|
+
raise UrlError("missing ']' in host")
|
|
203
|
+
if not _valid_optional_port(host[i + 1:]):
|
|
204
|
+
raise UrlError("invalid port")
|
|
205
|
+
else:
|
|
206
|
+
i = host.rfind(b":")
|
|
207
|
+
if i >= 0 and not _valid_optional_port(host[i:]):
|
|
208
|
+
raise UrlError("invalid port")
|
|
209
|
+
return unescape(host, MODE_HOST)
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def _parse_authority(u, authority):
|
|
213
|
+
i = authority.rfind(b"@")
|
|
214
|
+
u.host = _parse_host(authority if i < 0 else authority[i + 1:])
|
|
215
|
+
if i < 0:
|
|
216
|
+
return
|
|
217
|
+
userinfo = authority[:i]
|
|
218
|
+
if not USERINFO.match(userinfo):
|
|
219
|
+
raise UrlError("invalid userinfo")
|
|
220
|
+
colon = userinfo.find(b":")
|
|
221
|
+
if colon < 0:
|
|
222
|
+
u.user = escape(unescape(userinfo, MODE_USER_PASSWORD), MODE_USER_PASSWORD)
|
|
223
|
+
else:
|
|
224
|
+
name = unescape(userinfo[:colon], MODE_USER_PASSWORD)
|
|
225
|
+
password = unescape(userinfo[colon + 1:], MODE_USER_PASSWORD)
|
|
226
|
+
u.user = escape(name, MODE_USER_PASSWORD) + b":" + escape(password, MODE_USER_PASSWORD)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def parse_url(raw_url):
|
|
230
|
+
"""net/url Parse."""
|
|
231
|
+
h = raw_url.find(b"#")
|
|
232
|
+
u = _parse_no_fragment(raw_url if h < 0 else raw_url[:h])
|
|
233
|
+
if h >= 0:
|
|
234
|
+
frag = raw_url[h + 1:]
|
|
235
|
+
u.fragment = unescape(frag, MODE_FRAGMENT)
|
|
236
|
+
u.raw_fragment = b"" if escape(u.fragment, MODE_FRAGMENT) == frag else frag
|
|
237
|
+
return u
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def _parse_no_fragment(raw):
|
|
241
|
+
if CONTROL.search(raw):
|
|
242
|
+
raise UrlError("invalid control character in URL")
|
|
243
|
+
u = Url()
|
|
244
|
+
if raw == b"*":
|
|
245
|
+
u.path = b"*"
|
|
246
|
+
return u
|
|
247
|
+
scheme, rest = _get_scheme(raw)
|
|
248
|
+
u.scheme = scheme.lower()
|
|
249
|
+
|
|
250
|
+
q = rest.find(b"?")
|
|
251
|
+
if q >= 0 and q == len(rest) - 1:
|
|
252
|
+
u.force_query = True
|
|
253
|
+
rest = rest[:q]
|
|
254
|
+
elif q >= 0:
|
|
255
|
+
u.raw_query = rest[q + 1:]
|
|
256
|
+
rest = rest[:q]
|
|
257
|
+
|
|
258
|
+
if not rest.startswith(b"/"):
|
|
259
|
+
if u.scheme:
|
|
260
|
+
# rootless paths are opaque (RFC 3986)
|
|
261
|
+
u.opaque = rest
|
|
262
|
+
return u
|
|
263
|
+
# a relative-path reference: the first segment cannot contain ':'
|
|
264
|
+
slash = rest.find(b"/")
|
|
265
|
+
first = rest if slash < 0 else rest[:slash]
|
|
266
|
+
if b":" in first:
|
|
267
|
+
raise UrlError("first path segment in URL cannot contain colon")
|
|
268
|
+
|
|
269
|
+
if (u.scheme or not rest.startswith(b"///")) and rest.startswith(b"//"):
|
|
270
|
+
authority = rest[2:]
|
|
271
|
+
rest = b""
|
|
272
|
+
slash = authority.find(b"/")
|
|
273
|
+
if slash >= 0:
|
|
274
|
+
rest = authority[slash:]
|
|
275
|
+
authority = authority[:slash]
|
|
276
|
+
_parse_authority(u, authority)
|
|
277
|
+
_set_path(u, rest)
|
|
278
|
+
return u
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def _resolve_path(base, ref):
|
|
282
|
+
if not ref:
|
|
283
|
+
full = base
|
|
284
|
+
elif ref[0] != 47:
|
|
285
|
+
slash = base.rfind(b"/")
|
|
286
|
+
full = (b"" if slash < 0 else base[:slash + 1]) + ref
|
|
287
|
+
else:
|
|
288
|
+
full = ref
|
|
289
|
+
if not full:
|
|
290
|
+
return b""
|
|
291
|
+
dst = bytearray(b"/")
|
|
292
|
+
first = True
|
|
293
|
+
remaining = full
|
|
294
|
+
found = True
|
|
295
|
+
elem = b""
|
|
296
|
+
while found:
|
|
297
|
+
slash = remaining.find(b"/")
|
|
298
|
+
found = slash >= 0
|
|
299
|
+
elem = remaining[:slash] if found else remaining
|
|
300
|
+
remaining = remaining[slash + 1:] if found else b""
|
|
301
|
+
if elem == b".":
|
|
302
|
+
first = False
|
|
303
|
+
continue
|
|
304
|
+
if elem == b"..":
|
|
305
|
+
s = bytes(dst[1:])
|
|
306
|
+
index = s.rfind(b"/")
|
|
307
|
+
dst = bytearray(b"/")
|
|
308
|
+
if index < 0:
|
|
309
|
+
first = True
|
|
310
|
+
else:
|
|
311
|
+
dst += s[:index]
|
|
312
|
+
else:
|
|
313
|
+
if not first:
|
|
314
|
+
dst.append(47)
|
|
315
|
+
dst += elem
|
|
316
|
+
first = False
|
|
317
|
+
if elem == b"." or elem == b"..":
|
|
318
|
+
dst.append(47)
|
|
319
|
+
if len(dst) > 1 and dst[1] == 47:
|
|
320
|
+
dst = dst[1:]
|
|
321
|
+
return bytes(dst)
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def resolve_reference(base, ref):
|
|
325
|
+
"""net/url ResolveReference."""
|
|
326
|
+
url = ref.copy()
|
|
327
|
+
if not ref.scheme:
|
|
328
|
+
url.scheme = base.scheme
|
|
329
|
+
if ref.scheme or ref.host or ref.user is not None:
|
|
330
|
+
_set_path(url, _resolve_path(_escaped_path(ref), b""))
|
|
331
|
+
return url
|
|
332
|
+
if ref.opaque:
|
|
333
|
+
url.user = None
|
|
334
|
+
url.host = b""
|
|
335
|
+
url.path = b""
|
|
336
|
+
return url
|
|
337
|
+
if not ref.path and not ref.force_query and not ref.raw_query:
|
|
338
|
+
url.raw_query = base.raw_query
|
|
339
|
+
if not ref.fragment:
|
|
340
|
+
url.fragment = base.fragment
|
|
341
|
+
url.raw_fragment = base.raw_fragment
|
|
342
|
+
if not ref.path and base.opaque:
|
|
343
|
+
url.opaque = base.opaque
|
|
344
|
+
url.user = None
|
|
345
|
+
url.host = b""
|
|
346
|
+
url.path = b""
|
|
347
|
+
return url
|
|
348
|
+
url.host = base.host
|
|
349
|
+
url.user = base.user
|
|
350
|
+
_set_path(url, _resolve_path(_escaped_path(base), _escaped_path(ref)))
|
|
351
|
+
return url
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def _split_host_port(host_port):
|
|
355
|
+
host = host_port
|
|
356
|
+
port = b""
|
|
357
|
+
colon = host.rfind(b":")
|
|
358
|
+
if colon >= 0 and _valid_optional_port(host[colon:]):
|
|
359
|
+
port = host[colon + 1:]
|
|
360
|
+
host = host[:colon]
|
|
361
|
+
if len(host) >= 2 and host[0] == 91 and host[-1] == 93:
|
|
362
|
+
host = host[1:-1]
|
|
363
|
+
return host, port
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def _valid_utf8(b):
|
|
367
|
+
try:
|
|
368
|
+
b.decode("utf-8")
|
|
369
|
+
return True
|
|
370
|
+
except UnicodeDecodeError:
|
|
371
|
+
return False
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def _unescape_for_display(s):
|
|
375
|
+
"""Decodes %XX escapes except those of reserved and control characters; escaped bytes of non-ASCII text are
|
|
376
|
+
decoded when they form valid UTF-8."""
|
|
377
|
+
if b"%" not in s:
|
|
378
|
+
return s
|
|
379
|
+
out = bytearray()
|
|
380
|
+
n = len(s)
|
|
381
|
+
i = 0
|
|
382
|
+
while i < n:
|
|
383
|
+
if s[i] != 37 or i + 2 >= n or s[i + 1] not in HEX or s[i + 2] not in HEX:
|
|
384
|
+
out.append(s[i])
|
|
385
|
+
i += 1
|
|
386
|
+
continue
|
|
387
|
+
c = (_unhex(s[i + 1]) << 4) | _unhex(s[i + 2])
|
|
388
|
+
if c < 0x80:
|
|
389
|
+
if c < 0x20 or c == 0x7F or c in RESERVED_FOR_DISPLAY:
|
|
390
|
+
out += s[i:i + 3]
|
|
391
|
+
else:
|
|
392
|
+
out.append(c)
|
|
393
|
+
i += 3
|
|
394
|
+
continue
|
|
395
|
+
# a run of escaped bytes >= 0x80: decoded together when it is valid UTF-8
|
|
396
|
+
j = i
|
|
397
|
+
run = bytearray()
|
|
398
|
+
while j + 2 < n and s[j] == 37 and s[j + 1] in HEX and s[j + 2] in HEX:
|
|
399
|
+
v = (_unhex(s[j + 1]) << 4) | _unhex(s[j + 2])
|
|
400
|
+
if v < 0x80:
|
|
401
|
+
break
|
|
402
|
+
run.append(v)
|
|
403
|
+
j += 3
|
|
404
|
+
out += run if _valid_utf8(bytes(run)) else s[i:j]
|
|
405
|
+
i = j
|
|
406
|
+
return bytes(out)
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
def display_string(u):
|
|
410
|
+
"""Writes a resolved URL as the .NET version does (System.Uri.ToString): the scheme and host in lower case, no
|
|
411
|
+
default port, an empty path as "/", and percent-escapes decoded except those of the characters that delimit URL
|
|
412
|
+
parts ("%2F" stays) and of control characters; non-ASCII text is written as is."""
|
|
413
|
+
scheme = u.scheme.lower()
|
|
414
|
+
b = bytearray(scheme + b":")
|
|
415
|
+
if not u.opaque:
|
|
416
|
+
if u.host or u.user is not None:
|
|
417
|
+
b += b"//"
|
|
418
|
+
if u.user is not None:
|
|
419
|
+
b += u.user + b"@"
|
|
420
|
+
host, port = _split_host_port(u.host)
|
|
421
|
+
b += host.decode("utf-8", "replace").lower().encode("utf-8")
|
|
422
|
+
if port and port != DEFAULT_PORTS.get(scheme):
|
|
423
|
+
b += b":" + port
|
|
424
|
+
path = _escaped_path(u)
|
|
425
|
+
if not path and u.host:
|
|
426
|
+
path = b"/"
|
|
427
|
+
b += _unescape_for_display(path)
|
|
428
|
+
else:
|
|
429
|
+
b += u.opaque
|
|
430
|
+
if u.raw_query or u.force_query:
|
|
431
|
+
b += b"?" + _unescape_for_display(u.raw_query)
|
|
432
|
+
if u.fragment:
|
|
433
|
+
b += b"#" + _unescape_for_display(_escaped_fragment(u))
|
|
434
|
+
return bytes(b).decode("utf-8", "replace")
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
def _b(s):
|
|
438
|
+
return s.encode("utf-8", "surrogatepass")
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
def absolute_url(s):
|
|
442
|
+
"""Parses a page URL with its own scheme and host; None for anything else."""
|
|
443
|
+
if not s:
|
|
444
|
+
return None
|
|
445
|
+
try:
|
|
446
|
+
u = parse_url(_b(s))
|
|
447
|
+
except UrlError:
|
|
448
|
+
return None
|
|
449
|
+
if not u.scheme or (not u.host and not u.opaque and u.scheme != b"file"):
|
|
450
|
+
return None
|
|
451
|
+
return u
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
def _absolute_link(link):
|
|
455
|
+
"""A link with its own scheme ("https://...", "mailto:...", "file:..."). "/path" and "//host/path" are
|
|
456
|
+
relative."""
|
|
457
|
+
try:
|
|
458
|
+
return bool(parse_url(_b(trim_space(link))).scheme)
|
|
459
|
+
except UrlError:
|
|
460
|
+
return False
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def correct_link(raw, page_url, base):
|
|
464
|
+
"""The raw attribute value with its character references decoded and made absolute against the page.
|
|
465
|
+
Percent-escapes are part of the URL and are kept. An empty link gives the page URL."""
|
|
466
|
+
if not raw:
|
|
467
|
+
return page_url
|
|
468
|
+
link = dec.decode(raw, True) if "&" in raw or "\r" in raw else raw
|
|
469
|
+
if base is None or _absolute_link(link):
|
|
470
|
+
return link
|
|
471
|
+
try:
|
|
472
|
+
return display_string(resolve_reference(base, parse_url(_b(trim_space(link)))))
|
|
473
|
+
except UrlError:
|
|
474
|
+
return link
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
def after_scheme(raw_href, scheme):
|
|
478
|
+
"""Everything after the first ':' of a link with the scheme (mailto, tel), or ""."""
|
|
479
|
+
href = trim_space(dec.decode(raw_href, True))
|
|
480
|
+
if len(href) < len(scheme) or ascii_lower(href[:len(scheme)]) != scheme:
|
|
481
|
+
return ""
|
|
482
|
+
i = href.find(":")
|
|
483
|
+
return "" if i < 0 else href[i + 1:]
|