sdo-client 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdo/__init__.py +74 -0
- sdo/_decompress.py +253 -0
- sdo/_encoding.py +739 -0
- sdo/_json.py +50 -0
- sdo/_logging.py +93 -0
- sdo/_secret.py +74 -0
- sdo/_transport.py +639 -0
- sdo/_urls.py +80 -0
- sdo/_version.py +8 -0
- sdo/_wire.py +126 -0
- sdo/client.py +459 -0
- sdo/config.py +452 -0
- sdo/errors.py +247 -0
- sdo/jobs.py +490 -0
- sdo/plugins.py +143 -0
- sdo/py.typed +0 -0
- sdo/response.py +177 -0
- sdo/usage.py +62 -0
- sdo_client-0.1.0.dist-info/METADATA +377 -0
- sdo_client-0.1.0.dist-info/RECORD +23 -0
- sdo_client-0.1.0.dist-info/WHEEL +4 -0
- sdo_client-0.1.0.dist-info/licenses/LICENSE +201 -0
- sdo_client-0.1.0.dist-info/licenses/NOTICE +8 -0
sdo/_encoding.py
ADDED
|
@@ -0,0 +1,739 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""The WHATWG Encoding Standard "decode" ladder for ``ScrapeResult.text``.
|
|
3
|
+
|
|
4
|
+
JavaScript (``TextDecoder``) and Go (``htmlindex``) accept only the labels of the
|
|
5
|
+
Encoding Standard; Python's ``codecs`` accepts hundreds of names, including
|
|
6
|
+
codecs that are not text encodings at all and one (``punycode``) whose decoder
|
|
7
|
+
is quadratic. A page controls the label it declares, so a label is never passed
|
|
8
|
+
to ``codecs`` directly: it is resolved through the WHATWG table below and
|
|
9
|
+
mapped to a Python codec (or a decode table) chosen here.
|
|
10
|
+
|
|
11
|
+
Ladder: BOM > Content-Type ``charset`` > ``*/json`` > HTML ``<meta>`` in the
|
|
12
|
+
first 1024 bytes > UTF-8. An unknown label, or one that maps to WHATWG
|
|
13
|
+
"replacement", is ignored and the ladder continues.
|
|
14
|
+
|
|
15
|
+
Exactness: UTF-8, UTF-16, every single-byte encoding and gb18030/GBK follow the
|
|
16
|
+
WHATWG decoders byte for byte. Big5, EUC-JP, ISO-2022-JP, Shift_JIS and EUC-KR
|
|
17
|
+
use the closest Python codec: well-formed text decodes as WHATWG specifies
|
|
18
|
+
(except 203 HKSCS-2008 Big5 additions Python does not map), but recovery from
|
|
19
|
+
malformed bytes may place U+FFFD differently.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import re
|
|
25
|
+
|
|
26
|
+
# -- labels -> WHATWG encoding names (https://encoding.spec.whatwg.org/#names-and-labels)
|
|
27
|
+
|
|
28
|
+
_LABELS_BY_ENCODING: dict[str, tuple[str, ...]] = {
|
|
29
|
+
"utf-8": (
|
|
30
|
+
"unicode-1-1-utf-8",
|
|
31
|
+
"unicode11utf8",
|
|
32
|
+
"unicode20utf8",
|
|
33
|
+
"utf-8",
|
|
34
|
+
"utf8",
|
|
35
|
+
"x-unicode20utf8",
|
|
36
|
+
),
|
|
37
|
+
"ibm866": ("866", "cp866", "csibm866", "ibm866"),
|
|
38
|
+
"iso-8859-2": (
|
|
39
|
+
"csisolatin2",
|
|
40
|
+
"iso-8859-2",
|
|
41
|
+
"iso-ir-101",
|
|
42
|
+
"iso8859-2",
|
|
43
|
+
"iso88592",
|
|
44
|
+
"iso_8859-2",
|
|
45
|
+
"iso_8859-2:1987",
|
|
46
|
+
"l2",
|
|
47
|
+
"latin2",
|
|
48
|
+
),
|
|
49
|
+
"iso-8859-3": (
|
|
50
|
+
"csisolatin3",
|
|
51
|
+
"iso-8859-3",
|
|
52
|
+
"iso-ir-109",
|
|
53
|
+
"iso8859-3",
|
|
54
|
+
"iso88593",
|
|
55
|
+
"iso_8859-3",
|
|
56
|
+
"iso_8859-3:1988",
|
|
57
|
+
"l3",
|
|
58
|
+
"latin3",
|
|
59
|
+
),
|
|
60
|
+
"iso-8859-4": (
|
|
61
|
+
"csisolatin4",
|
|
62
|
+
"iso-8859-4",
|
|
63
|
+
"iso-ir-110",
|
|
64
|
+
"iso8859-4",
|
|
65
|
+
"iso88594",
|
|
66
|
+
"iso_8859-4",
|
|
67
|
+
"iso_8859-4:1988",
|
|
68
|
+
"l4",
|
|
69
|
+
"latin4",
|
|
70
|
+
),
|
|
71
|
+
"iso-8859-5": (
|
|
72
|
+
"csisolatincyrillic",
|
|
73
|
+
"cyrillic",
|
|
74
|
+
"iso-8859-5",
|
|
75
|
+
"iso-ir-144",
|
|
76
|
+
"iso8859-5",
|
|
77
|
+
"iso88595",
|
|
78
|
+
"iso_8859-5",
|
|
79
|
+
"iso_8859-5:1988",
|
|
80
|
+
),
|
|
81
|
+
"iso-8859-6": (
|
|
82
|
+
"arabic",
|
|
83
|
+
"asmo-708",
|
|
84
|
+
"csiso88596e",
|
|
85
|
+
"csiso88596i",
|
|
86
|
+
"csisolatinarabic",
|
|
87
|
+
"ecma-114",
|
|
88
|
+
"iso-8859-6",
|
|
89
|
+
"iso-8859-6-e",
|
|
90
|
+
"iso-8859-6-i",
|
|
91
|
+
"iso-ir-127",
|
|
92
|
+
"iso8859-6",
|
|
93
|
+
"iso88596",
|
|
94
|
+
"iso_8859-6",
|
|
95
|
+
"iso_8859-6:1987",
|
|
96
|
+
),
|
|
97
|
+
"iso-8859-7": (
|
|
98
|
+
"csisolatingreek",
|
|
99
|
+
"ecma-118",
|
|
100
|
+
"elot_928",
|
|
101
|
+
"greek",
|
|
102
|
+
"greek8",
|
|
103
|
+
"iso-8859-7",
|
|
104
|
+
"iso-ir-126",
|
|
105
|
+
"iso8859-7",
|
|
106
|
+
"iso88597",
|
|
107
|
+
"iso_8859-7",
|
|
108
|
+
"iso_8859-7:1987",
|
|
109
|
+
"sun_eu_greek",
|
|
110
|
+
),
|
|
111
|
+
"iso-8859-8": (
|
|
112
|
+
"csiso88598e",
|
|
113
|
+
"csisolatinhebrew",
|
|
114
|
+
"hebrew",
|
|
115
|
+
"iso-8859-8",
|
|
116
|
+
"iso-8859-8-e",
|
|
117
|
+
"iso-ir-138",
|
|
118
|
+
"iso8859-8",
|
|
119
|
+
"iso88598",
|
|
120
|
+
"iso_8859-8",
|
|
121
|
+
"iso_8859-8:1988",
|
|
122
|
+
"visual",
|
|
123
|
+
),
|
|
124
|
+
"iso-8859-8-i": ("csiso88598i", "iso-8859-8-i", "logical"),
|
|
125
|
+
"iso-8859-10": (
|
|
126
|
+
"csisolatin6",
|
|
127
|
+
"iso-8859-10",
|
|
128
|
+
"iso-ir-157",
|
|
129
|
+
"iso8859-10",
|
|
130
|
+
"iso885910",
|
|
131
|
+
"l6",
|
|
132
|
+
"latin6",
|
|
133
|
+
),
|
|
134
|
+
"iso-8859-13": ("iso-8859-13", "iso8859-13", "iso885913"),
|
|
135
|
+
"iso-8859-14": ("iso-8859-14", "iso8859-14", "iso885914"),
|
|
136
|
+
"iso-8859-15": (
|
|
137
|
+
"csisolatin9",
|
|
138
|
+
"iso-8859-15",
|
|
139
|
+
"iso8859-15",
|
|
140
|
+
"iso885915",
|
|
141
|
+
"iso_8859-15",
|
|
142
|
+
"l9",
|
|
143
|
+
),
|
|
144
|
+
"iso-8859-16": ("iso-8859-16",),
|
|
145
|
+
"koi8-r": ("cskoi8r", "koi", "koi8", "koi8-r", "koi8_r"),
|
|
146
|
+
"koi8-u": ("koi8-ru", "koi8-u"),
|
|
147
|
+
"macintosh": ("csmacintosh", "mac", "macintosh", "x-mac-roman"),
|
|
148
|
+
"windows-874": (
|
|
149
|
+
"dos-874",
|
|
150
|
+
"iso-8859-11",
|
|
151
|
+
"iso8859-11",
|
|
152
|
+
"iso885911",
|
|
153
|
+
"tis-620",
|
|
154
|
+
"windows-874",
|
|
155
|
+
),
|
|
156
|
+
"windows-1250": ("cp1250", "windows-1250", "x-cp1250"),
|
|
157
|
+
"windows-1251": ("cp1251", "windows-1251", "x-cp1251"),
|
|
158
|
+
"windows-1252": (
|
|
159
|
+
"ansi_x3.4-1968",
|
|
160
|
+
"ascii",
|
|
161
|
+
"cp1252",
|
|
162
|
+
"cp819",
|
|
163
|
+
"csisolatin1",
|
|
164
|
+
"ibm819",
|
|
165
|
+
"iso-8859-1",
|
|
166
|
+
"iso-ir-100",
|
|
167
|
+
"iso8859-1",
|
|
168
|
+
"iso88591",
|
|
169
|
+
"iso_8859-1",
|
|
170
|
+
"iso_8859-1:1987",
|
|
171
|
+
"l1",
|
|
172
|
+
"latin1",
|
|
173
|
+
"us-ascii",
|
|
174
|
+
"windows-1252",
|
|
175
|
+
"x-cp1252",
|
|
176
|
+
),
|
|
177
|
+
"windows-1253": ("cp1253", "windows-1253", "x-cp1253"),
|
|
178
|
+
"windows-1254": (
|
|
179
|
+
"cp1254",
|
|
180
|
+
"csisolatin5",
|
|
181
|
+
"iso-8859-9",
|
|
182
|
+
"iso-ir-148",
|
|
183
|
+
"iso8859-9",
|
|
184
|
+
"iso88599",
|
|
185
|
+
"iso_8859-9",
|
|
186
|
+
"iso_8859-9:1989",
|
|
187
|
+
"l5",
|
|
188
|
+
"latin5",
|
|
189
|
+
"windows-1254",
|
|
190
|
+
"x-cp1254",
|
|
191
|
+
),
|
|
192
|
+
"windows-1255": ("cp1255", "windows-1255", "x-cp1255"),
|
|
193
|
+
"windows-1256": ("cp1256", "windows-1256", "x-cp1256"),
|
|
194
|
+
"windows-1257": ("cp1257", "windows-1257", "x-cp1257"),
|
|
195
|
+
"windows-1258": ("cp1258", "windows-1258", "x-cp1258"),
|
|
196
|
+
"x-mac-cyrillic": ("x-mac-cyrillic", "x-mac-ukrainian"),
|
|
197
|
+
"gbk": (
|
|
198
|
+
"chinese",
|
|
199
|
+
"csgb2312",
|
|
200
|
+
"csiso58gb231280",
|
|
201
|
+
"gb2312",
|
|
202
|
+
"gb_2312",
|
|
203
|
+
"gb_2312-80",
|
|
204
|
+
"gbk",
|
|
205
|
+
"iso-ir-58",
|
|
206
|
+
"x-gbk",
|
|
207
|
+
),
|
|
208
|
+
"gb18030": ("gb18030",),
|
|
209
|
+
"big5": ("big5", "big5-hkscs", "cn-big5", "csbig5", "x-x-big5"),
|
|
210
|
+
"euc-jp": ("cseucpkdfmtjapanese", "euc-jp", "x-euc-jp"),
|
|
211
|
+
"iso-2022-jp": ("csiso2022jp", "iso-2022-jp"),
|
|
212
|
+
"shift_jis": (
|
|
213
|
+
"csshiftjis",
|
|
214
|
+
"ms932",
|
|
215
|
+
"ms_kanji",
|
|
216
|
+
"shift-jis",
|
|
217
|
+
"shift_jis",
|
|
218
|
+
"sjis",
|
|
219
|
+
"windows-31j",
|
|
220
|
+
"x-sjis",
|
|
221
|
+
),
|
|
222
|
+
"euc-kr": (
|
|
223
|
+
"cseuckr",
|
|
224
|
+
"csksc56011987",
|
|
225
|
+
"euc-kr",
|
|
226
|
+
"iso-ir-149",
|
|
227
|
+
"korean",
|
|
228
|
+
"ks_c_5601-1987",
|
|
229
|
+
"ks_c_5601-1989",
|
|
230
|
+
"ksc5601",
|
|
231
|
+
"ksc_5601",
|
|
232
|
+
"windows-949",
|
|
233
|
+
),
|
|
234
|
+
"replacement": (
|
|
235
|
+
"csiso2022kr",
|
|
236
|
+
"hz-gb-2312",
|
|
237
|
+
"iso-2022-cn",
|
|
238
|
+
"iso-2022-cn-ext",
|
|
239
|
+
"iso-2022-kr",
|
|
240
|
+
"replacement",
|
|
241
|
+
),
|
|
242
|
+
"utf-16be": ("unicodefffe", "utf-16be"),
|
|
243
|
+
"utf-16le": (
|
|
244
|
+
"csunicode",
|
|
245
|
+
"iso-10646-ucs-2",
|
|
246
|
+
"ucs-2",
|
|
247
|
+
"unicode",
|
|
248
|
+
"unicodefeff",
|
|
249
|
+
"utf-16",
|
|
250
|
+
"utf-16le",
|
|
251
|
+
),
|
|
252
|
+
"x-user-defined": ("x-user-defined",),
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
_LABELS: dict[str, str] = {
|
|
256
|
+
label: enc for enc, labels in _LABELS_BY_ENCODING.items() for label in labels
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
# -- WHATWG encoding -> how Python decodes it --------------------------------
|
|
260
|
+
|
|
261
|
+
#: Multi-byte encodings: the Python codec whose decoder matches the WHATWG one.
|
|
262
|
+
#: WHATWG's decoders are supersets of the legacy standards their labels name.
|
|
263
|
+
_MULTI_BYTE = {
|
|
264
|
+
"utf-8": "utf-8",
|
|
265
|
+
"big5": "big5hkscs",
|
|
266
|
+
"euc-jp": "euc_jp",
|
|
267
|
+
"iso-2022-jp": "iso2022_jp_ext",
|
|
268
|
+
"shift_jis": "cp932",
|
|
269
|
+
"euc-kr": "cp949",
|
|
270
|
+
"utf-16be": "utf-16-be",
|
|
271
|
+
"utf-16le": "utf-16-le",
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
#: Single-byte encodings: the Python codec to start the 256-entry decode table
|
|
275
|
+
#: from, before the WHATWG corrections below.
|
|
276
|
+
_SINGLE_BYTE = {
|
|
277
|
+
"ibm866": "cp866",
|
|
278
|
+
"iso-8859-2": "iso8859_2",
|
|
279
|
+
"iso-8859-3": "iso8859_3",
|
|
280
|
+
"iso-8859-4": "iso8859_4",
|
|
281
|
+
"iso-8859-5": "iso8859_5",
|
|
282
|
+
"iso-8859-6": "iso8859_6",
|
|
283
|
+
"iso-8859-7": "iso8859_7",
|
|
284
|
+
"iso-8859-8": "iso8859_8",
|
|
285
|
+
"iso-8859-8-i": "iso8859_8",
|
|
286
|
+
"iso-8859-10": "iso8859_10",
|
|
287
|
+
"iso-8859-13": "iso8859_13",
|
|
288
|
+
"iso-8859-14": "iso8859_14",
|
|
289
|
+
"iso-8859-15": "iso8859_15",
|
|
290
|
+
"iso-8859-16": "iso8859_16",
|
|
291
|
+
"koi8-r": "koi8_r",
|
|
292
|
+
"koi8-u": "koi8_u",
|
|
293
|
+
"macintosh": "mac_roman",
|
|
294
|
+
"windows-874": "cp874",
|
|
295
|
+
"windows-1250": "cp1250",
|
|
296
|
+
"windows-1251": "cp1251",
|
|
297
|
+
"windows-1252": "cp1252",
|
|
298
|
+
"windows-1253": "cp1253",
|
|
299
|
+
"windows-1254": "cp1254",
|
|
300
|
+
"windows-1255": "cp1255",
|
|
301
|
+
"windows-1256": "cp1256",
|
|
302
|
+
"windows-1257": "cp1257",
|
|
303
|
+
"windows-1258": "cp1258",
|
|
304
|
+
"x-mac-cyrillic": "mac_cyrillic",
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
#: Bytes where the WHATWG index differs from the Python codec (beyond the
|
|
308
|
+
#: windows-* C1 rule applied in ``_single_byte_table``).
|
|
309
|
+
_OVERRIDES = {
|
|
310
|
+
"koi8-u": {0xAE: "ў", 0xBE: "Ў"}, # WHATWG KOI8-U is KOI8-RU
|
|
311
|
+
"windows-1255": {0xCA: "ֺ"},
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
_UNMAPPED = "\ufffd" # a byte the WHATWG index leaves unmapped
|
|
315
|
+
_TABLES: dict[str, str] = {}
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def _single_byte_table(encoding: str) -> str:
|
|
319
|
+
table = _TABLES.get(encoding)
|
|
320
|
+
if table is None:
|
|
321
|
+
codec = _SINGLE_BYTE[encoding]
|
|
322
|
+
chars = []
|
|
323
|
+
for b in range(256):
|
|
324
|
+
try:
|
|
325
|
+
chars.append(bytes([b]).decode(codec))
|
|
326
|
+
except UnicodeDecodeError:
|
|
327
|
+
# A windows-* hole in 0x80-0x9F decodes to the C1 control of the
|
|
328
|
+
# same value in WHATWG (and browsers); cp1252 leaves it unmapped.
|
|
329
|
+
chr_c1 = chr(b) if encoding.startswith("windows-") and b < 0xA0 else _UNMAPPED
|
|
330
|
+
chars.append(chr_c1)
|
|
331
|
+
for b, ch in _OVERRIDES.get(encoding, {}).items():
|
|
332
|
+
chars[b] = ch
|
|
333
|
+
table = _TABLES[encoding] = "".join(chars)
|
|
334
|
+
return table
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
#: x-user-defined: ASCII, then bytes 0x80-0xFF map to U+F780-U+F7FF.
|
|
338
|
+
_X_USER_DEFINED = "".join(chr(b) for b in range(0x80)) + "".join(
|
|
339
|
+
chr(0xF780 + b) for b in range(0x80)
|
|
340
|
+
)
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
# -- gb18030 (and GBK, which WHATWG decodes with the same decoder) -----------
|
|
344
|
+
#
|
|
345
|
+
# Python's gb18030 codec agrees with the WHATWG decoder on every well-formed
|
|
346
|
+
# sequence except the 21 below (GB18030-2022 moved them off the Private Use
|
|
347
|
+
# Area), but not on the single byte 0x80 (WHATWG: the euro sign) nor on how it
|
|
348
|
+
# recovers from a broken sequence. So the bytes are split here, per the WHATWG
|
|
349
|
+
# algorithm: runs of well-formed sequences go to the codec in one call, and
|
|
350
|
+
# everything else is decoded step by step.
|
|
351
|
+
|
|
352
|
+
_GB_OVERRIDES = {
|
|
353
|
+
b"\xa3\xa0": "\u3000",
|
|
354
|
+
b"\xa6\xd9": "\ufe10",
|
|
355
|
+
b"\xa6\xda": "\ufe12",
|
|
356
|
+
b"\xa6\xdb": "\ufe11",
|
|
357
|
+
b"\xa6\xdc": "\ufe13",
|
|
358
|
+
b"\xa6\xdd": "\ufe14",
|
|
359
|
+
b"\xa6\xde": "\ufe15",
|
|
360
|
+
b"\xa6\xdf": "\ufe16",
|
|
361
|
+
b"\xa6\xec": "\ufe17",
|
|
362
|
+
b"\xa6\xed": "\ufe18",
|
|
363
|
+
b"\xa6\xf3": "\ufe19",
|
|
364
|
+
b"\xa8\xbc": "\u1e3f",
|
|
365
|
+
b"\xfe\x59": "\u9fb4",
|
|
366
|
+
b"\xfe\x61": "\u9fb5",
|
|
367
|
+
b"\xfe\x66": "\u9fb6",
|
|
368
|
+
b"\xfe\x67": "\u9fb7",
|
|
369
|
+
b"\xfe\x6d": "\u9fb8",
|
|
370
|
+
b"\xfe\x7e": "\u9fb9",
|
|
371
|
+
b"\xfe\x90": "\u9fba",
|
|
372
|
+
b"\xfe\xa0": "\u9fbb",
|
|
373
|
+
b"\x81\x35\xf4\x37": "\ue7c7",
|
|
374
|
+
}
|
|
375
|
+
#: A run of well-formed sequences that are not overrides: ASCII, a two-byte
|
|
376
|
+
#: sequence, or a four-byte sequence (the second byte tells them apart). The
|
|
377
|
+
#: repeat is possessive: a greedy one keeps a backtrack point per sequence,
|
|
378
|
+
#: over a hundred times the body's size in memory for a long ASCII run.
|
|
379
|
+
_GB_RUN = re.compile(
|
|
380
|
+
rb"(?:[\x00-\x7f]"
|
|
381
|
+
rb"|(?!\xa3\xa0|\xa6[\xd9-\xdf\xec\xed\xf3]|\xa8\xbc|\xfe[\x59\x61\x66\x67\x6d\x7e\x90\xa0])"
|
|
382
|
+
rb"[\x81-\xfe][\x40-\x7e\x80-\xfe]"
|
|
383
|
+
rb"|(?!\x81\x35\xf4\x37)[\x81-\xfe][\x30-\x39][\x81-\xfe][\x30-\x39])++"
|
|
384
|
+
)
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def _gb_sequence(seq: bytes) -> str:
|
|
388
|
+
"""One well-formed sequence: an override, the codec's answer, or U+FFFD for
|
|
389
|
+
a four-byte pointer WHATWG leaves unmapped."""
|
|
390
|
+
if seq in _GB_OVERRIDES:
|
|
391
|
+
return _GB_OVERRIDES[seq]
|
|
392
|
+
try:
|
|
393
|
+
return seq.decode("gb18030")
|
|
394
|
+
except UnicodeDecodeError:
|
|
395
|
+
return "\ufffd"
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
def _gb_step(data: bytes, i: int) -> tuple[str, int]:
|
|
399
|
+
"""Decode the sequence at ``data[i]`` the WHATWG way: ``(text, consumed)``.
|
|
400
|
+
A byte that ends a broken sequence is not consumed when it can start the
|
|
401
|
+
next one."""
|
|
402
|
+
n = len(data)
|
|
403
|
+
first = data[i]
|
|
404
|
+
if first < 0x80:
|
|
405
|
+
return chr(first), 1
|
|
406
|
+
if first == 0x80:
|
|
407
|
+
return "\u20ac", 1
|
|
408
|
+
if first == 0xFF or i + 1 == n:
|
|
409
|
+
return "\ufffd", 1
|
|
410
|
+
second = data[i + 1]
|
|
411
|
+
if 0x30 <= second <= 0x39:
|
|
412
|
+
if i + 2 == n:
|
|
413
|
+
return "\ufffd", 2
|
|
414
|
+
if not 0x81 <= data[i + 2] <= 0xFE:
|
|
415
|
+
return "\ufffd", 1
|
|
416
|
+
if i + 3 == n:
|
|
417
|
+
return "\ufffd", 3
|
|
418
|
+
if not 0x30 <= data[i + 3] <= 0x39:
|
|
419
|
+
return "\ufffd", 1
|
|
420
|
+
return _gb_sequence(data[i : i + 4]), 4
|
|
421
|
+
if 0x40 <= second <= 0x7E or 0x80 <= second <= 0xFE:
|
|
422
|
+
return _gb_sequence(data[i : i + 2]), 2
|
|
423
|
+
return "\ufffd", 1 if second < 0x80 else 2
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def _decode_gb18030(data: bytes) -> str:
|
|
427
|
+
out: list[str] = []
|
|
428
|
+
i, n = 0, len(data)
|
|
429
|
+
while i < n:
|
|
430
|
+
run = _GB_RUN.match(data, i)
|
|
431
|
+
if run:
|
|
432
|
+
end = run.end()
|
|
433
|
+
try:
|
|
434
|
+
out.append(data[i:end].decode("gb18030"))
|
|
435
|
+
i = end
|
|
436
|
+
continue
|
|
437
|
+
except UnicodeDecodeError: # an unmapped four-byte pointer inside
|
|
438
|
+
while i < end:
|
|
439
|
+
text, used = _gb_step(data, i)
|
|
440
|
+
out.append(text)
|
|
441
|
+
i += used
|
|
442
|
+
continue
|
|
443
|
+
text, used = _gb_step(data, i)
|
|
444
|
+
out.append(text)
|
|
445
|
+
i += used
|
|
446
|
+
return "".join(out)
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
def _decode_as(data: bytes, encoding: str) -> str:
|
|
450
|
+
if encoding in ("gbk", "gb18030"):
|
|
451
|
+
return _decode_gb18030(data)
|
|
452
|
+
if encoding in _MULTI_BYTE:
|
|
453
|
+
return data.decode(_MULTI_BYTE[encoding], errors="replace")
|
|
454
|
+
if encoding == "x-user-defined":
|
|
455
|
+
table = _X_USER_DEFINED
|
|
456
|
+
else:
|
|
457
|
+
table = _single_byte_table(encoding)
|
|
458
|
+
# Every code point of a latin-1 decode is below 256: an index into the table.
|
|
459
|
+
return data.decode("latin-1").translate(table)
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def lookup(label: str) -> str | None:
|
|
463
|
+
"""Resolve a label to a WHATWG encoding name, or ``None`` when it is unknown
|
|
464
|
+
or maps to "replacement" (both are ignored by the ladder)."""
|
|
465
|
+
if not label.isascii(): # ASCII case-insensitive only: "\u212aoi8-r" is not a label
|
|
466
|
+
return None
|
|
467
|
+
enc = _LABELS.get(label.strip("\t\n\f\r ").lower())
|
|
468
|
+
return None if enc in (None, "replacement") else enc
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
# -- the ladder ---------------------------------------------------------------
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
_HTTP_WS = "\t\n\r "
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
def _quoted_string(s: str, pos: int) -> tuple[str, int]:
|
|
478
|
+
"""WHATWG "collect an HTTP quoted string" (extract value) at ``s[pos] == '"'``;
|
|
479
|
+
an unterminated string runs to the end."""
|
|
480
|
+
out: list[str] = []
|
|
481
|
+
pos += 1
|
|
482
|
+
while pos < len(s):
|
|
483
|
+
ch = s[pos]
|
|
484
|
+
pos += 1
|
|
485
|
+
if ch == '"':
|
|
486
|
+
break
|
|
487
|
+
if ch == "\\":
|
|
488
|
+
if pos >= len(s):
|
|
489
|
+
out.append("\\")
|
|
490
|
+
break
|
|
491
|
+
ch = s[pos]
|
|
492
|
+
pos += 1
|
|
493
|
+
out.append(ch)
|
|
494
|
+
return "".join(out), pos
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
def _mime_parameters(s: str) -> dict[str, str]:
|
|
498
|
+
"""The parameters after a MIME type's first ``;``, parsed as the WHATWG
|
|
499
|
+
MIME Sniffing Standard does: names are lower-cased, a quoted value is
|
|
500
|
+
unquoted, and the FIRST occurrence of a name wins."""
|
|
501
|
+
params: dict[str, str] = {}
|
|
502
|
+
pos, n = 0, len(s)
|
|
503
|
+
while pos < n:
|
|
504
|
+
while pos < n and s[pos] in _HTTP_WS:
|
|
505
|
+
pos += 1
|
|
506
|
+
start = pos
|
|
507
|
+
while pos < n and s[pos] not in ";=":
|
|
508
|
+
pos += 1
|
|
509
|
+
name = s[start:pos].lower()
|
|
510
|
+
if pos < n and s[pos] == ";":
|
|
511
|
+
pos += 1
|
|
512
|
+
continue
|
|
513
|
+
pos += 1 # past "="
|
|
514
|
+
if pos >= n:
|
|
515
|
+
break
|
|
516
|
+
if s[pos] == '"':
|
|
517
|
+
value, pos = _quoted_string(s, pos)
|
|
518
|
+
while pos < n and s[pos] != ";":
|
|
519
|
+
pos += 1
|
|
520
|
+
else:
|
|
521
|
+
start = pos
|
|
522
|
+
while pos < n and s[pos] != ";":
|
|
523
|
+
pos += 1
|
|
524
|
+
value = s[start:pos].rstrip(_HTTP_WS)
|
|
525
|
+
pos += 1 # past ";"
|
|
526
|
+
if name and value and name not in params:
|
|
527
|
+
params[name] = value
|
|
528
|
+
return params
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
def _parse_content_type(content_type: str | None) -> tuple[str, str | None]:
|
|
532
|
+
"""Return ``(essence, charset)`` from a Content-Type value."""
|
|
533
|
+
if not content_type:
|
|
534
|
+
return "", None
|
|
535
|
+
essence, _, rest = content_type.strip(_HTTP_WS).partition(";")
|
|
536
|
+
return essence.strip(_HTTP_WS).lower(), _mime_parameters(rest).get("charset")
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
def _is_json(essence: str) -> bool:
|
|
540
|
+
return essence.endswith("/json") or essence.endswith("+json")
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
# -- the HTML <meta> prescan (HTML Standard, "prescan a byte stream to
|
|
544
|
+
# determine its encoding"), limited to <meta>: comments and the attributes of
|
|
545
|
+
# other tags are skipped, and running out of bytes inside a construct ends the
|
|
546
|
+
# prescan with no answer. --------------------------------------------------------
|
|
547
|
+
|
|
548
|
+
_SPACE = b"\t\n\x0c\r "
|
|
549
|
+
_FAILED = "failed" # a label that names no usable encoding (distinct from "none yet")
|
|
550
|
+
|
|
551
|
+
|
|
552
|
+
class _OutOfBytes(Exception):
|
|
553
|
+
pass
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
def _at(data: bytes, pos: int) -> int:
|
|
557
|
+
if pos >= len(data):
|
|
558
|
+
raise _OutOfBytes
|
|
559
|
+
return data[pos]
|
|
560
|
+
|
|
561
|
+
|
|
562
|
+
def _lower(byte: int) -> str:
|
|
563
|
+
return chr(byte + 0x20 if 0x41 <= byte <= 0x5A else byte)
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
def _attribute(data: bytes, pos: int) -> tuple[tuple[str, str] | None, int]:
|
|
567
|
+
""" "Get an attribute": ``((name, value) | None, position)``."""
|
|
568
|
+
while _at(data, pos) in _SPACE or data[pos] == 0x2F:
|
|
569
|
+
pos += 1
|
|
570
|
+
if data[pos] == 0x3E:
|
|
571
|
+
return None, pos
|
|
572
|
+
name: list[str] = []
|
|
573
|
+
value: list[str] = []
|
|
574
|
+
while True:
|
|
575
|
+
byte = _at(data, pos)
|
|
576
|
+
if byte == 0x3D and name:
|
|
577
|
+
pos += 1
|
|
578
|
+
break
|
|
579
|
+
if byte in _SPACE:
|
|
580
|
+
while _at(data, pos) in _SPACE:
|
|
581
|
+
pos += 1
|
|
582
|
+
if data[pos] != 0x3D:
|
|
583
|
+
return ("".join(name), ""), pos
|
|
584
|
+
pos += 1
|
|
585
|
+
break
|
|
586
|
+
if byte in (0x2F, 0x3E):
|
|
587
|
+
return ("".join(name), ""), pos
|
|
588
|
+
name.append(_lower(byte))
|
|
589
|
+
pos += 1
|
|
590
|
+
while _at(data, pos) in _SPACE:
|
|
591
|
+
pos += 1
|
|
592
|
+
byte = data[pos]
|
|
593
|
+
if byte in (0x22, 0x27):
|
|
594
|
+
while True:
|
|
595
|
+
pos += 1
|
|
596
|
+
inner = _at(data, pos)
|
|
597
|
+
if inner == byte:
|
|
598
|
+
return ("".join(name), "".join(value)), pos + 1
|
|
599
|
+
value.append(_lower(inner))
|
|
600
|
+
if byte == 0x3E:
|
|
601
|
+
return ("".join(name), ""), pos
|
|
602
|
+
while True:
|
|
603
|
+
value.append(_lower(byte))
|
|
604
|
+
pos += 1
|
|
605
|
+
byte = _at(data, pos)
|
|
606
|
+
if byte in _SPACE or byte == 0x3E:
|
|
607
|
+
return ("".join(name), "".join(value)), pos
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
def _label(text: str) -> str:
|
|
611
|
+
return lookup(text) or _FAILED
|
|
612
|
+
|
|
613
|
+
|
|
614
|
+
def _charset_from_content(content: str) -> str | None:
|
|
615
|
+
""" "Extract a character encoding from a meta element" (the ``content``
|
|
616
|
+
attribute of an http-equiv meta)."""
|
|
617
|
+
pos, n = 0, len(content)
|
|
618
|
+
while True:
|
|
619
|
+
found = content.find("charset", pos)
|
|
620
|
+
if found < 0:
|
|
621
|
+
return None
|
|
622
|
+
pos = found + 7
|
|
623
|
+
while pos < n and content[pos] in _HTTP_WS + "\x0c":
|
|
624
|
+
pos += 1
|
|
625
|
+
if pos >= n or content[pos] != "=":
|
|
626
|
+
continue
|
|
627
|
+
pos += 1
|
|
628
|
+
while pos < n and content[pos] in _HTTP_WS + "\x0c":
|
|
629
|
+
pos += 1
|
|
630
|
+
if pos >= n:
|
|
631
|
+
return None
|
|
632
|
+
quote = content[pos]
|
|
633
|
+
if quote in "\"'":
|
|
634
|
+
end = content.find(quote, pos + 1)
|
|
635
|
+
return None if end < 0 else _label(content[pos + 1 : end])
|
|
636
|
+
end = pos
|
|
637
|
+
while end < n and content[end] not in _HTTP_WS + "\x0c;":
|
|
638
|
+
end += 1
|
|
639
|
+
return _label(content[pos:end])
|
|
640
|
+
|
|
641
|
+
|
|
642
|
+
def _meta(data: bytes, pos: int) -> tuple[str | None, int]:
|
|
643
|
+
"""The attributes of one ``<meta>``; ``(encoding | None, position)``."""
|
|
644
|
+
seen: set[str] = set()
|
|
645
|
+
got_pragma = False
|
|
646
|
+
need_pragma: bool | None = None
|
|
647
|
+
charset: str | None = None
|
|
648
|
+
while True:
|
|
649
|
+
attribute, pos = _attribute(data, pos)
|
|
650
|
+
if attribute is None:
|
|
651
|
+
break
|
|
652
|
+
name, value = attribute
|
|
653
|
+
if name in seen:
|
|
654
|
+
continue
|
|
655
|
+
seen.add(name)
|
|
656
|
+
if name == "http-equiv":
|
|
657
|
+
got_pragma = got_pragma or value == "content-type"
|
|
658
|
+
elif name == "content":
|
|
659
|
+
found = _charset_from_content(value)
|
|
660
|
+
if found not in (None, _FAILED) and charset is None:
|
|
661
|
+
charset, need_pragma = found, True
|
|
662
|
+
elif name == "charset":
|
|
663
|
+
charset, need_pragma = _label(value), False
|
|
664
|
+
if need_pragma is None or (need_pragma and not got_pragma) or charset in (None, _FAILED):
|
|
665
|
+
return None, pos
|
|
666
|
+
return charset, pos
|
|
667
|
+
|
|
668
|
+
|
|
669
|
+
def _prescan(data: bytes) -> str | None:
|
|
670
|
+
"""The encoding a ``<meta>`` in ``data`` declares, or ``None``."""
|
|
671
|
+
pos, n = 0, len(data)
|
|
672
|
+
try:
|
|
673
|
+
while pos < n:
|
|
674
|
+
if data.startswith(b"<!--", pos):
|
|
675
|
+
end = data.find(b"-->", pos + 2)
|
|
676
|
+
if end < 0:
|
|
677
|
+
return None
|
|
678
|
+
pos = end + 2
|
|
679
|
+
elif data[pos : pos + 5].lower() == b"<meta" and _at(data, pos + 5) in _SPACE + b"/":
|
|
680
|
+
found, pos = _meta(data, pos + 5)
|
|
681
|
+
if found is not None:
|
|
682
|
+
return found
|
|
683
|
+
elif data[pos] == 0x3C and _is_tag(data, pos):
|
|
684
|
+
while _at(data, pos) not in _SPACE + b">":
|
|
685
|
+
pos += 1
|
|
686
|
+
attribute: tuple[str, str] | None = ("", "")
|
|
687
|
+
while attribute is not None:
|
|
688
|
+
attribute, pos = _attribute(data, pos)
|
|
689
|
+
elif data.startswith((b"<!", b"</", b"<?"), pos):
|
|
690
|
+
end = data.find(b">", pos + 1)
|
|
691
|
+
if end < 0:
|
|
692
|
+
return None
|
|
693
|
+
pos = end
|
|
694
|
+
pos += 1
|
|
695
|
+
except _OutOfBytes:
|
|
696
|
+
return None
|
|
697
|
+
return None
|
|
698
|
+
|
|
699
|
+
|
|
700
|
+
def _is_tag(data: bytes, pos: int) -> bool:
|
|
701
|
+
"""``<`` or ``</`` followed by an ASCII letter."""
|
|
702
|
+
pos += 2 if data[pos + 1 : pos + 2] == b"/" else 1
|
|
703
|
+
return pos < len(data) and (0x41 <= data[pos] <= 0x5A or 0x61 <= data[pos] <= 0x7A)
|
|
704
|
+
|
|
705
|
+
|
|
706
|
+
_META_SNIFF_BYTES = 1024
|
|
707
|
+
_HTML_TYPES = ("", "text/html", "application/xhtml+xml")
|
|
708
|
+
|
|
709
|
+
|
|
710
|
+
def decode(body: bytes, content_type: str | None) -> str:
|
|
711
|
+
"""Decode a response body per the WHATWG ladder."""
|
|
712
|
+
if body.startswith(b"\xef\xbb\xbf"):
|
|
713
|
+
return _decode_as(body[3:], "utf-8")
|
|
714
|
+
if body.startswith(b"\xfe\xff"):
|
|
715
|
+
return _decode_as(body[2:], "utf-16be")
|
|
716
|
+
if body.startswith(b"\xff\xfe"):
|
|
717
|
+
return _decode_as(body[2:], "utf-16le")
|
|
718
|
+
|
|
719
|
+
essence, charset = _parse_content_type(content_type)
|
|
720
|
+
if charset:
|
|
721
|
+
enc = lookup(charset)
|
|
722
|
+
if enc is not None:
|
|
723
|
+
return _decode_as(body, enc)
|
|
724
|
+
if _is_json(essence):
|
|
725
|
+
return _decode_as(body, "utf-8")
|
|
726
|
+
if essence in _HTML_TYPES:
|
|
727
|
+
enc = _prescan(body[:_META_SNIFF_BYTES])
|
|
728
|
+
if enc is not None:
|
|
729
|
+
# HTML: a <meta> cannot switch to UTF-16 (the bytes it was read
|
|
730
|
+
# from were ASCII-compatible), and x-user-defined means 1252.
|
|
731
|
+
if enc in ("utf-16be", "utf-16le"):
|
|
732
|
+
enc = "utf-8"
|
|
733
|
+
elif enc == "x-user-defined":
|
|
734
|
+
enc = "windows-1252"
|
|
735
|
+
return _decode_as(body, enc)
|
|
736
|
+
return _decode_as(body, "utf-8")
|
|
737
|
+
|
|
738
|
+
|
|
739
|
+
__all__ = ["decode", "lookup"]
|