sdo-client 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
sdo/_encoding.py ADDED
@@ -0,0 +1,739 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """The WHATWG Encoding Standard "decode" ladder for ``ScrapeResult.text``.
3
+
4
+ JavaScript (``TextDecoder``) and Go (``htmlindex``) accept only the labels of the
5
+ Encoding Standard; Python's ``codecs`` accepts hundreds of names, including
6
+ codecs that are not text encodings at all and one (``punycode``) whose decoder
7
+ is quadratic. A page controls the label it declares, so a label is never passed
8
+ to ``codecs`` directly: it is resolved through the WHATWG table below and
9
+ mapped to a Python codec (or a decode table) chosen here.
10
+
11
+ Ladder: BOM > Content-Type ``charset`` > ``*/json`` > HTML ``<meta>`` in the
12
+ first 1024 bytes > UTF-8. An unknown label, or one that maps to WHATWG
13
+ "replacement", is ignored and the ladder continues.
14
+
15
+ Exactness: UTF-8, UTF-16, every single-byte encoding and gb18030/GBK follow the
16
+ WHATWG decoders byte for byte. Big5, EUC-JP, ISO-2022-JP, Shift_JIS and EUC-KR
17
+ use the closest Python codec: well-formed text decodes as WHATWG specifies
18
+ (except 203 HKSCS-2008 Big5 additions Python does not map), but recovery from
19
+ malformed bytes may place U+FFFD differently.
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ import re
25
+
26
+ # -- labels -> WHATWG encoding names (https://encoding.spec.whatwg.org/#names-and-labels)
27
+
28
+ _LABELS_BY_ENCODING: dict[str, tuple[str, ...]] = {
29
+ "utf-8": (
30
+ "unicode-1-1-utf-8",
31
+ "unicode11utf8",
32
+ "unicode20utf8",
33
+ "utf-8",
34
+ "utf8",
35
+ "x-unicode20utf8",
36
+ ),
37
+ "ibm866": ("866", "cp866", "csibm866", "ibm866"),
38
+ "iso-8859-2": (
39
+ "csisolatin2",
40
+ "iso-8859-2",
41
+ "iso-ir-101",
42
+ "iso8859-2",
43
+ "iso88592",
44
+ "iso_8859-2",
45
+ "iso_8859-2:1987",
46
+ "l2",
47
+ "latin2",
48
+ ),
49
+ "iso-8859-3": (
50
+ "csisolatin3",
51
+ "iso-8859-3",
52
+ "iso-ir-109",
53
+ "iso8859-3",
54
+ "iso88593",
55
+ "iso_8859-3",
56
+ "iso_8859-3:1988",
57
+ "l3",
58
+ "latin3",
59
+ ),
60
+ "iso-8859-4": (
61
+ "csisolatin4",
62
+ "iso-8859-4",
63
+ "iso-ir-110",
64
+ "iso8859-4",
65
+ "iso88594",
66
+ "iso_8859-4",
67
+ "iso_8859-4:1988",
68
+ "l4",
69
+ "latin4",
70
+ ),
71
+ "iso-8859-5": (
72
+ "csisolatincyrillic",
73
+ "cyrillic",
74
+ "iso-8859-5",
75
+ "iso-ir-144",
76
+ "iso8859-5",
77
+ "iso88595",
78
+ "iso_8859-5",
79
+ "iso_8859-5:1988",
80
+ ),
81
+ "iso-8859-6": (
82
+ "arabic",
83
+ "asmo-708",
84
+ "csiso88596e",
85
+ "csiso88596i",
86
+ "csisolatinarabic",
87
+ "ecma-114",
88
+ "iso-8859-6",
89
+ "iso-8859-6-e",
90
+ "iso-8859-6-i",
91
+ "iso-ir-127",
92
+ "iso8859-6",
93
+ "iso88596",
94
+ "iso_8859-6",
95
+ "iso_8859-6:1987",
96
+ ),
97
+ "iso-8859-7": (
98
+ "csisolatingreek",
99
+ "ecma-118",
100
+ "elot_928",
101
+ "greek",
102
+ "greek8",
103
+ "iso-8859-7",
104
+ "iso-ir-126",
105
+ "iso8859-7",
106
+ "iso88597",
107
+ "iso_8859-7",
108
+ "iso_8859-7:1987",
109
+ "sun_eu_greek",
110
+ ),
111
+ "iso-8859-8": (
112
+ "csiso88598e",
113
+ "csisolatinhebrew",
114
+ "hebrew",
115
+ "iso-8859-8",
116
+ "iso-8859-8-e",
117
+ "iso-ir-138",
118
+ "iso8859-8",
119
+ "iso88598",
120
+ "iso_8859-8",
121
+ "iso_8859-8:1988",
122
+ "visual",
123
+ ),
124
+ "iso-8859-8-i": ("csiso88598i", "iso-8859-8-i", "logical"),
125
+ "iso-8859-10": (
126
+ "csisolatin6",
127
+ "iso-8859-10",
128
+ "iso-ir-157",
129
+ "iso8859-10",
130
+ "iso885910",
131
+ "l6",
132
+ "latin6",
133
+ ),
134
+ "iso-8859-13": ("iso-8859-13", "iso8859-13", "iso885913"),
135
+ "iso-8859-14": ("iso-8859-14", "iso8859-14", "iso885914"),
136
+ "iso-8859-15": (
137
+ "csisolatin9",
138
+ "iso-8859-15",
139
+ "iso8859-15",
140
+ "iso885915",
141
+ "iso_8859-15",
142
+ "l9",
143
+ ),
144
+ "iso-8859-16": ("iso-8859-16",),
145
+ "koi8-r": ("cskoi8r", "koi", "koi8", "koi8-r", "koi8_r"),
146
+ "koi8-u": ("koi8-ru", "koi8-u"),
147
+ "macintosh": ("csmacintosh", "mac", "macintosh", "x-mac-roman"),
148
+ "windows-874": (
149
+ "dos-874",
150
+ "iso-8859-11",
151
+ "iso8859-11",
152
+ "iso885911",
153
+ "tis-620",
154
+ "windows-874",
155
+ ),
156
+ "windows-1250": ("cp1250", "windows-1250", "x-cp1250"),
157
+ "windows-1251": ("cp1251", "windows-1251", "x-cp1251"),
158
+ "windows-1252": (
159
+ "ansi_x3.4-1968",
160
+ "ascii",
161
+ "cp1252",
162
+ "cp819",
163
+ "csisolatin1",
164
+ "ibm819",
165
+ "iso-8859-1",
166
+ "iso-ir-100",
167
+ "iso8859-1",
168
+ "iso88591",
169
+ "iso_8859-1",
170
+ "iso_8859-1:1987",
171
+ "l1",
172
+ "latin1",
173
+ "us-ascii",
174
+ "windows-1252",
175
+ "x-cp1252",
176
+ ),
177
+ "windows-1253": ("cp1253", "windows-1253", "x-cp1253"),
178
+ "windows-1254": (
179
+ "cp1254",
180
+ "csisolatin5",
181
+ "iso-8859-9",
182
+ "iso-ir-148",
183
+ "iso8859-9",
184
+ "iso88599",
185
+ "iso_8859-9",
186
+ "iso_8859-9:1989",
187
+ "l5",
188
+ "latin5",
189
+ "windows-1254",
190
+ "x-cp1254",
191
+ ),
192
+ "windows-1255": ("cp1255", "windows-1255", "x-cp1255"),
193
+ "windows-1256": ("cp1256", "windows-1256", "x-cp1256"),
194
+ "windows-1257": ("cp1257", "windows-1257", "x-cp1257"),
195
+ "windows-1258": ("cp1258", "windows-1258", "x-cp1258"),
196
+ "x-mac-cyrillic": ("x-mac-cyrillic", "x-mac-ukrainian"),
197
+ "gbk": (
198
+ "chinese",
199
+ "csgb2312",
200
+ "csiso58gb231280",
201
+ "gb2312",
202
+ "gb_2312",
203
+ "gb_2312-80",
204
+ "gbk",
205
+ "iso-ir-58",
206
+ "x-gbk",
207
+ ),
208
+ "gb18030": ("gb18030",),
209
+ "big5": ("big5", "big5-hkscs", "cn-big5", "csbig5", "x-x-big5"),
210
+ "euc-jp": ("cseucpkdfmtjapanese", "euc-jp", "x-euc-jp"),
211
+ "iso-2022-jp": ("csiso2022jp", "iso-2022-jp"),
212
+ "shift_jis": (
213
+ "csshiftjis",
214
+ "ms932",
215
+ "ms_kanji",
216
+ "shift-jis",
217
+ "shift_jis",
218
+ "sjis",
219
+ "windows-31j",
220
+ "x-sjis",
221
+ ),
222
+ "euc-kr": (
223
+ "cseuckr",
224
+ "csksc56011987",
225
+ "euc-kr",
226
+ "iso-ir-149",
227
+ "korean",
228
+ "ks_c_5601-1987",
229
+ "ks_c_5601-1989",
230
+ "ksc5601",
231
+ "ksc_5601",
232
+ "windows-949",
233
+ ),
234
+ "replacement": (
235
+ "csiso2022kr",
236
+ "hz-gb-2312",
237
+ "iso-2022-cn",
238
+ "iso-2022-cn-ext",
239
+ "iso-2022-kr",
240
+ "replacement",
241
+ ),
242
+ "utf-16be": ("unicodefffe", "utf-16be"),
243
+ "utf-16le": (
244
+ "csunicode",
245
+ "iso-10646-ucs-2",
246
+ "ucs-2",
247
+ "unicode",
248
+ "unicodefeff",
249
+ "utf-16",
250
+ "utf-16le",
251
+ ),
252
+ "x-user-defined": ("x-user-defined",),
253
+ }
254
+
255
+ _LABELS: dict[str, str] = {
256
+ label: enc for enc, labels in _LABELS_BY_ENCODING.items() for label in labels
257
+ }
258
+
259
+ # -- WHATWG encoding -> how Python decodes it --------------------------------
260
+
261
+ #: Multi-byte encodings: the Python codec whose decoder matches the WHATWG one.
262
+ #: WHATWG's decoders are supersets of the legacy standards their labels name.
263
+ _MULTI_BYTE = {
264
+ "utf-8": "utf-8",
265
+ "big5": "big5hkscs",
266
+ "euc-jp": "euc_jp",
267
+ "iso-2022-jp": "iso2022_jp_ext",
268
+ "shift_jis": "cp932",
269
+ "euc-kr": "cp949",
270
+ "utf-16be": "utf-16-be",
271
+ "utf-16le": "utf-16-le",
272
+ }
273
+
274
+ #: Single-byte encodings: the Python codec to start the 256-entry decode table
275
+ #: from, before the WHATWG corrections below.
276
+ _SINGLE_BYTE = {
277
+ "ibm866": "cp866",
278
+ "iso-8859-2": "iso8859_2",
279
+ "iso-8859-3": "iso8859_3",
280
+ "iso-8859-4": "iso8859_4",
281
+ "iso-8859-5": "iso8859_5",
282
+ "iso-8859-6": "iso8859_6",
283
+ "iso-8859-7": "iso8859_7",
284
+ "iso-8859-8": "iso8859_8",
285
+ "iso-8859-8-i": "iso8859_8",
286
+ "iso-8859-10": "iso8859_10",
287
+ "iso-8859-13": "iso8859_13",
288
+ "iso-8859-14": "iso8859_14",
289
+ "iso-8859-15": "iso8859_15",
290
+ "iso-8859-16": "iso8859_16",
291
+ "koi8-r": "koi8_r",
292
+ "koi8-u": "koi8_u",
293
+ "macintosh": "mac_roman",
294
+ "windows-874": "cp874",
295
+ "windows-1250": "cp1250",
296
+ "windows-1251": "cp1251",
297
+ "windows-1252": "cp1252",
298
+ "windows-1253": "cp1253",
299
+ "windows-1254": "cp1254",
300
+ "windows-1255": "cp1255",
301
+ "windows-1256": "cp1256",
302
+ "windows-1257": "cp1257",
303
+ "windows-1258": "cp1258",
304
+ "x-mac-cyrillic": "mac_cyrillic",
305
+ }
306
+
307
+ #: Bytes where the WHATWG index differs from the Python codec (beyond the
308
+ #: windows-* C1 rule applied in ``_single_byte_table``).
309
+ _OVERRIDES = {
310
+ "koi8-u": {0xAE: "ў", 0xBE: "Ў"}, # WHATWG KOI8-U is KOI8-RU
311
+ "windows-1255": {0xCA: "ֺ"},
312
+ }
313
+
314
+ _UNMAPPED = "\ufffd" # a byte the WHATWG index leaves unmapped
315
+ _TABLES: dict[str, str] = {}
316
+
317
+
318
+ def _single_byte_table(encoding: str) -> str:
319
+ table = _TABLES.get(encoding)
320
+ if table is None:
321
+ codec = _SINGLE_BYTE[encoding]
322
+ chars = []
323
+ for b in range(256):
324
+ try:
325
+ chars.append(bytes([b]).decode(codec))
326
+ except UnicodeDecodeError:
327
+ # A windows-* hole in 0x80-0x9F decodes to the C1 control of the
328
+ # same value in WHATWG (and browsers); cp1252 leaves it unmapped.
329
+ chr_c1 = chr(b) if encoding.startswith("windows-") and b < 0xA0 else _UNMAPPED
330
+ chars.append(chr_c1)
331
+ for b, ch in _OVERRIDES.get(encoding, {}).items():
332
+ chars[b] = ch
333
+ table = _TABLES[encoding] = "".join(chars)
334
+ return table
335
+
336
+
337
+ #: x-user-defined: ASCII, then bytes 0x80-0xFF map to U+F780-U+F7FF.
338
+ _X_USER_DEFINED = "".join(chr(b) for b in range(0x80)) + "".join(
339
+ chr(0xF780 + b) for b in range(0x80)
340
+ )
341
+
342
+
343
+ # -- gb18030 (and GBK, which WHATWG decodes with the same decoder) -----------
344
+ #
345
+ # Python's gb18030 codec agrees with the WHATWG decoder on every well-formed
346
+ # sequence except the 21 below (GB18030-2022 moved them off the Private Use
347
+ # Area), but not on the single byte 0x80 (WHATWG: the euro sign) nor on how it
348
+ # recovers from a broken sequence. So the bytes are split here, per the WHATWG
349
+ # algorithm: runs of well-formed sequences go to the codec in one call, and
350
+ # everything else is decoded step by step.
351
+
352
+ _GB_OVERRIDES = {
353
+ b"\xa3\xa0": "\u3000",
354
+ b"\xa6\xd9": "\ufe10",
355
+ b"\xa6\xda": "\ufe12",
356
+ b"\xa6\xdb": "\ufe11",
357
+ b"\xa6\xdc": "\ufe13",
358
+ b"\xa6\xdd": "\ufe14",
359
+ b"\xa6\xde": "\ufe15",
360
+ b"\xa6\xdf": "\ufe16",
361
+ b"\xa6\xec": "\ufe17",
362
+ b"\xa6\xed": "\ufe18",
363
+ b"\xa6\xf3": "\ufe19",
364
+ b"\xa8\xbc": "\u1e3f",
365
+ b"\xfe\x59": "\u9fb4",
366
+ b"\xfe\x61": "\u9fb5",
367
+ b"\xfe\x66": "\u9fb6",
368
+ b"\xfe\x67": "\u9fb7",
369
+ b"\xfe\x6d": "\u9fb8",
370
+ b"\xfe\x7e": "\u9fb9",
371
+ b"\xfe\x90": "\u9fba",
372
+ b"\xfe\xa0": "\u9fbb",
373
+ b"\x81\x35\xf4\x37": "\ue7c7",
374
+ }
375
+ #: A run of well-formed sequences that are not overrides: ASCII, a two-byte
376
+ #: sequence, or a four-byte sequence (the second byte tells them apart). The
377
+ #: repeat is possessive: a greedy one keeps a backtrack point per sequence,
378
+ #: over a hundred times the body's size in memory for a long ASCII run.
379
+ _GB_RUN = re.compile(
380
+ rb"(?:[\x00-\x7f]"
381
+ rb"|(?!\xa3\xa0|\xa6[\xd9-\xdf\xec\xed\xf3]|\xa8\xbc|\xfe[\x59\x61\x66\x67\x6d\x7e\x90\xa0])"
382
+ rb"[\x81-\xfe][\x40-\x7e\x80-\xfe]"
383
+ rb"|(?!\x81\x35\xf4\x37)[\x81-\xfe][\x30-\x39][\x81-\xfe][\x30-\x39])++"
384
+ )
385
+
386
+
387
+ def _gb_sequence(seq: bytes) -> str:
388
+ """One well-formed sequence: an override, the codec's answer, or U+FFFD for
389
+ a four-byte pointer WHATWG leaves unmapped."""
390
+ if seq in _GB_OVERRIDES:
391
+ return _GB_OVERRIDES[seq]
392
+ try:
393
+ return seq.decode("gb18030")
394
+ except UnicodeDecodeError:
395
+ return "\ufffd"
396
+
397
+
398
+ def _gb_step(data: bytes, i: int) -> tuple[str, int]:
399
+ """Decode the sequence at ``data[i]`` the WHATWG way: ``(text, consumed)``.
400
+ A byte that ends a broken sequence is not consumed when it can start the
401
+ next one."""
402
+ n = len(data)
403
+ first = data[i]
404
+ if first < 0x80:
405
+ return chr(first), 1
406
+ if first == 0x80:
407
+ return "\u20ac", 1
408
+ if first == 0xFF or i + 1 == n:
409
+ return "\ufffd", 1
410
+ second = data[i + 1]
411
+ if 0x30 <= second <= 0x39:
412
+ if i + 2 == n:
413
+ return "\ufffd", 2
414
+ if not 0x81 <= data[i + 2] <= 0xFE:
415
+ return "\ufffd", 1
416
+ if i + 3 == n:
417
+ return "\ufffd", 3
418
+ if not 0x30 <= data[i + 3] <= 0x39:
419
+ return "\ufffd", 1
420
+ return _gb_sequence(data[i : i + 4]), 4
421
+ if 0x40 <= second <= 0x7E or 0x80 <= second <= 0xFE:
422
+ return _gb_sequence(data[i : i + 2]), 2
423
+ return "\ufffd", 1 if second < 0x80 else 2
424
+
425
+
426
+ def _decode_gb18030(data: bytes) -> str:
427
+ out: list[str] = []
428
+ i, n = 0, len(data)
429
+ while i < n:
430
+ run = _GB_RUN.match(data, i)
431
+ if run:
432
+ end = run.end()
433
+ try:
434
+ out.append(data[i:end].decode("gb18030"))
435
+ i = end
436
+ continue
437
+ except UnicodeDecodeError: # an unmapped four-byte pointer inside
438
+ while i < end:
439
+ text, used = _gb_step(data, i)
440
+ out.append(text)
441
+ i += used
442
+ continue
443
+ text, used = _gb_step(data, i)
444
+ out.append(text)
445
+ i += used
446
+ return "".join(out)
447
+
448
+
449
+ def _decode_as(data: bytes, encoding: str) -> str:
450
+ if encoding in ("gbk", "gb18030"):
451
+ return _decode_gb18030(data)
452
+ if encoding in _MULTI_BYTE:
453
+ return data.decode(_MULTI_BYTE[encoding], errors="replace")
454
+ if encoding == "x-user-defined":
455
+ table = _X_USER_DEFINED
456
+ else:
457
+ table = _single_byte_table(encoding)
458
+ # Every code point of a latin-1 decode is below 256: an index into the table.
459
+ return data.decode("latin-1").translate(table)
460
+
461
+
462
+ def lookup(label: str) -> str | None:
463
+ """Resolve a label to a WHATWG encoding name, or ``None`` when it is unknown
464
+ or maps to "replacement" (both are ignored by the ladder)."""
465
+ if not label.isascii(): # ASCII case-insensitive only: "\u212aoi8-r" is not a label
466
+ return None
467
+ enc = _LABELS.get(label.strip("\t\n\f\r ").lower())
468
+ return None if enc in (None, "replacement") else enc
469
+
470
+
471
+ # -- the ladder ---------------------------------------------------------------
472
+
473
+
474
+ _HTTP_WS = "\t\n\r "
475
+
476
+
477
+ def _quoted_string(s: str, pos: int) -> tuple[str, int]:
478
+ """WHATWG "collect an HTTP quoted string" (extract value) at ``s[pos] == '"'``;
479
+ an unterminated string runs to the end."""
480
+ out: list[str] = []
481
+ pos += 1
482
+ while pos < len(s):
483
+ ch = s[pos]
484
+ pos += 1
485
+ if ch == '"':
486
+ break
487
+ if ch == "\\":
488
+ if pos >= len(s):
489
+ out.append("\\")
490
+ break
491
+ ch = s[pos]
492
+ pos += 1
493
+ out.append(ch)
494
+ return "".join(out), pos
495
+
496
+
497
+ def _mime_parameters(s: str) -> dict[str, str]:
498
+ """The parameters after a MIME type's first ``;``, parsed as the WHATWG
499
+ MIME Sniffing Standard does: names are lower-cased, a quoted value is
500
+ unquoted, and the FIRST occurrence of a name wins."""
501
+ params: dict[str, str] = {}
502
+ pos, n = 0, len(s)
503
+ while pos < n:
504
+ while pos < n and s[pos] in _HTTP_WS:
505
+ pos += 1
506
+ start = pos
507
+ while pos < n and s[pos] not in ";=":
508
+ pos += 1
509
+ name = s[start:pos].lower()
510
+ if pos < n and s[pos] == ";":
511
+ pos += 1
512
+ continue
513
+ pos += 1 # past "="
514
+ if pos >= n:
515
+ break
516
+ if s[pos] == '"':
517
+ value, pos = _quoted_string(s, pos)
518
+ while pos < n and s[pos] != ";":
519
+ pos += 1
520
+ else:
521
+ start = pos
522
+ while pos < n and s[pos] != ";":
523
+ pos += 1
524
+ value = s[start:pos].rstrip(_HTTP_WS)
525
+ pos += 1 # past ";"
526
+ if name and value and name not in params:
527
+ params[name] = value
528
+ return params
529
+
530
+
531
+ def _parse_content_type(content_type: str | None) -> tuple[str, str | None]:
532
+ """Return ``(essence, charset)`` from a Content-Type value."""
533
+ if not content_type:
534
+ return "", None
535
+ essence, _, rest = content_type.strip(_HTTP_WS).partition(";")
536
+ return essence.strip(_HTTP_WS).lower(), _mime_parameters(rest).get("charset")
537
+
538
+
539
+ def _is_json(essence: str) -> bool:
540
+ return essence.endswith("/json") or essence.endswith("+json")
541
+
542
+
543
+ # -- the HTML <meta> prescan (HTML Standard, "prescan a byte stream to
544
+ # determine its encoding"), limited to <meta>: comments and the attributes of
545
+ # other tags are skipped, and running out of bytes inside a construct ends the
546
+ # prescan with no answer. --------------------------------------------------------
547
+
548
+ _SPACE = b"\t\n\x0c\r "
549
+ _FAILED = "failed" # a label that names no usable encoding (distinct from "none yet")
550
+
551
+
552
+ class _OutOfBytes(Exception):
553
+ pass
554
+
555
+
556
+ def _at(data: bytes, pos: int) -> int:
557
+ if pos >= len(data):
558
+ raise _OutOfBytes
559
+ return data[pos]
560
+
561
+
562
+ def _lower(byte: int) -> str:
563
+ return chr(byte + 0x20 if 0x41 <= byte <= 0x5A else byte)
564
+
565
+
566
+ def _attribute(data: bytes, pos: int) -> tuple[tuple[str, str] | None, int]:
567
+ """ "Get an attribute": ``((name, value) | None, position)``."""
568
+ while _at(data, pos) in _SPACE or data[pos] == 0x2F:
569
+ pos += 1
570
+ if data[pos] == 0x3E:
571
+ return None, pos
572
+ name: list[str] = []
573
+ value: list[str] = []
574
+ while True:
575
+ byte = _at(data, pos)
576
+ if byte == 0x3D and name:
577
+ pos += 1
578
+ break
579
+ if byte in _SPACE:
580
+ while _at(data, pos) in _SPACE:
581
+ pos += 1
582
+ if data[pos] != 0x3D:
583
+ return ("".join(name), ""), pos
584
+ pos += 1
585
+ break
586
+ if byte in (0x2F, 0x3E):
587
+ return ("".join(name), ""), pos
588
+ name.append(_lower(byte))
589
+ pos += 1
590
+ while _at(data, pos) in _SPACE:
591
+ pos += 1
592
+ byte = data[pos]
593
+ if byte in (0x22, 0x27):
594
+ while True:
595
+ pos += 1
596
+ inner = _at(data, pos)
597
+ if inner == byte:
598
+ return ("".join(name), "".join(value)), pos + 1
599
+ value.append(_lower(inner))
600
+ if byte == 0x3E:
601
+ return ("".join(name), ""), pos
602
+ while True:
603
+ value.append(_lower(byte))
604
+ pos += 1
605
+ byte = _at(data, pos)
606
+ if byte in _SPACE or byte == 0x3E:
607
+ return ("".join(name), "".join(value)), pos
608
+
609
+
610
+ def _label(text: str) -> str:
611
+ return lookup(text) or _FAILED
612
+
613
+
614
+ def _charset_from_content(content: str) -> str | None:
615
+ """ "Extract a character encoding from a meta element" (the ``content``
616
+ attribute of an http-equiv meta)."""
617
+ pos, n = 0, len(content)
618
+ while True:
619
+ found = content.find("charset", pos)
620
+ if found < 0:
621
+ return None
622
+ pos = found + 7
623
+ while pos < n and content[pos] in _HTTP_WS + "\x0c":
624
+ pos += 1
625
+ if pos >= n or content[pos] != "=":
626
+ continue
627
+ pos += 1
628
+ while pos < n and content[pos] in _HTTP_WS + "\x0c":
629
+ pos += 1
630
+ if pos >= n:
631
+ return None
632
+ quote = content[pos]
633
+ if quote in "\"'":
634
+ end = content.find(quote, pos + 1)
635
+ return None if end < 0 else _label(content[pos + 1 : end])
636
+ end = pos
637
+ while end < n and content[end] not in _HTTP_WS + "\x0c;":
638
+ end += 1
639
+ return _label(content[pos:end])
640
+
641
+
642
+ def _meta(data: bytes, pos: int) -> tuple[str | None, int]:
643
+ """The attributes of one ``<meta>``; ``(encoding | None, position)``."""
644
+ seen: set[str] = set()
645
+ got_pragma = False
646
+ need_pragma: bool | None = None
647
+ charset: str | None = None
648
+ while True:
649
+ attribute, pos = _attribute(data, pos)
650
+ if attribute is None:
651
+ break
652
+ name, value = attribute
653
+ if name in seen:
654
+ continue
655
+ seen.add(name)
656
+ if name == "http-equiv":
657
+ got_pragma = got_pragma or value == "content-type"
658
+ elif name == "content":
659
+ found = _charset_from_content(value)
660
+ if found not in (None, _FAILED) and charset is None:
661
+ charset, need_pragma = found, True
662
+ elif name == "charset":
663
+ charset, need_pragma = _label(value), False
664
+ if need_pragma is None or (need_pragma and not got_pragma) or charset in (None, _FAILED):
665
+ return None, pos
666
+ return charset, pos
667
+
668
+
669
+ def _prescan(data: bytes) -> str | None:
670
+ """The encoding a ``<meta>`` in ``data`` declares, or ``None``."""
671
+ pos, n = 0, len(data)
672
+ try:
673
+ while pos < n:
674
+ if data.startswith(b"<!--", pos):
675
+ end = data.find(b"-->", pos + 2)
676
+ if end < 0:
677
+ return None
678
+ pos = end + 2
679
+ elif data[pos : pos + 5].lower() == b"<meta" and _at(data, pos + 5) in _SPACE + b"/":
680
+ found, pos = _meta(data, pos + 5)
681
+ if found is not None:
682
+ return found
683
+ elif data[pos] == 0x3C and _is_tag(data, pos):
684
+ while _at(data, pos) not in _SPACE + b">":
685
+ pos += 1
686
+ attribute: tuple[str, str] | None = ("", "")
687
+ while attribute is not None:
688
+ attribute, pos = _attribute(data, pos)
689
+ elif data.startswith((b"<!", b"</", b"<?"), pos):
690
+ end = data.find(b">", pos + 1)
691
+ if end < 0:
692
+ return None
693
+ pos = end
694
+ pos += 1
695
+ except _OutOfBytes:
696
+ return None
697
+ return None
698
+
699
+
700
+ def _is_tag(data: bytes, pos: int) -> bool:
701
+ """``<`` or ``</`` followed by an ASCII letter."""
702
+ pos += 2 if data[pos + 1 : pos + 2] == b"/" else 1
703
+ return pos < len(data) and (0x41 <= data[pos] <= 0x5A or 0x61 <= data[pos] <= 0x7A)
704
+
705
+
706
+ _META_SNIFF_BYTES = 1024
707
+ _HTML_TYPES = ("", "text/html", "application/xhtml+xml")
708
+
709
+
710
+ def decode(body: bytes, content_type: str | None) -> str:
711
+ """Decode a response body per the WHATWG ladder."""
712
+ if body.startswith(b"\xef\xbb\xbf"):
713
+ return _decode_as(body[3:], "utf-8")
714
+ if body.startswith(b"\xfe\xff"):
715
+ return _decode_as(body[2:], "utf-16be")
716
+ if body.startswith(b"\xff\xfe"):
717
+ return _decode_as(body[2:], "utf-16le")
718
+
719
+ essence, charset = _parse_content_type(content_type)
720
+ if charset:
721
+ enc = lookup(charset)
722
+ if enc is not None:
723
+ return _decode_as(body, enc)
724
+ if _is_json(essence):
725
+ return _decode_as(body, "utf-8")
726
+ if essence in _HTML_TYPES:
727
+ enc = _prescan(body[:_META_SNIFF_BYTES])
728
+ if enc is not None:
729
+ # HTML: a <meta> cannot switch to UTF-16 (the bytes it was read
730
+ # from were ASCII-compatible), and x-user-defined means 1252.
731
+ if enc in ("utf-16be", "utf-16le"):
732
+ enc = "utf-8"
733
+ elif enc == "x-user-defined":
734
+ enc = "windows-1252"
735
+ return _decode_as(body, enc)
736
+ return _decode_as(body, "utf-8")
737
+
738
+
739
+ __all__ = ["decode", "lookup"]