fasthtmlparser-pure 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,124 @@
1
+ """Character helpers shared by the parser, the text readers and the query engines.
2
+
3
+ Inside the library the page is a Python str and positions are char indexes; Node.pos converts them to byte offsets
4
+ into the UTF-8 source. Whitespace is what .NET's char.IsWhiteSpace sees (str.isspace() also takes U+001C..U+001F).
5
+ """
6
+
7
+ import re
8
+
9
+ # ASCII whitespace as the parser sees it: space, \t, \n, \v, \f, \r.
10
+ ASCII_SPACE = " \t\n\x0b\x0c\r"
11
+ # One whitespace char as .NET char.IsWhiteSpace sees it.
12
+ SPACE = ASCII_SPACE + "".join(map(chr, (0x85, 0xA0, 0x1680, *range(0x2000, 0x200B), 0x2028, 0x2029, 0x202F, 0x205F,
13
+ 0x3000)))
14
+ SPACE_SET = frozenset(SPACE)
15
+ SPACE_RUNS = re.compile("[" + re.escape(SPACE) + "]+")
16
+ HAS_SPACE = re.compile("[" + re.escape(SPACE) + "]")
17
+ ASTRAL = re.compile("[\U00010000-\U0010ffff]")
18
+
19
+ _ASCII_LOWER = {c: c + 32 for c in range(65, 91)}
20
+ _ASCII_UPPER = {c: c - 32 for c in range(97, 123)}
21
+
22
+
23
+ def ascii_lower(s):
24
+ """Lower case of ASCII letters only (str.lower() also maps U+212A KELVIN SIGN to 'k'); keeps the length."""
25
+ return s.lower() if s.isascii() else s.translate(_ASCII_LOWER)
26
+
27
+
28
+ def lower(s):
29
+ """Lower case: ASCII, and Unicode."""
30
+ return s.lower()
31
+
32
+
33
+ def upper(s):
34
+ return s.upper()
35
+
36
+
37
+ def trim_space(s):
38
+ """Trims Unicode whitespace (char.IsWhiteSpace) from both ends."""
39
+ return s.strip(SPACE)
40
+
41
+
42
+ def trim_range(s, start, end):
43
+ """s[start:end] clamped to the string and trimmed of Unicode whitespace."""
44
+ if start < 0:
45
+ start = 0
46
+ if start >= end:
47
+ return ""
48
+ return s[start:end].strip(SPACE)
49
+
50
+
51
+ def utf16_length(s):
52
+ """The length in UTF-16 code units, as .NET counts."""
53
+ n = len(s)
54
+ if s.isascii():
55
+ return n
56
+ return n + len(ASTRAL.findall(s))
57
+
58
+
59
+ def equal_fold(a, b):
60
+ """Case-insensitive comparison (ASCII fast path, Unicode lower case otherwise)."""
61
+ if a == b:
62
+ return True
63
+ if a.isascii() and b.isascii():
64
+ return len(a) == len(b) and a.lower() == b.lower()
65
+ if utf16_length(a) != utf16_length(b):
66
+ return False
67
+ return a.lower() == b.lower()
68
+
69
+
70
+ def ascii_equal_fold(a, b):
71
+ """ASCII-only case-insensitive comparison."""
72
+ return len(a) == len(b) and ascii_lower(a) == ascii_lower(b)
73
+
74
+
75
+ def is_alpha(c):
76
+ return ("a" <= c <= "z") or ("A" <= c <= "Z")
77
+
78
+
79
+ def is_digit(c):
80
+ return "0" <= c <= "9"
81
+
82
+
83
+ def is_hex(c):
84
+ return ("0" <= c <= "9") or ("a" <= c <= "f") or ("A" <= c <= "F")
85
+
86
+
87
+ def quote(s):
88
+ """A string in double quotes, for error messages."""
89
+ return '"' + s.replace("\\", "\\\\").replace('"', '\\"') + '"'
90
+
91
+
92
+ class ByteOffsets:
93
+ """Char index to UTF-8 byte offset for one string: the non-ASCII chars and the extra bytes before each."""
94
+
95
+ __slots__ = ("_at", "_extra")
96
+
97
+ def __init__(self, s):
98
+ at = []
99
+ extra = []
100
+ total = 0
101
+ for m in re.finditer(r"[^\x00-\x7f]+", s):
102
+ run = m.group()
103
+ for k, c in enumerate(run):
104
+ o = ord(c)
105
+ total += 1 if o < 0x800 else (2 if o < 0x10000 else 3)
106
+ at.append(m.start() + k)
107
+ extra.append(total)
108
+ self._at = at
109
+ self._extra = extra
110
+
111
+ def __call__(self, i):
112
+ """The byte offset of the char at index i."""
113
+ at = self._at
114
+ if not at or i <= at[0]:
115
+ return i
116
+ # the number of non-ASCII chars before i
117
+ lo, hi = 0, len(at)
118
+ while lo < hi:
119
+ mid = (lo + hi) >> 1
120
+ if at[mid] < i:
121
+ lo = mid + 1
122
+ else:
123
+ hi = mid
124
+ return i + self._extra[lo - 1]