fasthtmlparser-pure 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fasthtmlparser_pure/__init__.py +41 -0
- fasthtmlparser_pure/_api.py +636 -0
- fasthtmlparser_pure/_api.pyi +342 -0
- fasthtmlparser_pure/_attrs.py +270 -0
- fasthtmlparser_pure/_chars.py +124 -0
- fasthtmlparser_pure/_css.py +861 -0
- fasthtmlparser_pure/_decode.py +144 -0
- fasthtmlparser_pure/_links.py +483 -0
- fasthtmlparser_pure/_parser.py +866 -0
- fasthtmlparser_pure/_query.py +128 -0
- fasthtmlparser_pure/_tables.py +2494 -0
- fasthtmlparser_pure/_text.py +257 -0
- fasthtmlparser_pure/_xpath.py +1594 -0
- fasthtmlparser_pure/py.typed +0 -0
- fasthtmlparser_pure-0.1.0.dist-info/METADATA +166 -0
- fasthtmlparser_pure-0.1.0.dist-info/RECORD +19 -0
- fasthtmlparser_pure-0.1.0.dist-info/WHEEL +5 -0
- fasthtmlparser_pure-0.1.0.dist-info/licenses/LICENSE.md +79 -0
- fasthtmlparser_pure-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Fast HTML / XML parser for scraping with XPath 1.0 and CSS selectors, in pure Python.
|
|
2
|
+
|
|
3
|
+
A pure-Python port of z3l.FastHtmlParser: a page becomes a flat list of tags that hold only positions in the source;
|
|
4
|
+
text, attributes and entities are read when they are asked for. The API is the one of the `fasthtmlparser` package
|
|
5
|
+
(Rust inside), so code runs on either.
|
|
6
|
+
|
|
7
|
+
>>> import fasthtmlparser_pure as fh
|
|
8
|
+
>>> doc = fh.parse('<ul><li class="a">one<li>two</ul>')
|
|
9
|
+
>>> [li.text for li in doc.css("li")]
|
|
10
|
+
['one', 'two']
|
|
11
|
+
>>> doc.xpath_value("count(//li)")
|
|
12
|
+
'2'
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from ._api import (
|
|
16
|
+
CssQuery,
|
|
17
|
+
CssSelectorError,
|
|
18
|
+
Document,
|
|
19
|
+
Node,
|
|
20
|
+
XPathError,
|
|
21
|
+
XPathQuery,
|
|
22
|
+
decode_html,
|
|
23
|
+
html_to_text,
|
|
24
|
+
html_to_text_line_breaks,
|
|
25
|
+
parse,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
__version__ = "0.1.0"
|
|
29
|
+
|
|
30
|
+
__all__ = [
|
|
31
|
+
"CssQuery",
|
|
32
|
+
"CssSelectorError",
|
|
33
|
+
"Document",
|
|
34
|
+
"Node",
|
|
35
|
+
"XPathError",
|
|
36
|
+
"XPathQuery",
|
|
37
|
+
"decode_html",
|
|
38
|
+
"html_to_text",
|
|
39
|
+
"html_to_text_line_breaks",
|
|
40
|
+
"parse",
|
|
41
|
+
]
|
|
@@ -0,0 +1,636 @@
|
|
|
1
|
+
"""The public classes: Document, Node, CssQuery and XPathQuery."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
import threading
|
|
5
|
+
|
|
6
|
+
from . import _attrs as at
|
|
7
|
+
from . import _decode as dec
|
|
8
|
+
from . import _links
|
|
9
|
+
from . import _text
|
|
10
|
+
from ._chars import SPACE, ByteOffsets, ascii_lower, trim_range
|
|
11
|
+
from ._css import CssMatcher, CssSelectorError, CssTree
|
|
12
|
+
from ._parser import BOM, COMMENT, NAMES, is_raw_text, parse_tags
|
|
13
|
+
from ._query import ANY, Query
|
|
14
|
+
from ._xpath import XPathError, XPathMatcher
|
|
15
|
+
|
|
16
|
+
INF = float("inf")
|
|
17
|
+
TEXT_SPECIAL = re.compile(r"[<&\r]")
|
|
18
|
+
|
|
19
|
+
# ----- query caches: the string methods compile a query once -----
|
|
20
|
+
|
|
21
|
+
_lock = threading.Lock()
|
|
22
|
+
_css_cache = {}
|
|
23
|
+
_xpath_cache = {}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _cached(cache, key, compile_):
|
|
27
|
+
q = cache.get(key)
|
|
28
|
+
if q is not None:
|
|
29
|
+
return q
|
|
30
|
+
q = compile_(key)
|
|
31
|
+
with _lock:
|
|
32
|
+
if len(cache) >= 1024:
|
|
33
|
+
cache.clear() # dynamically built queries must not grow the cache forever
|
|
34
|
+
cache[key] = q
|
|
35
|
+
return q
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _css(selector):
|
|
39
|
+
if not isinstance(selector, str):
|
|
40
|
+
raise TypeError("a CSS selector must be a str")
|
|
41
|
+
return _cached(_css_cache, selector, CssMatcher)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _xpath(expression):
|
|
45
|
+
if not isinstance(expression, str):
|
|
46
|
+
raise TypeError("an XPath expression must be a str")
|
|
47
|
+
return _cached(_xpath_cache, expression, XPathMatcher)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _source_text(html):
|
|
51
|
+
if isinstance(html, str):
|
|
52
|
+
return html
|
|
53
|
+
if isinstance(html, (bytes, bytearray, memoryview)):
|
|
54
|
+
return bytes(html).decode("utf-8", "replace")
|
|
55
|
+
raise TypeError("parse() takes a str or bytes")
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def parse(html, url=None):
|
|
59
|
+
"""Parses an HTML or XML page. Bytes are read as UTF-8 (invalid sequences become U+FFFD).
|
|
60
|
+
|
|
61
|
+
With url, relative links (Node.href, Node.src) are resolved against it.
|
|
62
|
+
"""
|
|
63
|
+
return Document(html, url)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def decode_html(text):
|
|
67
|
+
"""Decodes the character references of a text (CR LF and a lone CR become LF)."""
|
|
68
|
+
if not isinstance(text, str):
|
|
69
|
+
raise TypeError("decode_html() takes a str")
|
|
70
|
+
return dec.decode(text, False)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def html_to_text(html):
|
|
74
|
+
"""The visible text of an HTML fragment: tags removed, whitespace collapsed, entities decoded."""
|
|
75
|
+
if not isinstance(html, str):
|
|
76
|
+
raise TypeError("html_to_text() takes a str")
|
|
77
|
+
return _text.clear_tags(html)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def html_to_text_line_breaks(html):
|
|
81
|
+
"""The text of an HTML fragment with the visible line structure kept."""
|
|
82
|
+
if not isinstance(html, str):
|
|
83
|
+
raise TypeError("html_to_text_line_breaks() takes a str")
|
|
84
|
+
return _text.clear_tags_line_breaks(html)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class Document:
|
|
88
|
+
"""A parsed HTML or XML page: a flat list of tags in document order.
|
|
89
|
+
|
|
90
|
+
`len(doc)`, `doc[i]` (negative indexes too) and iteration give the tags, comments included.
|
|
91
|
+
"""
|
|
92
|
+
|
|
93
|
+
__slots__ = ("src", "_url", "keys", "customs", "parents", "depths", "starts", "ends", "inner_starts",
|
|
94
|
+
"inner_ends", "lower_tags", "_tree", "_base", "_base_parsed", "_offsets", "__weakref__")
|
|
95
|
+
|
|
96
|
+
def __init__(self, html, url=None):
|
|
97
|
+
src = _source_text(html)
|
|
98
|
+
if src.startswith(BOM):
|
|
99
|
+
src = src[1:]
|
|
100
|
+
if url is not None and not isinstance(url, str):
|
|
101
|
+
raise TypeError("url must be a str")
|
|
102
|
+
self.src = src
|
|
103
|
+
self._url = url or ""
|
|
104
|
+
(self.keys, self.customs, self.parents, self.depths, self.starts, self.ends, self.inner_starts,
|
|
105
|
+
self.inner_ends) = parse_tags(src)
|
|
106
|
+
self.lower_tags = {}
|
|
107
|
+
self._tree = None
|
|
108
|
+
self._base = None
|
|
109
|
+
self._base_parsed = False
|
|
110
|
+
self._offsets = None
|
|
111
|
+
|
|
112
|
+
# ----- internals shared by the query engines -----
|
|
113
|
+
|
|
114
|
+
def css_tree(self):
|
|
115
|
+
t = self._tree
|
|
116
|
+
if t is None:
|
|
117
|
+
t = self._tree = CssTree(self)
|
|
118
|
+
return t
|
|
119
|
+
|
|
120
|
+
def base(self):
|
|
121
|
+
"""The page URL parsed once, for resolving relative links."""
|
|
122
|
+
if not self._base_parsed:
|
|
123
|
+
self._base = _links.absolute_url(self._url)
|
|
124
|
+
self._base_parsed = True
|
|
125
|
+
return self._base
|
|
126
|
+
|
|
127
|
+
def byte_offset(self, i):
|
|
128
|
+
if self._offsets is None:
|
|
129
|
+
self._offsets = ByteOffsets(self.src)
|
|
130
|
+
return self._offsets(i)
|
|
131
|
+
|
|
132
|
+
def name_of(self, i):
|
|
133
|
+
k = self.keys[i]
|
|
134
|
+
return NAMES[k] if k >= 0 else self.customs[i]
|
|
135
|
+
|
|
136
|
+
def body_of(self, i):
|
|
137
|
+
start = self.inner_starts[i]
|
|
138
|
+
end = self.inner_ends[i]
|
|
139
|
+
if start < 0 or end <= 0:
|
|
140
|
+
return ""
|
|
141
|
+
return self.src[start:end + 1].strip(SPACE)
|
|
142
|
+
|
|
143
|
+
def text_of(self, i):
|
|
144
|
+
body = self.body_of(i)
|
|
145
|
+
# no tags, references or CR: the body is the text
|
|
146
|
+
if TEXT_SPECIAL.search(body) is None:
|
|
147
|
+
return body
|
|
148
|
+
# No child tags and no '<' at all: the body is the text, except in script, style, textarea, title, ...
|
|
149
|
+
if not self.has_children_at(i) and ("<" not in body or is_raw_text(self.keys[i])):
|
|
150
|
+
return dec.decode(body, False) if "&" in body or "\r" in body else body
|
|
151
|
+
return _text.clear_tags(body)
|
|
152
|
+
|
|
153
|
+
def has_children_at(self, i):
|
|
154
|
+
return i + 1 < len(self.starts) and self.starts[i + 1] < self.ends[i]
|
|
155
|
+
|
|
156
|
+
def source_of(self, i):
|
|
157
|
+
src = self.src
|
|
158
|
+
start = self.starts[i]
|
|
159
|
+
end = self.ends[i]
|
|
160
|
+
if self.inner_starts[i] > 0 and self.inner_ends[i] > 0:
|
|
161
|
+
length = self.inner_starts[i] - start - 2
|
|
162
|
+
elif 0 < end and end - 1 < len(src) and src[end - 1] == "/":
|
|
163
|
+
length = end - start - 2
|
|
164
|
+
else:
|
|
165
|
+
length = end - start - 1
|
|
166
|
+
return trim_range(src, start + 1, start + 1 + length)
|
|
167
|
+
|
|
168
|
+
def next_matching(self, i, q):
|
|
169
|
+
starts = self.starts
|
|
170
|
+
end = self.ends[i]
|
|
171
|
+
for k in range(i + 1, len(starts)):
|
|
172
|
+
if starts[k] > end and q.matches(self, k):
|
|
173
|
+
return k
|
|
174
|
+
return -1
|
|
175
|
+
|
|
176
|
+
def sibling_matching(self, i, q, forward):
|
|
177
|
+
# deeper tags are inside the siblings; a shallower one ends (or is) the parent
|
|
178
|
+
depths = self.depths
|
|
179
|
+
depth = depths[i]
|
|
180
|
+
count = len(depths)
|
|
181
|
+
step = 1 if forward else -1
|
|
182
|
+
k = i + step
|
|
183
|
+
while 0 <= k < count and depths[k] >= depth:
|
|
184
|
+
if depths[k] == depth and q.matches(self, k):
|
|
185
|
+
return k
|
|
186
|
+
k += step
|
|
187
|
+
return -1
|
|
188
|
+
|
|
189
|
+
def _node(self, i):
|
|
190
|
+
return Node(self, i) if i >= 0 else None
|
|
191
|
+
|
|
192
|
+
def _nodes(self, indexes):
|
|
193
|
+
return [Node(self, i) for i in indexes]
|
|
194
|
+
|
|
195
|
+
# ----- the public API -----
|
|
196
|
+
|
|
197
|
+
@property
|
|
198
|
+
def url(self):
|
|
199
|
+
"""The page URL given to parse(), or ""."""
|
|
200
|
+
return self._url
|
|
201
|
+
|
|
202
|
+
@property
|
|
203
|
+
def source(self):
|
|
204
|
+
"""The page source."""
|
|
205
|
+
return self.src
|
|
206
|
+
|
|
207
|
+
def __len__(self):
|
|
208
|
+
return len(self.keys)
|
|
209
|
+
|
|
210
|
+
def __getitem__(self, index):
|
|
211
|
+
if not isinstance(index, int):
|
|
212
|
+
raise TypeError("tag indexes must be integers")
|
|
213
|
+
n = len(self.keys)
|
|
214
|
+
i = index + n if index < 0 else index
|
|
215
|
+
if i < 0 or i >= n:
|
|
216
|
+
raise IndexError("tag index out of range")
|
|
217
|
+
return Node(self, i)
|
|
218
|
+
|
|
219
|
+
def __iter__(self):
|
|
220
|
+
for i in range(len(self.keys)):
|
|
221
|
+
yield Node(self, i)
|
|
222
|
+
|
|
223
|
+
def __repr__(self):
|
|
224
|
+
return f"<Document url={self._url!r} tags={len(self.keys)}>"
|
|
225
|
+
|
|
226
|
+
def find(self, name=None, **criteria):
|
|
227
|
+
"""The first tag that matches: by name (any name, case-insensitive) and the criteria; or None."""
|
|
228
|
+
r = Query.of(name, criteria).scan(self, 0, INF, True)
|
|
229
|
+
return Node(self, r[0]) if r else None
|
|
230
|
+
|
|
231
|
+
def find_all(self, name=None, **criteria):
|
|
232
|
+
"""All tags that match, in document order."""
|
|
233
|
+
return self._nodes(Query.of(name, criteria).scan(self, 0, INF, False))
|
|
234
|
+
|
|
235
|
+
def css(self, selector):
|
|
236
|
+
"""The tags that match the CSS selector (querySelectorAll). Raises CssSelectorError on an invalid one."""
|
|
237
|
+
return self._nodes(_css(selector).scan(self, None, 0, INF, False))
|
|
238
|
+
|
|
239
|
+
def css_first(self, selector):
|
|
240
|
+
"""The first tag that matches the CSS selector (querySelector), or None."""
|
|
241
|
+
r = _css(selector).scan(self, None, 0, INF, True)
|
|
242
|
+
return Node(self, r[0]) if r else None
|
|
243
|
+
|
|
244
|
+
def xpath(self, expression):
|
|
245
|
+
"""The tags the XPath 1.0 expression selects."""
|
|
246
|
+
return self._nodes(_xpath(expression).tags(self, None))
|
|
247
|
+
|
|
248
|
+
def xpath_first(self, expression):
|
|
249
|
+
"""The first tag the XPath expression selects, or None."""
|
|
250
|
+
r = _xpath(expression).tags(self, None)
|
|
251
|
+
return Node(self, r[0]) if r else None
|
|
252
|
+
|
|
253
|
+
def xpath_values(self, expression):
|
|
254
|
+
"""The values the XPath expression selects: decoded attributes, text(), visible text of tags, or the result
|
|
255
|
+
of count(), sum(), string(), ..."""
|
|
256
|
+
return _xpath(expression).values(self, None)
|
|
257
|
+
|
|
258
|
+
def xpath_value(self, expression):
|
|
259
|
+
"""The first value the XPath expression selects, or ""."""
|
|
260
|
+
v = _xpath(expression).value(self, None)
|
|
261
|
+
return "" if v is None else v
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
class Node:
|
|
265
|
+
"""One tag of a document. Positions (pos) are byte offsets into the UTF-8 source.
|
|
266
|
+
|
|
267
|
+
Nodes compare equal when they are the same tag of the same document, and are hashable.
|
|
268
|
+
"""
|
|
269
|
+
|
|
270
|
+
__slots__ = ("_doc", "_i")
|
|
271
|
+
|
|
272
|
+
def __init__(self, doc, index):
|
|
273
|
+
self._doc = doc
|
|
274
|
+
self._i = index
|
|
275
|
+
|
|
276
|
+
@property
|
|
277
|
+
def name(self):
|
|
278
|
+
"""The lower-case tag name ("div", "!--" for comments, "custom-tag", ...)."""
|
|
279
|
+
return self._doc.name_of(self._i)
|
|
280
|
+
|
|
281
|
+
@property
|
|
282
|
+
def index(self):
|
|
283
|
+
"""The position of the tag in the document."""
|
|
284
|
+
return self._i
|
|
285
|
+
|
|
286
|
+
@property
|
|
287
|
+
def depth(self):
|
|
288
|
+
"""The nesting depth: 0 for top-level tags."""
|
|
289
|
+
return self._doc.depths[self._i]
|
|
290
|
+
|
|
291
|
+
@property
|
|
292
|
+
def pos(self):
|
|
293
|
+
"""The byte offsets of the opening '<' and of the last '>' in the UTF-8 source."""
|
|
294
|
+
d = self._doc
|
|
295
|
+
# the end is the last byte of the last char (a cut-off page ends inside the text)
|
|
296
|
+
return d.byte_offset(d.starts[self._i]), d.byte_offset(d.ends[self._i] + 1) - 1
|
|
297
|
+
|
|
298
|
+
@property
|
|
299
|
+
def text(self):
|
|
300
|
+
"""The visible text: tags, comments, script and style removed, entities decoded, whitespace collapsed."""
|
|
301
|
+
return self._doc.text_of(self._i)
|
|
302
|
+
|
|
303
|
+
@property
|
|
304
|
+
def text_line_breaks(self):
|
|
305
|
+
"""The text with the visible line structure kept: <br> and block elements give new lines, p and headings
|
|
306
|
+
paragraphs."""
|
|
307
|
+
d = self._doc
|
|
308
|
+
if d.inner_starts[self._i] < 0 or d.inner_ends[self._i] <= 0:
|
|
309
|
+
return ""
|
|
310
|
+
return _text.clear_tags_line_breaks(d.body_of(self._i))
|
|
311
|
+
|
|
312
|
+
@property
|
|
313
|
+
def body(self):
|
|
314
|
+
"""The inner HTML (trimmed); "" for tags that are not closed."""
|
|
315
|
+
return self._doc.body_of(self._i)
|
|
316
|
+
|
|
317
|
+
@property
|
|
318
|
+
def outer_html(self):
|
|
319
|
+
"""The source of the whole element; "" for tags that are not closed."""
|
|
320
|
+
d = self._doc
|
|
321
|
+
i = self._i
|
|
322
|
+
if d.inner_starts[i] <= 0 or d.inner_ends[i] <= 0:
|
|
323
|
+
return ""
|
|
324
|
+
return trim_range(d.src, d.starts[i], d.ends[i] + 1)
|
|
325
|
+
|
|
326
|
+
@property
|
|
327
|
+
def source(self):
|
|
328
|
+
"""The opening tag without '<' and '>': the name and the attributes."""
|
|
329
|
+
return self._doc.source_of(self._i)
|
|
330
|
+
|
|
331
|
+
@property
|
|
332
|
+
def attrs(self):
|
|
333
|
+
"""The attributes as a dict of decoded values, names in lower case (the first of a repeated name wins)."""
|
|
334
|
+
d = self._doc
|
|
335
|
+
i = self._i
|
|
336
|
+
out = {}
|
|
337
|
+
if d.keys[i] == COMMENT:
|
|
338
|
+
return out
|
|
339
|
+
s = at.AttrScanner(d, i)
|
|
340
|
+
while s.next():
|
|
341
|
+
name = s.name()
|
|
342
|
+
key = name.lower()
|
|
343
|
+
if key not in out:
|
|
344
|
+
out[key] = at.attr_value(d, i, ascii_lower(name))
|
|
345
|
+
return out
|
|
346
|
+
|
|
347
|
+
def attr(self, name):
|
|
348
|
+
"""The decoded value of the first attribute with the name (case-insensitive); "" when it is missing."""
|
|
349
|
+
return at.attr_value(self._doc, self._i, ascii_lower(name))
|
|
350
|
+
|
|
351
|
+
def attr_raw(self, name):
|
|
352
|
+
"""The value of the attribute as written in the source."""
|
|
353
|
+
return at.attr_raw(self._doc, self._i, ascii_lower(name))
|
|
354
|
+
|
|
355
|
+
def has_class(self, class_):
|
|
356
|
+
"""Whether one of the space-separated classes equals class_ (case-insensitive)."""
|
|
357
|
+
return at.has_class(self._doc, self._i, class_)
|
|
358
|
+
|
|
359
|
+
@property
|
|
360
|
+
def href(self):
|
|
361
|
+
"""The link a browser follows: href decoded and made absolute against the page URL. An empty or missing
|
|
362
|
+
href gives the page URL."""
|
|
363
|
+
d = self._doc
|
|
364
|
+
return _links.correct_link(at.attr_raw(d, self._i, "href"), d.url, d.base())
|
|
365
|
+
|
|
366
|
+
@property
|
|
367
|
+
def src(self):
|
|
368
|
+
"""src decoded and made absolute against the page URL."""
|
|
369
|
+
d = self._doc
|
|
370
|
+
return _links.correct_link(at.attr_raw(d, self._i, "src"), d.url, d.base())
|
|
371
|
+
|
|
372
|
+
def href_from(self, base_url):
|
|
373
|
+
"""href resolved against base_url instead of the page URL."""
|
|
374
|
+
return _links.correct_link(at.attr_raw(self._doc, self._i, "href"), base_url, _links.absolute_url(base_url))
|
|
375
|
+
|
|
376
|
+
def src_from(self, base_url):
|
|
377
|
+
"""src resolved against base_url instead of the page URL."""
|
|
378
|
+
return _links.correct_link(at.attr_raw(self._doc, self._i, "src"), base_url, _links.absolute_url(base_url))
|
|
379
|
+
|
|
380
|
+
@property
|
|
381
|
+
def mailto(self):
|
|
382
|
+
"""The address of a mailto: link, or ""."""
|
|
383
|
+
return _links.after_scheme(at.attr_raw(self._doc, self._i, "href"), "mailto")
|
|
384
|
+
|
|
385
|
+
@property
|
|
386
|
+
def tel(self):
|
|
387
|
+
"""The number of a tel: link, or ""."""
|
|
388
|
+
return _links.after_scheme(at.attr_raw(self._doc, self._i, "href"), "tel")
|
|
389
|
+
|
|
390
|
+
@property
|
|
391
|
+
def text_after(self):
|
|
392
|
+
"""The source text right after the tag up to the next '<'."""
|
|
393
|
+
src = self._doc.src
|
|
394
|
+
start = self._doc.ends[self._i] + 1
|
|
395
|
+
if start >= len(src):
|
|
396
|
+
return ""
|
|
397
|
+
nxt = src.find("<", start)
|
|
398
|
+
return src[start:] if nxt < 0 else src[start:nxt]
|
|
399
|
+
|
|
400
|
+
# ----- navigation -----
|
|
401
|
+
|
|
402
|
+
@property
|
|
403
|
+
def parent(self):
|
|
404
|
+
"""The enclosing tag, or None."""
|
|
405
|
+
return self._doc._node(self._doc.parents[self._i])
|
|
406
|
+
|
|
407
|
+
@property
|
|
408
|
+
def children(self):
|
|
409
|
+
"""The direct children."""
|
|
410
|
+
d = self._doc
|
|
411
|
+
ends = d.ends
|
|
412
|
+
depths = d.depths
|
|
413
|
+
depth = depths[self._i] + 1
|
|
414
|
+
end = ends[self._i]
|
|
415
|
+
out = []
|
|
416
|
+
i = self._i + 1
|
|
417
|
+
count = len(ends)
|
|
418
|
+
while i < count and ends[i] <= end:
|
|
419
|
+
if depths[i] == depth:
|
|
420
|
+
out.append(Node(d, i))
|
|
421
|
+
i += 1
|
|
422
|
+
return out
|
|
423
|
+
|
|
424
|
+
@property
|
|
425
|
+
def has_children(self):
|
|
426
|
+
"""Whether another tag starts inside this one."""
|
|
427
|
+
return self._doc.has_children_at(self._i)
|
|
428
|
+
|
|
429
|
+
@property
|
|
430
|
+
def next(self):
|
|
431
|
+
"""The tag that follows in document order (the first child, if there is one), or None."""
|
|
432
|
+
i = self._i + 1
|
|
433
|
+
return Node(self._doc, i) if i < len(self._doc.keys) else None
|
|
434
|
+
|
|
435
|
+
@property
|
|
436
|
+
def previous(self):
|
|
437
|
+
"""The tag before this one in document order, or None."""
|
|
438
|
+
return Node(self._doc, self._i - 1) if self._i > 0 else None
|
|
439
|
+
|
|
440
|
+
@property
|
|
441
|
+
def next_sibling(self):
|
|
442
|
+
"""The next tag at the same level inside the same parent, or None."""
|
|
443
|
+
return self._doc._node(self._doc.sibling_matching(self._i, ANY, True))
|
|
444
|
+
|
|
445
|
+
@property
|
|
446
|
+
def previous_sibling(self):
|
|
447
|
+
"""The previous tag at the same level inside the same parent, or None."""
|
|
448
|
+
return self._doc._node(self._doc.sibling_matching(self._i, ANY, False))
|
|
449
|
+
|
|
450
|
+
# ----- search -----
|
|
451
|
+
|
|
452
|
+
def find(self, name=None, **criteria):
|
|
453
|
+
"""The first tag inside this one that matches, or None."""
|
|
454
|
+
d = self._doc
|
|
455
|
+
r = Query.of(name, criteria).scan(d, self._i + 1, d.ends[self._i], True)
|
|
456
|
+
return Node(d, r[0]) if r else None
|
|
457
|
+
|
|
458
|
+
def find_all(self, name=None, **criteria):
|
|
459
|
+
"""All tags inside this one that match."""
|
|
460
|
+
d = self._doc
|
|
461
|
+
return d._nodes(Query.of(name, criteria).scan(d, self._i + 1, d.ends[self._i], False))
|
|
462
|
+
|
|
463
|
+
def matches(self, name=None, **criteria):
|
|
464
|
+
"""Whether this tag matches."""
|
|
465
|
+
return Query.of(name, criteria).matches(self._doc, self._i)
|
|
466
|
+
|
|
467
|
+
def find_parent(self, name=None, **criteria):
|
|
468
|
+
"""The nearest enclosing tag that matches, or None."""
|
|
469
|
+
q = Query.of(name, criteria)
|
|
470
|
+
d = self._doc
|
|
471
|
+
p = d.parents[self._i]
|
|
472
|
+
while p >= 0:
|
|
473
|
+
if q.matches(d, p):
|
|
474
|
+
return Node(d, p)
|
|
475
|
+
p = d.parents[p]
|
|
476
|
+
return None
|
|
477
|
+
|
|
478
|
+
def find_next(self, name=None, **criteria):
|
|
479
|
+
"""The first tag after the end of this one that matches (tags inside it are skipped), or None."""
|
|
480
|
+
return self._doc._node(self._doc.next_matching(self._i, Query.of(name, criteria)))
|
|
481
|
+
|
|
482
|
+
def find_next_sibling(self, name=None, **criteria):
|
|
483
|
+
"""The next sibling that matches, or None."""
|
|
484
|
+
return self._doc._node(self._doc.sibling_matching(self._i, Query.of(name, criteria), True))
|
|
485
|
+
|
|
486
|
+
def find_previous_sibling(self, name=None, **criteria):
|
|
487
|
+
"""The nearest previous sibling that matches, or None."""
|
|
488
|
+
return self._doc._node(self._doc.sibling_matching(self._i, Query.of(name, criteria), False))
|
|
489
|
+
|
|
490
|
+
def text_after_until(self, name=None, **criteria):
|
|
491
|
+
"""The source between the end of this tag and the next tag that matches; "" when there is none."""
|
|
492
|
+
d = self._doc
|
|
493
|
+
k = d.next_matching(self._i, Query.of(name, criteria))
|
|
494
|
+
if k < 0:
|
|
495
|
+
return ""
|
|
496
|
+
start = d.ends[self._i] + 1
|
|
497
|
+
end = d.starts[k]
|
|
498
|
+
return d.src[start:end] if start < end else ""
|
|
499
|
+
|
|
500
|
+
def css(self, selector):
|
|
501
|
+
"""The tags inside this one that match the CSS selector; it may start with '>' for the children."""
|
|
502
|
+
d = self._doc
|
|
503
|
+
return d._nodes(_css(selector).scan(d, self._i, self._i + 1, d.ends[self._i], False))
|
|
504
|
+
|
|
505
|
+
def css_first(self, selector):
|
|
506
|
+
"""The first tag inside this one that matches the CSS selector, or None."""
|
|
507
|
+
d = self._doc
|
|
508
|
+
r = _css(selector).scan(d, self._i, self._i + 1, d.ends[self._i], True)
|
|
509
|
+
return Node(d, r[0]) if r else None
|
|
510
|
+
|
|
511
|
+
def css_matches(self, selector):
|
|
512
|
+
"""Whether this tag matches the CSS selector (element.matches())."""
|
|
513
|
+
return _css(selector).matches(self._doc, self._i)
|
|
514
|
+
|
|
515
|
+
def xpath(self, expression):
|
|
516
|
+
"""The tags the XPath expression selects, relative paths starting at this tag."""
|
|
517
|
+
return self._doc._nodes(_xpath(expression).tags(self._doc, self._i))
|
|
518
|
+
|
|
519
|
+
def xpath_first(self, expression):
|
|
520
|
+
"""The first tag the XPath expression selects from this tag, or None."""
|
|
521
|
+
r = _xpath(expression).tags(self._doc, self._i)
|
|
522
|
+
return Node(self._doc, r[0]) if r else None
|
|
523
|
+
|
|
524
|
+
def xpath_values(self, expression):
|
|
525
|
+
"""The values the XPath expression selects from this tag."""
|
|
526
|
+
return _xpath(expression).values(self._doc, self._i)
|
|
527
|
+
|
|
528
|
+
def xpath_value(self, expression):
|
|
529
|
+
"""The first value the XPath expression selects from this tag, or ""."""
|
|
530
|
+
v = _xpath(expression).value(self._doc, self._i)
|
|
531
|
+
return "" if v is None else v
|
|
532
|
+
|
|
533
|
+
def __eq__(self, other):
|
|
534
|
+
return isinstance(other, Node) and other._doc is self._doc and other._i == self._i
|
|
535
|
+
|
|
536
|
+
def __ne__(self, other):
|
|
537
|
+
return not self.__eq__(other)
|
|
538
|
+
|
|
539
|
+
def __hash__(self):
|
|
540
|
+
return hash((id(self._doc), self._i))
|
|
541
|
+
|
|
542
|
+
def __repr__(self):
|
|
543
|
+
source = self.source
|
|
544
|
+
short = source[:80]
|
|
545
|
+
more = "..." if len(short) < len(source) else ""
|
|
546
|
+
return f"<Node {self._i} <{short}{more}>>"
|
|
547
|
+
|
|
548
|
+
|
|
549
|
+
def _target(obj):
|
|
550
|
+
if isinstance(obj, Document):
|
|
551
|
+
return obj, None
|
|
552
|
+
if isinstance(obj, Node):
|
|
553
|
+
return obj._doc, obj._i
|
|
554
|
+
raise TypeError("expected a Document or a Node")
|
|
555
|
+
|
|
556
|
+
|
|
557
|
+
class CssQuery:
|
|
558
|
+
"""A compiled CSS selector: compile once, use on any number of pages and threads."""
|
|
559
|
+
|
|
560
|
+
__slots__ = ("_m",)
|
|
561
|
+
|
|
562
|
+
def __init__(self, selector):
|
|
563
|
+
if not isinstance(selector, str):
|
|
564
|
+
raise TypeError("a CSS selector must be a str")
|
|
565
|
+
self._m = CssMatcher(selector)
|
|
566
|
+
|
|
567
|
+
@property
|
|
568
|
+
def selector(self):
|
|
569
|
+
return self._m.selector
|
|
570
|
+
|
|
571
|
+
def all(self, target, /):
|
|
572
|
+
"""The matching tags in a document, or inside a node."""
|
|
573
|
+
d, i = _target(target)
|
|
574
|
+
if i is None:
|
|
575
|
+
return d._nodes(self._m.scan(d, None, 0, INF, False))
|
|
576
|
+
return d._nodes(self._m.scan(d, i, i + 1, d.ends[i], False))
|
|
577
|
+
|
|
578
|
+
def first(self, target, /):
|
|
579
|
+
"""The first matching tag in a document, or inside a node, or None."""
|
|
580
|
+
d, i = _target(target)
|
|
581
|
+
if i is None:
|
|
582
|
+
r = self._m.scan(d, None, 0, INF, True)
|
|
583
|
+
else:
|
|
584
|
+
r = self._m.scan(d, i, i + 1, d.ends[i], True)
|
|
585
|
+
return Node(d, r[0]) if r else None
|
|
586
|
+
|
|
587
|
+
def matches(self, node, /):
|
|
588
|
+
"""Whether the tag matches (element.matches())."""
|
|
589
|
+
if not isinstance(node, Node):
|
|
590
|
+
raise TypeError("expected a Node")
|
|
591
|
+
return self._m.matches(node._doc, node._i)
|
|
592
|
+
|
|
593
|
+
def __repr__(self):
|
|
594
|
+
return f"CssQuery({self._m.selector!r})"
|
|
595
|
+
|
|
596
|
+
|
|
597
|
+
class XPathQuery:
|
|
598
|
+
"""A compiled XPath 1.0 expression: compile once, use on any number of pages and threads."""
|
|
599
|
+
|
|
600
|
+
__slots__ = ("_m",)
|
|
601
|
+
|
|
602
|
+
def __init__(self, expression):
|
|
603
|
+
if not isinstance(expression, str):
|
|
604
|
+
raise TypeError("an XPath expression must be a str")
|
|
605
|
+
self._m = XPathMatcher(expression)
|
|
606
|
+
|
|
607
|
+
@property
|
|
608
|
+
def expression(self):
|
|
609
|
+
return self._m.expression
|
|
610
|
+
|
|
611
|
+
@property
|
|
612
|
+
def is_node_set(self):
|
|
613
|
+
"""Whether the expression gives tags (a node-set), not a number, string or boolean."""
|
|
614
|
+
return self._m.is_node_set
|
|
615
|
+
|
|
616
|
+
def nodes(self, target, /):
|
|
617
|
+
"""The selected tags, evaluated on a document or from a node."""
|
|
618
|
+
d, i = _target(target)
|
|
619
|
+
return d._nodes(self._m.tags(d, i))
|
|
620
|
+
|
|
621
|
+
def values(self, target, /):
|
|
622
|
+
"""The values of the selected nodes, or the result of the expression as one value."""
|
|
623
|
+
d, i = _target(target)
|
|
624
|
+
return self._m.values(d, i)
|
|
625
|
+
|
|
626
|
+
def value(self, target, /):
|
|
627
|
+
"""The value of the first selected node, or the result of the expression; None when it selects nothing."""
|
|
628
|
+
d, i = _target(target)
|
|
629
|
+
return self._m.value(d, i)
|
|
630
|
+
|
|
631
|
+
def __repr__(self):
|
|
632
|
+
return f"XPathQuery({self._m.expression!r})"
|
|
633
|
+
|
|
634
|
+
|
|
635
|
+
__all__ = ["CssQuery", "CssSelectorError", "Document", "Node", "XPathError", "XPathQuery", "decode_html",
|
|
636
|
+
"html_to_text", "html_to_text_line_breaks", "parse"]
|