selectolax 0.4.4__cp310-cp310-macosx_10_9_x86_64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- selectolax/__init__.py +8 -0
- selectolax/base.pxi +4 -0
- selectolax/lexbor/attrs.pxi +120 -0
- selectolax/lexbor/node.pxi +1112 -0
- selectolax/lexbor/node_remove.pxi +29 -0
- selectolax/lexbor/selection.pxi +215 -0
- selectolax/lexbor/util.pxi +20 -0
- selectolax/lexbor.c +53768 -0
- selectolax/lexbor.cpython-310-darwin.so +0 -0
- selectolax/lexbor.pxd +599 -0
- selectolax/lexbor.pyi +1248 -0
- selectolax/lexbor.pyx +677 -0
- selectolax/modest/node.pxi +991 -0
- selectolax/modest/selection.pxi +195 -0
- selectolax/modest/util.pxi +20 -0
- selectolax/parser.c +47848 -0
- selectolax/parser.cpython-310-darwin.so +0 -0
- selectolax/parser.pxd +578 -0
- selectolax/parser.pyi +770 -0
- selectolax/parser.pyx +443 -0
- selectolax/py.typed +0 -0
- selectolax/utils.pxi +117 -0
- selectolax-0.4.4.dist-info/METADATA +222 -0
- selectolax-0.4.4.dist-info/RECORD +27 -0
- selectolax-0.4.4.dist-info/WHEEL +6 -0
- selectolax-0.4.4.dist-info/licenses/LICENSE +10 -0
- selectolax-0.4.4.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,991 @@
|
|
|
1
|
+
cimport cython
|
|
2
|
+
from cpython.exc cimport PyErr_NoMemory
|
|
3
|
+
|
|
4
|
+
from libc.stdlib cimport free
|
|
5
|
+
from libc.stdlib cimport malloc
|
|
6
|
+
from libc.stdlib cimport realloc
|
|
7
|
+
from libc.string cimport memcpy
|
|
8
|
+
|
|
9
|
+
DEF _STACK_SIZE = 100
|
|
10
|
+
DEF _ENCODING = 'UTF-8'
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@cython.final
|
|
14
|
+
cdef class Stack:
|
|
15
|
+
def __cinit__(self, size_t capacity=25):
|
|
16
|
+
self.capacity = capacity
|
|
17
|
+
self.top = 0
|
|
18
|
+
self._stack = <myhtml_tree_node_t**> malloc(capacity * sizeof(myhtml_tree_node_t))
|
|
19
|
+
if self._stack == NULL:
|
|
20
|
+
raise MemoryError("Failed to allocate memory for stack")
|
|
21
|
+
|
|
22
|
+
def __dealloc__(self):
|
|
23
|
+
free(self._stack)
|
|
24
|
+
|
|
25
|
+
cdef bint is_empty(self):
|
|
26
|
+
return self.top <= 0
|
|
27
|
+
|
|
28
|
+
cdef int push(self, myhtml_tree_node_t* res) except -1:
|
|
29
|
+
if self.top >= self.capacity:
|
|
30
|
+
if self.resize() < 0:
|
|
31
|
+
return -1
|
|
32
|
+
self._stack[self.top] = res
|
|
33
|
+
self.top += 1
|
|
34
|
+
|
|
35
|
+
cdef myhtml_tree_node_t * pop(self):
|
|
36
|
+
self.top = self.top - 1
|
|
37
|
+
return self._stack[self.top]
|
|
38
|
+
|
|
39
|
+
cdef int resize(self) except -1:
|
|
40
|
+
self.capacity *= 2
|
|
41
|
+
self._stack = <myhtml_tree_node_t**> realloc(<void*> self._stack, self.capacity * sizeof(myhtml_tree_node_t))
|
|
42
|
+
if self._stack == NULL:
|
|
43
|
+
PyErr_NoMemory()
|
|
44
|
+
return -1
|
|
45
|
+
return 0
|
|
46
|
+
|
|
47
|
+
cdef class _Attributes:
|
|
48
|
+
"""A dict-like object that represents attributes."""
|
|
49
|
+
cdef myhtml_tree_node_t * node
|
|
50
|
+
cdef unicode decode_errors
|
|
51
|
+
|
|
52
|
+
@staticmethod
|
|
53
|
+
cdef _Attributes create(myhtml_tree_node_t *node, unicode decode_errors):
|
|
54
|
+
obj = <_Attributes>_Attributes.__new__(_Attributes)
|
|
55
|
+
obj.node = node
|
|
56
|
+
obj.decode_errors = decode_errors
|
|
57
|
+
return obj
|
|
58
|
+
|
|
59
|
+
def __iter__(self):
|
|
60
|
+
cdef myhtml_tree_attr_t *attr = myhtml_node_attribute_first(self.node)
|
|
61
|
+
while attr:
|
|
62
|
+
if attr.key.data == NULL:
|
|
63
|
+
attr = attr.next
|
|
64
|
+
continue
|
|
65
|
+
key = attr.key.data.decode(_ENCODING, self.decode_errors)
|
|
66
|
+
attr = attr.next
|
|
67
|
+
yield key
|
|
68
|
+
|
|
69
|
+
def __setitem__(self, str key, value):
|
|
70
|
+
value = str(value)
|
|
71
|
+
bytes_key = key.encode(_ENCODING)
|
|
72
|
+
bytes_value = value.encode(_ENCODING)
|
|
73
|
+
myhtml_attribute_remove_by_key(self.node, <char*>bytes_key, len(bytes_key))
|
|
74
|
+
myhtml_attribute_add(self.node, <char*>bytes_key, len(bytes_key), <char*>bytes_value, len(bytes_value),
|
|
75
|
+
MyENCODING_UTF_8)
|
|
76
|
+
|
|
77
|
+
def __delitem__(self, key):
|
|
78
|
+
try:
|
|
79
|
+
self.__getitem__(key)
|
|
80
|
+
except KeyError:
|
|
81
|
+
raise KeyError(key)
|
|
82
|
+
bytes_key = key.encode(_ENCODING)
|
|
83
|
+
myhtml_attribute_remove_by_key(self.node, <char*>bytes_key, len(bytes_key))
|
|
84
|
+
|
|
85
|
+
def __getitem__(self, str key):
|
|
86
|
+
bytes_key = key.encode(_ENCODING)
|
|
87
|
+
cdef myhtml_tree_attr_t * attr = myhtml_attribute_by_key(self.node, <char*>bytes_key, len(bytes_key))
|
|
88
|
+
if attr != NULL:
|
|
89
|
+
if attr.value.data != NULL:
|
|
90
|
+
return attr.value.data.decode(_ENCODING, self.decode_errors)
|
|
91
|
+
elif attr.key.data != NULL:
|
|
92
|
+
return None
|
|
93
|
+
raise KeyError(key)
|
|
94
|
+
|
|
95
|
+
def __len__(self):
|
|
96
|
+
return len(list(self.__iter__()))
|
|
97
|
+
|
|
98
|
+
def keys(self):
|
|
99
|
+
return self.__iter__()
|
|
100
|
+
|
|
101
|
+
def items(self):
|
|
102
|
+
for key in self.__iter__():
|
|
103
|
+
yield key, self[key]
|
|
104
|
+
|
|
105
|
+
def values(self):
|
|
106
|
+
for key in self.__iter__():
|
|
107
|
+
yield self[key]
|
|
108
|
+
|
|
109
|
+
def get(self, key, default=None):
|
|
110
|
+
try:
|
|
111
|
+
return self[key]
|
|
112
|
+
except KeyError:
|
|
113
|
+
return default
|
|
114
|
+
|
|
115
|
+
def sget(self, key, default=""):
|
|
116
|
+
"""Same as get, but returns empty strings instead of None values for empty attributes."""
|
|
117
|
+
try:
|
|
118
|
+
val = self[key]
|
|
119
|
+
if val is None:
|
|
120
|
+
val = ""
|
|
121
|
+
return val
|
|
122
|
+
except KeyError:
|
|
123
|
+
return default
|
|
124
|
+
|
|
125
|
+
def __contains__(self, key):
|
|
126
|
+
try:
|
|
127
|
+
self[key]
|
|
128
|
+
except KeyError:
|
|
129
|
+
return False
|
|
130
|
+
else:
|
|
131
|
+
return True
|
|
132
|
+
|
|
133
|
+
def __repr__(self):
|
|
134
|
+
cdef const char *c_text
|
|
135
|
+
c_text = myhtml_tag_name_by_id(self.node.tree, self.node.tag_id, NULL)
|
|
136
|
+
tag_name = c_text.decode(_ENCODING, 'ignore') if c_text != NULL else 'unknown'
|
|
137
|
+
return "<%s attributes, %s items>" % (tag_name, len(self))
|
|
138
|
+
|
|
139
|
+
ctypedef fused str_or_Node:
|
|
140
|
+
str
|
|
141
|
+
bytes
|
|
142
|
+
Node
|
|
143
|
+
|
|
144
|
+
cdef class Node:
|
|
145
|
+
"""A class that represents HTML node (element)."""
|
|
146
|
+
cdef myhtml_tree_node_t *node
|
|
147
|
+
cdef public HTMLParser parser
|
|
148
|
+
|
|
149
|
+
@staticmethod
|
|
150
|
+
cdef Node new(myhtml_tree_node_t *node, HTMLParser parser):
|
|
151
|
+
# custom __init__ for C, because __cinit__ doesn't accept C types
|
|
152
|
+
cdef Node cls = Node.__new__(Node)
|
|
153
|
+
cls.node = node
|
|
154
|
+
# Keep reference to the selector object, so myhtml structures will not be garbage collected prematurely
|
|
155
|
+
cls.parser = parser
|
|
156
|
+
return cls
|
|
157
|
+
|
|
158
|
+
@property
|
|
159
|
+
def attributes(self):
|
|
160
|
+
"""Get all attributes that belong to the current node.
|
|
161
|
+
|
|
162
|
+
The value of empty attributes is None.
|
|
163
|
+
|
|
164
|
+
Returns
|
|
165
|
+
-------
|
|
166
|
+
attributes : dictionary of all attributes.
|
|
167
|
+
|
|
168
|
+
Examples
|
|
169
|
+
--------
|
|
170
|
+
|
|
171
|
+
>>> tree = HTMLParser("<div data id='my_id'></div>")
|
|
172
|
+
>>> node = tree.css_first('div')
|
|
173
|
+
>>> node.attributes
|
|
174
|
+
{'data': None, 'id': 'my_id'}
|
|
175
|
+
"""
|
|
176
|
+
cdef myhtml_tree_attr_t *attr = myhtml_node_attribute_first(self.node)
|
|
177
|
+
attributes = dict()
|
|
178
|
+
|
|
179
|
+
while attr:
|
|
180
|
+
if attr.key.data == NULL:
|
|
181
|
+
attr = attr.next
|
|
182
|
+
continue
|
|
183
|
+
key = attr.key.data.decode(_ENCODING, self.parser.decode_errors)
|
|
184
|
+
if attr.value.data:
|
|
185
|
+
value = attr.value.data.decode(_ENCODING, self.parser.decode_errors)
|
|
186
|
+
else:
|
|
187
|
+
value = None
|
|
188
|
+
attributes[key] = value
|
|
189
|
+
|
|
190
|
+
attr = attr.next
|
|
191
|
+
|
|
192
|
+
return attributes
|
|
193
|
+
|
|
194
|
+
@property
|
|
195
|
+
def attrs(self):
|
|
196
|
+
"""A dict-like object that is similar to the ``attributes`` property, but operates directly on the Node data.
|
|
197
|
+
|
|
198
|
+
.. warning:: Use ``attributes`` instead, if you don't want to modify Node attributes.
|
|
199
|
+
|
|
200
|
+
Returns
|
|
201
|
+
-------
|
|
202
|
+
attributes : Attributes mapping object.
|
|
203
|
+
|
|
204
|
+
Examples
|
|
205
|
+
--------
|
|
206
|
+
|
|
207
|
+
>>> tree = HTMLParser("<div id='a'></div>")
|
|
208
|
+
>>> node = tree.css_first('div')
|
|
209
|
+
>>> node.attrs
|
|
210
|
+
<div attributes, 1 items>
|
|
211
|
+
>>> node.attrs['id']
|
|
212
|
+
'a'
|
|
213
|
+
>>> node.attrs['foo'] = 'bar'
|
|
214
|
+
>>> del node.attrs['id']
|
|
215
|
+
>>> node.attributes
|
|
216
|
+
{'foo': 'bar'}
|
|
217
|
+
>>> node.attrs['id'] = 'new_id'
|
|
218
|
+
>>> node.html
|
|
219
|
+
'<div foo="bar" id="new_id"></div>'
|
|
220
|
+
"""
|
|
221
|
+
cdef _Attributes attributes = _Attributes.create(self.node, self.parser.decode_errors)
|
|
222
|
+
return attributes
|
|
223
|
+
|
|
224
|
+
@property
|
|
225
|
+
def mem_id(self):
|
|
226
|
+
"""Get the mem_id attribute of the node.
|
|
227
|
+
|
|
228
|
+
Returns
|
|
229
|
+
-------
|
|
230
|
+
text : int
|
|
231
|
+
"""
|
|
232
|
+
return <size_t> self.node
|
|
233
|
+
|
|
234
|
+
@property
|
|
235
|
+
def id(self):
|
|
236
|
+
"""Get the id attribute of the node.
|
|
237
|
+
|
|
238
|
+
Returns None if id does not set.
|
|
239
|
+
|
|
240
|
+
Returns
|
|
241
|
+
-------
|
|
242
|
+
text : str
|
|
243
|
+
"""
|
|
244
|
+
cdef char* key = 'id'
|
|
245
|
+
cdef myhtml_tree_attr_t *attr
|
|
246
|
+
attr = myhtml_attribute_by_key(self.node, key, 2)
|
|
247
|
+
return None if attr == NULL else attr.value.data.decode(_ENCODING, self.parser.decode_errors)
|
|
248
|
+
|
|
249
|
+
def __hash__(self):
|
|
250
|
+
return self.mem_id
|
|
251
|
+
|
|
252
|
+
def text(self, bool deep=True, str separator='', bool strip=False):
|
|
253
|
+
"""Returns the text of the node including text of all its child nodes.
|
|
254
|
+
|
|
255
|
+
Parameters
|
|
256
|
+
----------
|
|
257
|
+
strip : bool, default False
|
|
258
|
+
If true, calls ``str.strip()`` on each text part to remove extra white spaces.
|
|
259
|
+
separator : str, default ''
|
|
260
|
+
The separator to use when joining text from different nodes.
|
|
261
|
+
deep : bool, default True
|
|
262
|
+
If True, includes text from all child nodes.
|
|
263
|
+
|
|
264
|
+
Returns
|
|
265
|
+
-------
|
|
266
|
+
text : str
|
|
267
|
+
|
|
268
|
+
"""
|
|
269
|
+
text = ""
|
|
270
|
+
cdef const char* c_text
|
|
271
|
+
cdef myhtml_tree_node_t *node = self.node.child
|
|
272
|
+
|
|
273
|
+
if not deep:
|
|
274
|
+
if self.node.tag_id == MyHTML_TAG__TEXT:
|
|
275
|
+
c_text = myhtml_node_text(self.node, NULL)
|
|
276
|
+
if c_text != NULL:
|
|
277
|
+
node_text = c_text.decode(_ENCODING, self.parser.decode_errors)
|
|
278
|
+
text = append_text(text, node_text, separator, strip)
|
|
279
|
+
|
|
280
|
+
while node != NULL:
|
|
281
|
+
if node.tag_id == MyHTML_TAG__TEXT:
|
|
282
|
+
c_text = myhtml_node_text(node, NULL)
|
|
283
|
+
if c_text != NULL:
|
|
284
|
+
node_text = c_text.decode(_ENCODING, self.parser.decode_errors)
|
|
285
|
+
text = append_text(text, node_text, separator, strip)
|
|
286
|
+
node = node.next
|
|
287
|
+
else:
|
|
288
|
+
text = self._text_deep(self.node, separator=separator, strip=strip)
|
|
289
|
+
if separator and text and text.endswith(separator):
|
|
290
|
+
text = text[:-len(separator)]
|
|
291
|
+
return text
|
|
292
|
+
|
|
293
|
+
cdef inline _text_deep(self, myhtml_tree_node_t *node, separator='', strip=False):
|
|
294
|
+
text = ""
|
|
295
|
+
cdef Stack stack = Stack(_STACK_SIZE)
|
|
296
|
+
cdef myhtml_tree_node_t* current_node = NULL
|
|
297
|
+
|
|
298
|
+
if node.tag_id == MyHTML_TAG__TEXT:
|
|
299
|
+
c_text = myhtml_node_text(node, NULL)
|
|
300
|
+
if c_text != NULL:
|
|
301
|
+
node_text = c_text.decode(_ENCODING, self.parser.decode_errors)
|
|
302
|
+
text = append_text(text, node_text, separator, strip)
|
|
303
|
+
|
|
304
|
+
if node.child == NULL:
|
|
305
|
+
return text
|
|
306
|
+
|
|
307
|
+
stack.push(node.child)
|
|
308
|
+
|
|
309
|
+
# Depth-first left-to-right tree traversal
|
|
310
|
+
while not stack.is_empty():
|
|
311
|
+
current_node = stack.pop()
|
|
312
|
+
|
|
313
|
+
if current_node != NULL:
|
|
314
|
+
if current_node.tag_id == MyHTML_TAG__TEXT:
|
|
315
|
+
c_text = myhtml_node_text(current_node, NULL)
|
|
316
|
+
if c_text != NULL:
|
|
317
|
+
node_text = c_text.decode(_ENCODING, self.parser.decode_errors)
|
|
318
|
+
text = append_text(text, node_text, separator, strip)
|
|
319
|
+
|
|
320
|
+
if current_node.next is not NULL:
|
|
321
|
+
stack.push(current_node.next)
|
|
322
|
+
|
|
323
|
+
if current_node.child is not NULL:
|
|
324
|
+
stack.push(current_node.child)
|
|
325
|
+
|
|
326
|
+
return text
|
|
327
|
+
|
|
328
|
+
def iter(self, include_text=False):
|
|
329
|
+
"""Iterate over nodes on the current level.
|
|
330
|
+
|
|
331
|
+
Parameters
|
|
332
|
+
----------
|
|
333
|
+
include_text : bool
|
|
334
|
+
If True, includes text nodes as well.
|
|
335
|
+
|
|
336
|
+
Yields
|
|
337
|
+
-------
|
|
338
|
+
node
|
|
339
|
+
"""
|
|
340
|
+
|
|
341
|
+
cdef myhtml_tree_node_t *node = self.node.child
|
|
342
|
+
cdef Node next_node
|
|
343
|
+
|
|
344
|
+
while node != NULL:
|
|
345
|
+
if node.tag_id == MyHTML_TAG__TEXT and not include_text:
|
|
346
|
+
node = node.next
|
|
347
|
+
continue
|
|
348
|
+
|
|
349
|
+
next_node = Node.new(node, self.parser)
|
|
350
|
+
yield next_node
|
|
351
|
+
node = node.next
|
|
352
|
+
|
|
353
|
+
def traverse(self, include_text=False):
|
|
354
|
+
"""Iterate over all child and next nodes starting from the current level.
|
|
355
|
+
|
|
356
|
+
Parameters
|
|
357
|
+
----------
|
|
358
|
+
include_text : bool
|
|
359
|
+
If True, includes text nodes as well.
|
|
360
|
+
|
|
361
|
+
Yields
|
|
362
|
+
-------
|
|
363
|
+
node
|
|
364
|
+
"""
|
|
365
|
+
cdef Stack stack = Stack(_STACK_SIZE)
|
|
366
|
+
cdef myhtml_tree_node_t* current_node = NULL
|
|
367
|
+
cdef Node next_node
|
|
368
|
+
|
|
369
|
+
stack.push(self.node)
|
|
370
|
+
|
|
371
|
+
while not stack.is_empty():
|
|
372
|
+
current_node = stack.pop()
|
|
373
|
+
if current_node != NULL and not (current_node.tag_id == MyHTML_TAG__TEXT and not include_text):
|
|
374
|
+
next_node = Node.new(current_node, self.parser)
|
|
375
|
+
yield next_node
|
|
376
|
+
|
|
377
|
+
if current_node.next is not NULL:
|
|
378
|
+
stack.push(current_node.next)
|
|
379
|
+
|
|
380
|
+
if current_node.child is not NULL:
|
|
381
|
+
stack.push(current_node.child)
|
|
382
|
+
|
|
383
|
+
@property
|
|
384
|
+
def tag(self):
|
|
385
|
+
"""Return the name of the current tag (e.g. div, p, img).
|
|
386
|
+
|
|
387
|
+
Returns
|
|
388
|
+
-------
|
|
389
|
+
text : str
|
|
390
|
+
"""
|
|
391
|
+
cdef const char *c_text
|
|
392
|
+
c_text = myhtml_tag_name_by_id(self.node.tree, self.node.tag_id, NULL)
|
|
393
|
+
text = None
|
|
394
|
+
if c_text:
|
|
395
|
+
text = c_text.decode(_ENCODING, self.parser.decode_errors)
|
|
396
|
+
return text
|
|
397
|
+
|
|
398
|
+
@property
|
|
399
|
+
def child(self):
|
|
400
|
+
"""Alias for the `first_child` property.
|
|
401
|
+
|
|
402
|
+
**Deprecated**. Please use `first_child` instead.
|
|
403
|
+
"""
|
|
404
|
+
cdef Node node
|
|
405
|
+
if self.node.child:
|
|
406
|
+
node = Node.new(self.node.child, self.parser)
|
|
407
|
+
return node
|
|
408
|
+
return None
|
|
409
|
+
|
|
410
|
+
@property
|
|
411
|
+
def parent(self):
|
|
412
|
+
"""Return the parent node."""
|
|
413
|
+
cdef Node node
|
|
414
|
+
if self.node.parent:
|
|
415
|
+
node = Node.new(self.node.parent, self.parser)
|
|
416
|
+
return node
|
|
417
|
+
return None
|
|
418
|
+
|
|
419
|
+
@property
|
|
420
|
+
def next(self):
|
|
421
|
+
"""Return next node."""
|
|
422
|
+
cdef Node node
|
|
423
|
+
if self.node.next:
|
|
424
|
+
node = Node.new(self.node.next, self.parser)
|
|
425
|
+
return node
|
|
426
|
+
return None
|
|
427
|
+
|
|
428
|
+
@property
|
|
429
|
+
def prev(self):
|
|
430
|
+
"""Return previous node."""
|
|
431
|
+
cdef Node node
|
|
432
|
+
if self.node.prev:
|
|
433
|
+
node = Node.new(self.node.prev, self.parser)
|
|
434
|
+
return node
|
|
435
|
+
return None
|
|
436
|
+
|
|
437
|
+
@property
|
|
438
|
+
def last_child(self):
|
|
439
|
+
"""Return last child node."""
|
|
440
|
+
cdef Node node
|
|
441
|
+
if self.node.last_child:
|
|
442
|
+
node = Node.new(self.node.last_child, self.parser)
|
|
443
|
+
return node
|
|
444
|
+
return None
|
|
445
|
+
|
|
446
|
+
@property
|
|
447
|
+
def html(self):
|
|
448
|
+
"""Return HTML representation of the current node including all its child nodes.
|
|
449
|
+
|
|
450
|
+
Returns
|
|
451
|
+
-------
|
|
452
|
+
text : str
|
|
453
|
+
"""
|
|
454
|
+
cdef mycore_string_raw_t c_str
|
|
455
|
+
c_str.data = NULL
|
|
456
|
+
c_str.length = 0
|
|
457
|
+
c_str.size = 0
|
|
458
|
+
|
|
459
|
+
cdef mystatus_t status
|
|
460
|
+
status = myhtml_serialization(self.node, &c_str)
|
|
461
|
+
|
|
462
|
+
if status == 0 and c_str.data:
|
|
463
|
+
html = c_str.data.decode(_ENCODING).replace('<-undef>', '')
|
|
464
|
+
free(c_str.data)
|
|
465
|
+
return html
|
|
466
|
+
|
|
467
|
+
return None
|
|
468
|
+
|
|
469
|
+
def css(self, str query):
|
|
470
|
+
"""Evaluate CSS selector against current node and its child nodes."""
|
|
471
|
+
return find_nodes(self.parser, self.node, query)
|
|
472
|
+
|
|
473
|
+
def any_css_matches(self, tuple selectors):
|
|
474
|
+
"""Returns True if any of CSS selectors matches a node"""
|
|
475
|
+
return find_matches(self.parser, self.node, selectors)
|
|
476
|
+
|
|
477
|
+
def css_matches(self, str selector):
|
|
478
|
+
"""Returns True if CSS selector matches a node."""
|
|
479
|
+
return find_matches(self.parser, self.node, (selector, ))
|
|
480
|
+
|
|
481
|
+
def css_first(self, str query, default=None, bool strict=False):
|
|
482
|
+
"""Evaluate CSS selector against current node and its child nodes."""
|
|
483
|
+
results = self.css(query)
|
|
484
|
+
n_results = len(results)
|
|
485
|
+
|
|
486
|
+
if n_results > 0:
|
|
487
|
+
|
|
488
|
+
if strict and n_results > 1:
|
|
489
|
+
raise ValueError("Expected 1 match, but found %s matches" % n_results)
|
|
490
|
+
|
|
491
|
+
return results[0]
|
|
492
|
+
|
|
493
|
+
return default
|
|
494
|
+
|
|
495
|
+
def decompose(self, bool recursive=True):
|
|
496
|
+
"""Remove a Node from the tree.
|
|
497
|
+
|
|
498
|
+
Parameters
|
|
499
|
+
----------
|
|
500
|
+
recursive : bool, default True
|
|
501
|
+
Whenever to delete all its child nodes
|
|
502
|
+
|
|
503
|
+
Examples
|
|
504
|
+
--------
|
|
505
|
+
|
|
506
|
+
>>> tree = HTMLParser(html)
|
|
507
|
+
>>> for tag in tree.css('script'):
|
|
508
|
+
>>> tag.decompose()
|
|
509
|
+
|
|
510
|
+
"""
|
|
511
|
+
if recursive:
|
|
512
|
+
myhtml_node_delete_recursive(self.node)
|
|
513
|
+
else:
|
|
514
|
+
myhtml_node_delete(self.node)
|
|
515
|
+
|
|
516
|
+
def remove(self, bool recursive=True):
|
|
517
|
+
"""An alias for the decompose method."""
|
|
518
|
+
self.decompose(recursive)
|
|
519
|
+
|
|
520
|
+
def unwrap(self, delete_empty = False):
|
|
521
|
+
"""Replace node with whatever is inside this node.
|
|
522
|
+
|
|
523
|
+
Parameters
|
|
524
|
+
----------
|
|
525
|
+
delete_empty : bool, default False
|
|
526
|
+
Whenever to delete empty tags.
|
|
527
|
+
|
|
528
|
+
Examples
|
|
529
|
+
--------
|
|
530
|
+
|
|
531
|
+
>>> tree = HTMLParser("<div>Hello <i>world</i>!</div>")
|
|
532
|
+
>>> tree.css_first('i').unwrap()
|
|
533
|
+
>>> tree.html
|
|
534
|
+
'<html><head></head><body><div>Hello world!</div></body></html>'
|
|
535
|
+
|
|
536
|
+
Note: by default, empty tags are ignored, set "delete_empty" to "True" to change this.
|
|
537
|
+
"""
|
|
538
|
+
if self.node.child == NULL:
|
|
539
|
+
if delete_empty:
|
|
540
|
+
myhtml_node_delete(self.node)
|
|
541
|
+
return
|
|
542
|
+
cdef myhtml_tree_node_t* next_node
|
|
543
|
+
cdef myhtml_tree_node_t* current_node
|
|
544
|
+
|
|
545
|
+
if self.node.child.next != NULL:
|
|
546
|
+
current_node = self.node.child
|
|
547
|
+
next_node = current_node.next
|
|
548
|
+
|
|
549
|
+
while next_node != NULL:
|
|
550
|
+
next_node = current_node.next
|
|
551
|
+
myhtml_node_insert_before(self.node, current_node)
|
|
552
|
+
current_node = next_node
|
|
553
|
+
else:
|
|
554
|
+
myhtml_node_insert_before(self.node, self.node.child)
|
|
555
|
+
myhtml_node_delete(self.node)
|
|
556
|
+
|
|
557
|
+
def strip_tags(self, list tags, bool recursive = False):
|
|
558
|
+
"""Remove specified tags from the HTML tree.
|
|
559
|
+
|
|
560
|
+
Parameters
|
|
561
|
+
----------
|
|
562
|
+
tags : list
|
|
563
|
+
List of tags to remove.
|
|
564
|
+
recursive : bool, default True
|
|
565
|
+
Whenever to delete all its child nodes
|
|
566
|
+
|
|
567
|
+
Examples
|
|
568
|
+
--------
|
|
569
|
+
|
|
570
|
+
>>> tree = HTMLParser('<html><head></head><body><script></script><div>Hello world!</div></body></html>')
|
|
571
|
+
>>> tags = ['head', 'style', 'script', 'xmp', 'iframe', 'noembed', 'noframes']
|
|
572
|
+
>>> tree.strip_tags(tags)
|
|
573
|
+
>>> tree.html
|
|
574
|
+
'<html><body><div>Hello world!</div></body></html>'
|
|
575
|
+
|
|
576
|
+
"""
|
|
577
|
+
# ensure cython can recast element to a Node so that decompose will be called sooner.
|
|
578
|
+
cdef Node element
|
|
579
|
+
for tag in tags:
|
|
580
|
+
for element in self.css(tag):
|
|
581
|
+
element.decompose(recursive=recursive)
|
|
582
|
+
|
|
583
|
+
def unwrap_tags(self, list tags, delete_empty = False):
|
|
584
|
+
"""Unwraps specified tags from the HTML tree.
|
|
585
|
+
|
|
586
|
+
Works the same as the ``unwrap`` method, but applied to a list of tags.
|
|
587
|
+
|
|
588
|
+
Parameters
|
|
589
|
+
----------
|
|
590
|
+
tags : list
|
|
591
|
+
List of tags to remove.
|
|
592
|
+
delete_empty : bool, default False
|
|
593
|
+
Whenever to delete empty tags.
|
|
594
|
+
|
|
595
|
+
Examples
|
|
596
|
+
--------
|
|
597
|
+
|
|
598
|
+
>>> tree = HTMLParser("<div><a href="">Hello</a> <i>world</i>!</div>")
|
|
599
|
+
>>> tree.body.unwrap_tags(['i','a'])
|
|
600
|
+
>>> tree.body.html
|
|
601
|
+
'<body><div>Hello world!</div></body>'
|
|
602
|
+
|
|
603
|
+
Note: by default, empty tags are ignored, set "delete_empty" to "True" to change this.
|
|
604
|
+
"""
|
|
605
|
+
cdef Node element
|
|
606
|
+
for tag in tags:
|
|
607
|
+
for element in self.css(tag):
|
|
608
|
+
element.unwrap(delete_empty)
|
|
609
|
+
|
|
610
|
+
def replace_with(self, str_or_Node value):
|
|
611
|
+
"""Replace current Node with specified value.
|
|
612
|
+
|
|
613
|
+
Parameters
|
|
614
|
+
----------
|
|
615
|
+
value : str, bytes or Node
|
|
616
|
+
The text or Node instance to replace the Node with.
|
|
617
|
+
When a text string is passed, it's treated as text. All HTML tags will be escaped.
|
|
618
|
+
Convert and pass the ``Node`` object when you want to work with HTML.
|
|
619
|
+
Does not clone the ``Node`` object.
|
|
620
|
+
All future changes to the passed ``Node`` object will also be taken into account.
|
|
621
|
+
|
|
622
|
+
Examples
|
|
623
|
+
--------
|
|
624
|
+
|
|
625
|
+
>>> tree = HTMLParser('<div>Get <img src="" alt="Laptop"></div>')
|
|
626
|
+
>>> img = tree.css_first('img')
|
|
627
|
+
>>> img.replace_with(img.attributes.get('alt', ''))
|
|
628
|
+
>>> tree.body.child.html
|
|
629
|
+
'<div>Get Laptop</div>'
|
|
630
|
+
|
|
631
|
+
>>> html_parser = HTMLParser('<div>Get <span alt="Laptop"><img src="/jpg"> <div></div></span></div>')
|
|
632
|
+
>>> html_parser2 = HTMLParser('<div>Test</div>')
|
|
633
|
+
>>> img_node = html_parser.css_first('img')
|
|
634
|
+
>>> img_node.replace_with(html_parser2.body.child)
|
|
635
|
+
'<div>Get <span alt="Laptop"><div>Test</div> <div></div></span></div>'
|
|
636
|
+
"""
|
|
637
|
+
cdef myhtml_tree_node_t *node
|
|
638
|
+
if isinstance(value, (str, bytes, unicode)):
|
|
639
|
+
bytes_val = to_bytes(value)
|
|
640
|
+
node = myhtml_node_create(self.parser.html_tree, MyHTML_TAG__TEXT, MyHTML_NAMESPACE_HTML)
|
|
641
|
+
myhtml_node_text_set(node, <char*> bytes_val, len(bytes_val), MyENCODING_UTF_8)
|
|
642
|
+
myhtml_node_insert_before(self.node, node)
|
|
643
|
+
myhtml_node_delete(self.node)
|
|
644
|
+
elif isinstance(value, Node):
|
|
645
|
+
node = myhtml_node_clone_deep(self.parser.html_tree, <myhtml_tree_node_t *> value.node)
|
|
646
|
+
myhtml_node_insert_before(self.node, node)
|
|
647
|
+
myhtml_node_delete(self.node)
|
|
648
|
+
else:
|
|
649
|
+
raise TypeError("Expected a string or Node instance, but %s found" % type(value).__name__)
|
|
650
|
+
|
|
651
|
+
def insert_before(self, str_or_Node value):
|
|
652
|
+
"""
|
|
653
|
+
Insert a node before the current Node.
|
|
654
|
+
|
|
655
|
+
Parameters
|
|
656
|
+
----------
|
|
657
|
+
value : str, bytes or Node
|
|
658
|
+
The text or Node instance to insert before the Node.
|
|
659
|
+
When a text string is passed, it's treated as text. All HTML tags will be escaped.
|
|
660
|
+
Convert and pass the ``Node`` object when you want to work with HTML.
|
|
661
|
+
Does not clone the ``Node`` object.
|
|
662
|
+
All future changes to the passed ``Node`` object will also be taken into account.
|
|
663
|
+
|
|
664
|
+
Examples
|
|
665
|
+
--------
|
|
666
|
+
|
|
667
|
+
>>> tree = HTMLParser('<div>Get <img src="" alt="Laptop"></div>')
|
|
668
|
+
>>> img = tree.css_first('img')
|
|
669
|
+
>>> img.insert_before(img.attributes.get('alt', ''))
|
|
670
|
+
>>> tree.body.child.html
|
|
671
|
+
'<div>Get Laptop<img src="" alt="Laptop"></div>'
|
|
672
|
+
|
|
673
|
+
>>> html_parser = HTMLParser('<div>Get <span alt="Laptop"><img src="/jpg"> <div></div></span></div>')
|
|
674
|
+
>>> html_parser2 = HTMLParser('<div>Test</div>')
|
|
675
|
+
>>> img_node = html_parser.css_first('img')
|
|
676
|
+
>>> img_node.insert_before(html_parser2.body.child)
|
|
677
|
+
<div>Get <span alt="Laptop"><div>Test</div><img src="/jpg"> <div></div></span></div>'
|
|
678
|
+
"""
|
|
679
|
+
cdef myhtml_tree_node_t *node
|
|
680
|
+
if isinstance(value, (str, bytes, unicode)):
|
|
681
|
+
bytes_val = to_bytes(value)
|
|
682
|
+
node = myhtml_node_create(self.parser.html_tree, MyHTML_TAG__TEXT, MyHTML_NAMESPACE_HTML)
|
|
683
|
+
myhtml_node_text_set(node, <char*> bytes_val, len(bytes_val), MyENCODING_UTF_8)
|
|
684
|
+
myhtml_node_insert_before(self.node, node)
|
|
685
|
+
elif isinstance(value, Node):
|
|
686
|
+
node = myhtml_node_clone_deep(self.parser.html_tree, <myhtml_tree_node_t *> value.node)
|
|
687
|
+
myhtml_node_insert_before(self.node, node)
|
|
688
|
+
else:
|
|
689
|
+
raise TypeError("Expected a string or Node instance, but %s found" % type(value).__name__)
|
|
690
|
+
|
|
691
|
+
def insert_after(self, str_or_Node value):
|
|
692
|
+
"""
|
|
693
|
+
Insert a node after the current Node.
|
|
694
|
+
|
|
695
|
+
Parameters
|
|
696
|
+
----------
|
|
697
|
+
value : str, bytes or Node
|
|
698
|
+
The text or Node instance to insert after the Node.
|
|
699
|
+
When a text string is passed, it's treated as text. All HTML tags will be escaped.
|
|
700
|
+
Convert and pass the ``Node`` object when you want to work with HTML.
|
|
701
|
+
Does not clone the ``Node`` object.
|
|
702
|
+
All future changes to the passed ``Node`` object will also be taken into account.
|
|
703
|
+
|
|
704
|
+
Examples
|
|
705
|
+
--------
|
|
706
|
+
|
|
707
|
+
>>> tree = HTMLParser('<div>Get <img src="" alt="Laptop"></div>')
|
|
708
|
+
>>> img = tree.css_first('img')
|
|
709
|
+
>>> img.insert_after(img.attributes.get('alt', ''))
|
|
710
|
+
>>> tree.body.child.html
|
|
711
|
+
'<div>Get <img src="" alt="Laptop">Laptop</div>'
|
|
712
|
+
|
|
713
|
+
>>> html_parser = HTMLParser('<div>Get <span alt="Laptop"><img src="/jpg"> <div></div></span></div>')
|
|
714
|
+
>>> html_parser2 = HTMLParser('<div>Test</div>')
|
|
715
|
+
>>> img_node = html_parser.css_first('img')
|
|
716
|
+
>>> img_node.insert_after(html_parser2.body.child)
|
|
717
|
+
<div>Get <span alt="Laptop"><img src="/jpg"><div>Test</div> <div></div></span></div>'
|
|
718
|
+
"""
|
|
719
|
+
cdef myhtml_tree_node_t *node
|
|
720
|
+
if isinstance(value, (str, bytes, unicode)):
|
|
721
|
+
bytes_val = to_bytes(value)
|
|
722
|
+
node = myhtml_node_create(self.parser.html_tree, MyHTML_TAG__TEXT, MyHTML_NAMESPACE_HTML)
|
|
723
|
+
myhtml_node_text_set(node, <char*> bytes_val, len(bytes_val), MyENCODING_UTF_8)
|
|
724
|
+
myhtml_node_insert_after(self.node, node)
|
|
725
|
+
elif isinstance(value, Node):
|
|
726
|
+
node = myhtml_node_clone_deep(self.parser.html_tree, <myhtml_tree_node_t *> value.node)
|
|
727
|
+
myhtml_node_insert_after(self.node, node)
|
|
728
|
+
else:
|
|
729
|
+
raise TypeError("Expected a string or Node instance, but %s found" % type(value).__name__)
|
|
730
|
+
|
|
731
|
+
def insert_child(self, str_or_Node value):
|
|
732
|
+
"""
|
|
733
|
+
Insert a node inside (at the end of) the current Node.
|
|
734
|
+
|
|
735
|
+
Parameters
|
|
736
|
+
----------
|
|
737
|
+
value : str, bytes or Node
|
|
738
|
+
The text or Node instance to insert inside the Node.
|
|
739
|
+
When a text string is passed, it's treated as text. All HTML tags will be escaped.
|
|
740
|
+
Convert and pass the ``Node`` object when you want to work with HTML.
|
|
741
|
+
Does not clone the ``Node`` object.
|
|
742
|
+
All future changes to the passed ``Node`` object will also be taken into account.
|
|
743
|
+
|
|
744
|
+
Examples
|
|
745
|
+
--------
|
|
746
|
+
|
|
747
|
+
>>> tree = HTMLParser('<div>Get <img src=""></div>')
|
|
748
|
+
>>> div = tree.css_first('div')
|
|
749
|
+
>>> div.insert_child('Laptop')
|
|
750
|
+
>>> tree.body.child.html
|
|
751
|
+
'<div>Get <img src="">Laptop</div>'
|
|
752
|
+
|
|
753
|
+
>>> html_parser = HTMLParser('<div>Get <span alt="Laptop"> <div>Laptop</div> </span></div>')
|
|
754
|
+
>>> html_parser2 = HTMLParser('<div>Test</div>')
|
|
755
|
+
>>> span_node = html_parser.css_first('span')
|
|
756
|
+
>>> span_node.insert_child(html_parser2.body.child)
|
|
757
|
+
<div>Get <span alt="Laptop"> <div>Laptop</div> <div>Test</div> </span></div>'
|
|
758
|
+
"""
|
|
759
|
+
cdef myhtml_tree_node_t *node
|
|
760
|
+
if isinstance(value, (str, bytes, unicode)):
|
|
761
|
+
bytes_val = to_bytes(value)
|
|
762
|
+
node = myhtml_node_create(self.parser.html_tree, MyHTML_TAG__TEXT, MyHTML_NAMESPACE_HTML)
|
|
763
|
+
myhtml_node_text_set(node, <char*> bytes_val, len(bytes_val), MyENCODING_UTF_8)
|
|
764
|
+
myhtml_node_append_child(self.node, node)
|
|
765
|
+
elif isinstance(value, Node):
|
|
766
|
+
node = myhtml_node_clone_deep(self.parser.html_tree, <myhtml_tree_node_t *> value.node)
|
|
767
|
+
myhtml_node_append_child(self.node, node)
|
|
768
|
+
else:
|
|
769
|
+
raise TypeError("Expected a string or Node instance, but %s found" % type(value).__name__)
|
|
770
|
+
|
|
771
|
+
def unwrap_tags(self, list tags, delete_empty = False):
|
|
772
|
+
"""Unwraps specified tags from the HTML tree.
|
|
773
|
+
|
|
774
|
+
Works the same as th ``unwrap`` method, but applied to a list of tags.
|
|
775
|
+
|
|
776
|
+
Parameters
|
|
777
|
+
----------
|
|
778
|
+
tags : list
|
|
779
|
+
List of tags to remove.
|
|
780
|
+
delete_empty : bool, default False
|
|
781
|
+
Whenever to delete empty tags.
|
|
782
|
+
|
|
783
|
+
Examples
|
|
784
|
+
--------
|
|
785
|
+
|
|
786
|
+
>>> tree = HTMLParser("<div><a href="">Hello</a> <i>world</i>!</div>")
|
|
787
|
+
>>> tree.body.unwrap_tags(['i','a'])
|
|
788
|
+
>>> tree.body.html
|
|
789
|
+
'<body><div>Hello world!</div></body>'
|
|
790
|
+
|
|
791
|
+
Note: by default, empty tags are ignored, set "delete_empty" to "True" to change this.
|
|
792
|
+
"""
|
|
793
|
+
cdef Node element
|
|
794
|
+
for tag in tags:
|
|
795
|
+
for element in self.css(tag):
|
|
796
|
+
element.unwrap(delete_empty)
|
|
797
|
+
|
|
798
|
+
@property
|
|
799
|
+
def raw_value(self):
|
|
800
|
+
"""Return the raw (unparsed, original) value of a node.
|
|
801
|
+
|
|
802
|
+
Currently, works on text nodes only.
|
|
803
|
+
|
|
804
|
+
Returns
|
|
805
|
+
-------
|
|
806
|
+
|
|
807
|
+
raw_value : bytes
|
|
808
|
+
|
|
809
|
+
Examples
|
|
810
|
+
--------
|
|
811
|
+
|
|
812
|
+
>>> html_parser = HTMLParser('<div><test></div>')
|
|
813
|
+
>>> selector = html_parser.css_first('div')
|
|
814
|
+
>>> selector.child.html
|
|
815
|
+
'<test>'
|
|
816
|
+
>>> selector.child.raw_value
|
|
817
|
+
b'<test>'
|
|
818
|
+
"""
|
|
819
|
+
cdef int begin = self.node.token.element_begin
|
|
820
|
+
cdef int length = self.node.token.element_length
|
|
821
|
+
if self.node.tag_id != MyHTML_TAG__TEXT:
|
|
822
|
+
raise ValueError("Can't obtain raw value for non-text node.")
|
|
823
|
+
return self.parser.raw_html[begin:begin + length]
|
|
824
|
+
|
|
825
|
+
def select(self, query=None):
|
|
826
|
+
"""Select nodes given a CSS selector.
|
|
827
|
+
|
|
828
|
+
Works similarly to the ``css`` method, but supports chained filtering and extra features.
|
|
829
|
+
|
|
830
|
+
Parameters
|
|
831
|
+
----------
|
|
832
|
+
query : str or None
|
|
833
|
+
The CSS selector to use when searching for nodes.
|
|
834
|
+
|
|
835
|
+
Returns
|
|
836
|
+
-------
|
|
837
|
+
selector : The `Selector` class.
|
|
838
|
+
"""
|
|
839
|
+
return Selector(self, query)
|
|
840
|
+
|
|
841
|
+
def scripts_contain(self, str query):
|
|
842
|
+
"""Returns True if any of the script tags contain specified text.
|
|
843
|
+
|
|
844
|
+
Caches script tags on the first call to improve performance.
|
|
845
|
+
|
|
846
|
+
Parameters
|
|
847
|
+
----------
|
|
848
|
+
query : str
|
|
849
|
+
The query to check.
|
|
850
|
+
|
|
851
|
+
"""
|
|
852
|
+
cdef Node node
|
|
853
|
+
if self.parser.cached_script_texts is None:
|
|
854
|
+
nodes = find_nodes(self.parser, self.node, 'script')
|
|
855
|
+
text_nodes = []
|
|
856
|
+
for node in nodes:
|
|
857
|
+
node_text = node.text(deep=True)
|
|
858
|
+
if node_text:
|
|
859
|
+
text_nodes.append(node_text)
|
|
860
|
+
self.parser.cached_script_texts = text_nodes
|
|
861
|
+
|
|
862
|
+
for text in self.parser.cached_script_texts:
|
|
863
|
+
if query in text:
|
|
864
|
+
return True
|
|
865
|
+
return False
|
|
866
|
+
|
|
867
|
+
def script_srcs_contain(self, tuple queries):
|
|
868
|
+
"""Returns True if any of the script SRCs attributes contain on of the specified text.
|
|
869
|
+
|
|
870
|
+
Caches values on the first call to improve performance.
|
|
871
|
+
|
|
872
|
+
Parameters
|
|
873
|
+
----------
|
|
874
|
+
queries : tuple of str
|
|
875
|
+
|
|
876
|
+
"""
|
|
877
|
+
if self.parser.cached_script_srcs is None:
|
|
878
|
+
nodes = find_nodes(self.parser, self.node, 'script')
|
|
879
|
+
src_nodes = []
|
|
880
|
+
for node in nodes:
|
|
881
|
+
node_src = node.attrs.get('src')
|
|
882
|
+
if node_src:
|
|
883
|
+
src_nodes.append(node_src)
|
|
884
|
+
self.parser.cached_script_srcs = src_nodes
|
|
885
|
+
|
|
886
|
+
for text in self.parser.cached_script_srcs:
|
|
887
|
+
for query in queries:
|
|
888
|
+
if query in text:
|
|
889
|
+
return True
|
|
890
|
+
return False
|
|
891
|
+
|
|
892
|
+
def __repr__(self):
|
|
893
|
+
return '<Node %s>' % self.tag
|
|
894
|
+
|
|
895
|
+
def __eq__(self, other):
|
|
896
|
+
if isinstance(other, str):
|
|
897
|
+
return self.html == other
|
|
898
|
+
if not isinstance(other, Node):
|
|
899
|
+
return False
|
|
900
|
+
return self.html == other.html
|
|
901
|
+
|
|
902
|
+
@property
|
|
903
|
+
def text_content(self):
|
|
904
|
+
"""Returns the text of the node if it is a text node.
|
|
905
|
+
|
|
906
|
+
Returns None for other nodes.
|
|
907
|
+
Unlike the ``text`` method, does not include child nodes.
|
|
908
|
+
|
|
909
|
+
Returns
|
|
910
|
+
-------
|
|
911
|
+
text : str or None.
|
|
912
|
+
"""
|
|
913
|
+
text = ""
|
|
914
|
+
cdef const char* c_text
|
|
915
|
+
cdef myhtml_tree_node_t *node = self.node.child
|
|
916
|
+
|
|
917
|
+
if self.node.tag_id == MyHTML_TAG__TEXT:
|
|
918
|
+
c_text = myhtml_node_text(self.node, NULL)
|
|
919
|
+
if c_text != NULL:
|
|
920
|
+
node_text = c_text.decode(_ENCODING, self.parser.decode_errors)
|
|
921
|
+
return append_text(text, node_text)
|
|
922
|
+
return None
|
|
923
|
+
|
|
924
|
+
def merge_text_nodes(self):
|
|
925
|
+
"""Iterates over all text nodes and merges all text nodes that are close to each other.
|
|
926
|
+
|
|
927
|
+
This is useful for text extraction.
|
|
928
|
+
Use it when you need to strip HTML tags and merge "dangling" text.
|
|
929
|
+
|
|
930
|
+
Examples
|
|
931
|
+
--------
|
|
932
|
+
|
|
933
|
+
>>> tree = HTMLParser("<div><p><strong>J</strong>ohn</p><p>Doe</p></div>")
|
|
934
|
+
>>> node = tree.css_first('div')
|
|
935
|
+
>>> tree.unwrap_tags(["strong"])
|
|
936
|
+
>>> tree.text(deep=True, separator=" ", strip=True)
|
|
937
|
+
"J ohn Doe" # Text extraction produces an extra space because the strong tag was removed.
|
|
938
|
+
>>> node.merge_text_nodes()
|
|
939
|
+
>>> tree.text(deep=True, separator=" ", strip=True)
|
|
940
|
+
"John Doe"
|
|
941
|
+
"""
|
|
942
|
+
cdef Stack stack = Stack(_STACK_SIZE)
|
|
943
|
+
cdef myhtml_tree_node_t * current_node = NULL
|
|
944
|
+
cdef Node next_node
|
|
945
|
+
cdef const char* left_text
|
|
946
|
+
cdef const char* right_text
|
|
947
|
+
cdef char* final_text
|
|
948
|
+
cdef size_t left_length, right_length, final_length
|
|
949
|
+
|
|
950
|
+
stack.push(self.node)
|
|
951
|
+
|
|
952
|
+
while not stack.is_empty():
|
|
953
|
+
current_node = stack.pop()
|
|
954
|
+
|
|
955
|
+
if (current_node.tag_id == MyHTML_TAG__TEXT and current_node.prev and
|
|
956
|
+
current_node.prev.tag_id == MyHTML_TAG__TEXT):
|
|
957
|
+
left_text = myhtml_node_text(current_node.prev, &left_length)
|
|
958
|
+
right_text = myhtml_node_text(current_node, &right_length)
|
|
959
|
+
if left_text and right_text:
|
|
960
|
+
final_length = left_length + right_length
|
|
961
|
+
final_text = <char *>malloc(final_length + 1)
|
|
962
|
+
if final_text == NULL:
|
|
963
|
+
raise MemoryError("Can't allocate memory for a new node.")
|
|
964
|
+
memcpy(final_text, left_text, left_length)
|
|
965
|
+
memcpy(final_text + left_length, right_text, right_length + 1)
|
|
966
|
+
myhtml_node_text_set(current_node, <const char *>final_text, final_length, MyENCODING_UTF_8)
|
|
967
|
+
myhtml_node_delete(current_node.prev)
|
|
968
|
+
free(final_text)
|
|
969
|
+
|
|
970
|
+
if current_node.next is not NULL:
|
|
971
|
+
stack.push(current_node.next)
|
|
972
|
+
|
|
973
|
+
if current_node.child is not NULL:
|
|
974
|
+
stack.push(current_node.child)
|
|
975
|
+
|
|
976
|
+
cdef inline str append_text(str text, str node_text, str separator='', bint strip=False):
|
|
977
|
+
if strip:
|
|
978
|
+
text += node_text.strip() + separator
|
|
979
|
+
else:
|
|
980
|
+
text += node_text + separator
|
|
981
|
+
|
|
982
|
+
return text
|
|
983
|
+
|
|
984
|
+
|
|
985
|
+
cdef inline bytes to_bytes(str_or_Node value):
|
|
986
|
+
cdef bytes bytes_val
|
|
987
|
+
if isinstance(value, unicode):
|
|
988
|
+
bytes_val = <bytes>value.encode("utf-8")
|
|
989
|
+
elif isinstance(value, bytes):
|
|
990
|
+
bytes_val = <bytes>value
|
|
991
|
+
return bytes_val
|