webreader 2.3.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- webreader/__init__.py +165 -0
- webreader/__main__.py +6 -0
- webreader/cli.py +166 -0
- webreader/core.py +501 -0
- webreader/default.config +0 -0
- webreader/extractors.py +447 -0
- webreader/formatter.py +89 -0
- webreader/markdown.py +237 -0
- webreader/sanitizer.py +117 -0
- webreader/selectors.py +462 -0
- webreader-2.3.8.dist-info/METADATA +188 -0
- webreader-2.3.8.dist-info/RECORD +15 -0
- webreader-2.3.8.dist-info/WHEEL +5 -0
- webreader-2.3.8.dist-info/licenses/LICENSE +21 -0
- webreader-2.3.8.dist-info/top_level.txt +1 -0
webreader/selectors.py
ADDED
|
@@ -0,0 +1,462 @@
|
|
|
1
|
+
"""CSS selector engine operating on htmlparser document trees.
|
|
2
|
+
|
|
3
|
+
Supported syntax: type selectors, ``*``, ``#id``, ``.class``,
|
|
4
|
+
attribute selectors (``[attr]``, ``[attr=value]`` and the
|
|
5
|
+
``^=``, ``$=``, ``*=``, ``~=``, ``|=`` operators), the
|
|
6
|
+
pseudo-classes ``:first-child``, ``:last-child``, ``:nth-child(an+b)``,
|
|
7
|
+
``:not(compound)`` and ``:contains(text)``, compounds such as
|
|
8
|
+
``div.content#main[href="x"]``, the combinators whitespace
|
|
9
|
+
(descendant), ``>`` (child), ``+`` (adjacent sibling) and ``~``
|
|
10
|
+
(general sibling), and comma-separated selector lists.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
import re
|
|
14
|
+
from typing import List, Optional, Tuple
|
|
15
|
+
|
|
16
|
+
from pafer.exceptions import SelectorSyntaxError
|
|
17
|
+
|
|
18
|
+
from .core import COMMENT_TAG, ROOT_TAG, Node, ParsedDocument
|
|
19
|
+
|
|
20
|
+
_IDENT_RE = re.compile(r'[\w-]+')
|
|
21
|
+
_ATTR_RE = re.compile(r'^([\w:.-]+)\s*(\^=|\$=|\*=|~=|\|=|=)\s*(.*)$', re.DOTALL)
|
|
22
|
+
_NTH_RE = re.compile(r'^([+-]?\d*)n(?:\s*([+-])\s*(\d+))?$')
|
|
23
|
+
_INT_RE = re.compile(r'^[+-]?\d+$')
|
|
24
|
+
|
|
25
|
+
_SUPPORTED_PSEUDOS = {'first-child', 'last-child', 'nth-child', 'not', 'contains'}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class _Compound:
|
|
29
|
+
"""A single compound selector such as ``div.menu#main[href="/"]:first-child``."""
|
|
30
|
+
|
|
31
|
+
__slots__ = ('tag', 'node_id', 'classes', 'attrs', 'pseudos')
|
|
32
|
+
|
|
33
|
+
def __init__(self) -> None:
|
|
34
|
+
self.tag: Optional[str] = None
|
|
35
|
+
self.node_id: Optional[str] = None
|
|
36
|
+
self.classes: List[str] = []
|
|
37
|
+
self.attrs: List[Tuple[str, Optional[str], Optional[str]]] = []
|
|
38
|
+
self.pseudos: List[Tuple[str, object]] = []
|
|
39
|
+
|
|
40
|
+
def matches(self, node: Node) -> bool:
|
|
41
|
+
if self.tag not in (None, '*') and node.tag != self.tag:
|
|
42
|
+
return False
|
|
43
|
+
if self.node_id is not None and node.get_attr('id') != self.node_id:
|
|
44
|
+
return False
|
|
45
|
+
for cls in self.classes:
|
|
46
|
+
if not node.has_class(cls):
|
|
47
|
+
return False
|
|
48
|
+
for name, op, value in self.attrs:
|
|
49
|
+
if not _match_attr(node, name, op, value):
|
|
50
|
+
return False
|
|
51
|
+
for name, arg in self.pseudos:
|
|
52
|
+
if not _match_pseudo(node, name, arg):
|
|
53
|
+
return False
|
|
54
|
+
return True
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _match_attr(node: Node, name: str, op: Optional[str],
|
|
58
|
+
value: Optional[str]) -> bool:
|
|
59
|
+
actual = node.get_attr(name)
|
|
60
|
+
if actual is None:
|
|
61
|
+
return False
|
|
62
|
+
if op is None:
|
|
63
|
+
return True
|
|
64
|
+
if op == '=':
|
|
65
|
+
return actual == value
|
|
66
|
+
if op == '|=':
|
|
67
|
+
return actual == value or actual.startswith(value + '-')
|
|
68
|
+
if not value:
|
|
69
|
+
return False
|
|
70
|
+
if op == '^=':
|
|
71
|
+
return actual.startswith(value)
|
|
72
|
+
if op == '$=':
|
|
73
|
+
return actual.endswith(value)
|
|
74
|
+
if op == '*=':
|
|
75
|
+
return value in actual
|
|
76
|
+
if op == '~=':
|
|
77
|
+
return value in actual.split()
|
|
78
|
+
return False
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _siblings(node: Node) -> List[Node]:
|
|
82
|
+
if node.parent is None:
|
|
83
|
+
return [node]
|
|
84
|
+
return node.parent.child_elements()
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _sibling_index(node: Node) -> int:
|
|
88
|
+
for index, sibling in enumerate(_siblings(node)):
|
|
89
|
+
if sibling is node:
|
|
90
|
+
return index + 1
|
|
91
|
+
return 1
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _sibling_count(node: Node) -> int:
|
|
95
|
+
return len(_siblings(node))
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _match_pseudo(node: Node, name: str, arg: object) -> bool:
|
|
99
|
+
if name == 'first-child':
|
|
100
|
+
return _sibling_index(node) == 1
|
|
101
|
+
if name == 'last-child':
|
|
102
|
+
return _sibling_index(node) == _sibling_count(node)
|
|
103
|
+
if name == 'nth-child':
|
|
104
|
+
step, offset = arg # type: ignore[misc]
|
|
105
|
+
return _nth_matches(step, offset, _sibling_index(node))
|
|
106
|
+
if name == 'not':
|
|
107
|
+
return not arg.matches(node) # type: ignore[attr-defined]
|
|
108
|
+
if name == 'contains':
|
|
109
|
+
return arg in node.text_content() # type: ignore[operator]
|
|
110
|
+
raise SelectorSyntaxError(f'unsupported pseudo-class :{name}')
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _parse_nth(text: str) -> Tuple[int, int]:
|
|
114
|
+
"""Translate an :nth-child argument into an (a, b) an+b step form."""
|
|
115
|
+
value = text.strip().lower().replace(' ', '')
|
|
116
|
+
if value == 'odd':
|
|
117
|
+
return (2, 1)
|
|
118
|
+
if value == 'even':
|
|
119
|
+
return (2, 0)
|
|
120
|
+
if _INT_RE.match(value):
|
|
121
|
+
return (0, int(value))
|
|
122
|
+
match = _NTH_RE.match(value)
|
|
123
|
+
if not match:
|
|
124
|
+
raise SelectorSyntaxError(f'invalid an+b expression {text!r}')
|
|
125
|
+
a_text = match.group(1)
|
|
126
|
+
if a_text in ('', '+'):
|
|
127
|
+
step = 1
|
|
128
|
+
elif a_text == '-':
|
|
129
|
+
step = -1
|
|
130
|
+
else:
|
|
131
|
+
step = int(a_text)
|
|
132
|
+
offset = 0
|
|
133
|
+
if match.group(2):
|
|
134
|
+
offset = int(match.group(3))
|
|
135
|
+
if match.group(2) == '-':
|
|
136
|
+
offset = -offset
|
|
137
|
+
return (step, offset)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _nth_matches(step: int, offset: int, index: int) -> bool:
|
|
141
|
+
delta = index - offset
|
|
142
|
+
if step == 0:
|
|
143
|
+
return delta == 0
|
|
144
|
+
if delta % step != 0:
|
|
145
|
+
return False
|
|
146
|
+
return delta // step >= 0
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _read_parens(text: str, pos: int) -> Tuple[str, int]:
|
|
150
|
+
"""Consume a parenthesized group starting at text[pos] == '('."""
|
|
151
|
+
depth = 0
|
|
152
|
+
quote: Optional[str] = None
|
|
153
|
+
for index in range(pos, len(text)):
|
|
154
|
+
char = text[index]
|
|
155
|
+
if quote:
|
|
156
|
+
if char == quote:
|
|
157
|
+
quote = None
|
|
158
|
+
continue
|
|
159
|
+
if char in '"\'':
|
|
160
|
+
quote = char
|
|
161
|
+
elif char == '(':
|
|
162
|
+
depth += 1
|
|
163
|
+
elif char == ')':
|
|
164
|
+
depth -= 1
|
|
165
|
+
if depth == 0:
|
|
166
|
+
return text[pos + 1:index], index + 1
|
|
167
|
+
raise SelectorSyntaxError(f'unbalanced parentheses in {text!r}')
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _find_attr_end(text: str, pos: int) -> int:
|
|
171
|
+
quote: Optional[str] = None
|
|
172
|
+
for index in range(pos, len(text)):
|
|
173
|
+
char = text[index]
|
|
174
|
+
if quote:
|
|
175
|
+
if char == quote:
|
|
176
|
+
quote = None
|
|
177
|
+
continue
|
|
178
|
+
if char in '"\'':
|
|
179
|
+
quote = char
|
|
180
|
+
elif char == ']':
|
|
181
|
+
return index
|
|
182
|
+
return -1
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _parse_compound(text: str) -> _Compound:
|
|
186
|
+
compound = _Compound()
|
|
187
|
+
pos = 0
|
|
188
|
+
match = _IDENT_RE.match(text, pos)
|
|
189
|
+
if match and match.start() == pos:
|
|
190
|
+
compound.tag = match.group(0).lower()
|
|
191
|
+
pos = match.end()
|
|
192
|
+
matched = compound.tag is not None
|
|
193
|
+
|
|
194
|
+
while pos < len(text):
|
|
195
|
+
char = text[pos]
|
|
196
|
+
if char == '*':
|
|
197
|
+
if pos != 0 or compound.tag is not None:
|
|
198
|
+
raise SelectorSyntaxError(f'misplaced "*" in {text!r}')
|
|
199
|
+
compound.tag = '*'
|
|
200
|
+
pos += 1
|
|
201
|
+
matched = True
|
|
202
|
+
elif char == '#':
|
|
203
|
+
ident = _IDENT_RE.match(text, pos + 1)
|
|
204
|
+
if not ident:
|
|
205
|
+
raise SelectorSyntaxError(f'expected identifier after "#" in {text!r}')
|
|
206
|
+
compound.node_id = ident.group(0)
|
|
207
|
+
pos = ident.end()
|
|
208
|
+
matched = True
|
|
209
|
+
elif char == '.':
|
|
210
|
+
ident = _IDENT_RE.match(text, pos + 1)
|
|
211
|
+
if not ident:
|
|
212
|
+
raise SelectorSyntaxError(f'expected identifier after "." in {text!r}')
|
|
213
|
+
compound.classes.append(ident.group(0))
|
|
214
|
+
pos = ident.end()
|
|
215
|
+
matched = True
|
|
216
|
+
elif char == '[':
|
|
217
|
+
end = _find_attr_end(text, pos + 1)
|
|
218
|
+
if end == -1:
|
|
219
|
+
raise SelectorSyntaxError(f'unclosed attribute selector in {text!r}')
|
|
220
|
+
inner = text[pos + 1:end].strip()
|
|
221
|
+
if not inner:
|
|
222
|
+
raise SelectorSyntaxError(f'empty attribute selector in {text!r}')
|
|
223
|
+
_parse_attr(compound, inner, text)
|
|
224
|
+
pos = end + 1
|
|
225
|
+
matched = True
|
|
226
|
+
elif char == ':':
|
|
227
|
+
pos, matched = _parse_pseudo(compound, text, pos)
|
|
228
|
+
else:
|
|
229
|
+
break
|
|
230
|
+
|
|
231
|
+
if not matched:
|
|
232
|
+
raise SelectorSyntaxError(f'invalid selector component: {text!r}')
|
|
233
|
+
if pos != len(text):
|
|
234
|
+
raise SelectorSyntaxError(
|
|
235
|
+
f'unexpected {text[pos:]!r} in selector {text!r} '
|
|
236
|
+
'(pseudo-classes and functions are not supported)'
|
|
237
|
+
)
|
|
238
|
+
return compound
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _parse_attr(compound: _Compound, inner: str, text: str) -> None:
|
|
242
|
+
match = _ATTR_RE.match(inner)
|
|
243
|
+
if match:
|
|
244
|
+
name = match.group(1).lower()
|
|
245
|
+
op = match.group(2)
|
|
246
|
+
raw_value = match.group(3).strip()
|
|
247
|
+
quoted = (
|
|
248
|
+
len(raw_value) >= 2
|
|
249
|
+
and raw_value[0] == raw_value[-1]
|
|
250
|
+
and raw_value[0] in '"\''
|
|
251
|
+
)
|
|
252
|
+
value = raw_value[1:-1] if quoted else raw_value
|
|
253
|
+
if not quoted and not raw_value:
|
|
254
|
+
raise SelectorSyntaxError(f'missing attribute value in {text!r}')
|
|
255
|
+
compound.attrs.append((name, op, value))
|
|
256
|
+
elif '=' in inner:
|
|
257
|
+
raise SelectorSyntaxError(f'invalid attribute selector [{inner}] in {text!r}')
|
|
258
|
+
else:
|
|
259
|
+
compound.attrs.append((inner.lower(), None, None))
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def _parse_pseudo(compound: _Compound, text: str,
|
|
263
|
+
pos: int) -> Tuple[int, bool]:
|
|
264
|
+
ident = _IDENT_RE.match(text, pos + 1)
|
|
265
|
+
if not ident:
|
|
266
|
+
raise SelectorSyntaxError(
|
|
267
|
+
f'expected pseudo-class name after ":" in {text!r}'
|
|
268
|
+
)
|
|
269
|
+
name = ident.group(0).lower()
|
|
270
|
+
pos = ident.end()
|
|
271
|
+
if name not in _SUPPORTED_PSEUDOS:
|
|
272
|
+
raise SelectorSyntaxError(f'unsupported pseudo-class :{name} in {text!r}')
|
|
273
|
+
arg_text: Optional[str] = None
|
|
274
|
+
if pos < len(text) and text[pos] == '(':
|
|
275
|
+
arg_text, pos = _read_parens(text, pos)
|
|
276
|
+
|
|
277
|
+
if name in ('first-child', 'last-child'):
|
|
278
|
+
if arg_text is not None:
|
|
279
|
+
raise SelectorSyntaxError(f':{name} takes no argument')
|
|
280
|
+
compound.pseudos.append((name, None))
|
|
281
|
+
elif name == 'nth-child':
|
|
282
|
+
if arg_text is None:
|
|
283
|
+
raise SelectorSyntaxError(':nth-child requires an an+b argument')
|
|
284
|
+
compound.pseudos.append((name, _parse_nth(arg_text)))
|
|
285
|
+
elif name == 'not':
|
|
286
|
+
if arg_text is None or not arg_text.strip():
|
|
287
|
+
raise SelectorSyntaxError(':not requires a selector argument')
|
|
288
|
+
compound.pseudos.append((name, _parse_compound(arg_text.strip())))
|
|
289
|
+
else:
|
|
290
|
+
if arg_text is None:
|
|
291
|
+
raise SelectorSyntaxError(':contains requires a text argument')
|
|
292
|
+
value = arg_text.strip()
|
|
293
|
+
if len(value) >= 2 and value[0] == value[-1] and value[0] in '"\'':
|
|
294
|
+
value = value[1:-1]
|
|
295
|
+
if not value:
|
|
296
|
+
raise SelectorSyntaxError(f'empty :contains argument in {text!r}')
|
|
297
|
+
compound.pseudos.append((name, value))
|
|
298
|
+
return pos, True
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def _split_chain(selector: str) -> List[Tuple[Optional[str], _Compound]]:
|
|
302
|
+
"""Split one selector into [(combinator, compound), ...] parts."""
|
|
303
|
+
parts: List[Tuple[Optional[str], str]] = []
|
|
304
|
+
buffer: List[str] = []
|
|
305
|
+
pending: Optional[str] = None
|
|
306
|
+
brackets = 0
|
|
307
|
+
parens = 0
|
|
308
|
+
quote: Optional[str] = None
|
|
309
|
+
seen_space = False
|
|
310
|
+
|
|
311
|
+
for char in selector:
|
|
312
|
+
if quote:
|
|
313
|
+
buffer.append(char)
|
|
314
|
+
if char == quote:
|
|
315
|
+
quote = None
|
|
316
|
+
continue
|
|
317
|
+
if char in '"\'':
|
|
318
|
+
quote = char
|
|
319
|
+
buffer.append(char)
|
|
320
|
+
continue
|
|
321
|
+
if char == '[':
|
|
322
|
+
brackets += 1
|
|
323
|
+
elif char == ']':
|
|
324
|
+
brackets = max(0, brackets - 1)
|
|
325
|
+
elif char == '(':
|
|
326
|
+
parens += 1
|
|
327
|
+
elif char == ')':
|
|
328
|
+
parens = max(0, parens - 1)
|
|
329
|
+
nested = brackets > 0 or parens > 0
|
|
330
|
+
|
|
331
|
+
if not nested and char.isspace():
|
|
332
|
+
if buffer:
|
|
333
|
+
seen_space = True
|
|
334
|
+
continue
|
|
335
|
+
if not nested and char in '>+~':
|
|
336
|
+
if not buffer:
|
|
337
|
+
raise SelectorSyntaxError(f'misplaced {char!r} in selector {selector!r}')
|
|
338
|
+
parts.append((pending, ''.join(buffer)))
|
|
339
|
+
buffer = []
|
|
340
|
+
pending = char
|
|
341
|
+
seen_space = False
|
|
342
|
+
continue
|
|
343
|
+
if seen_space and buffer:
|
|
344
|
+
parts.append((pending, ''.join(buffer)))
|
|
345
|
+
buffer = []
|
|
346
|
+
pending = ' '
|
|
347
|
+
seen_space = False
|
|
348
|
+
buffer.append(char)
|
|
349
|
+
|
|
350
|
+
if not buffer:
|
|
351
|
+
if parts:
|
|
352
|
+
raise SelectorSyntaxError(f'selector {selector!r} ends with a combinator')
|
|
353
|
+
raise SelectorSyntaxError('empty selector')
|
|
354
|
+
parts.append((pending, ''.join(buffer)))
|
|
355
|
+
return [(combinator, _parse_compound(text)) for combinator, text in parts]
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def _prev_element(node: Node) -> Optional[Node]:
|
|
359
|
+
if node.parent is None:
|
|
360
|
+
return None
|
|
361
|
+
siblings = node.parent.child_elements()
|
|
362
|
+
for index, sibling in enumerate(siblings):
|
|
363
|
+
if sibling is node:
|
|
364
|
+
return siblings[index - 1] if index > 0 else None
|
|
365
|
+
return None
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def _matches_chain(node: Node, chain: List[Tuple[Optional[str], _Compound]],
|
|
369
|
+
index: int) -> bool:
|
|
370
|
+
combinator, compound = chain[index]
|
|
371
|
+
if not compound.matches(node):
|
|
372
|
+
return False
|
|
373
|
+
if index == 0:
|
|
374
|
+
return True
|
|
375
|
+
if combinator == '>':
|
|
376
|
+
parent = node.parent
|
|
377
|
+
return parent is not None and _matches_chain(parent, chain, index - 1)
|
|
378
|
+
if combinator == '+':
|
|
379
|
+
prev = _prev_element(node)
|
|
380
|
+
return prev is not None and _matches_chain(prev, chain, index - 1)
|
|
381
|
+
if combinator == '~':
|
|
382
|
+
prev = _prev_element(node)
|
|
383
|
+
while prev is not None:
|
|
384
|
+
if _matches_chain(prev, chain, index - 1):
|
|
385
|
+
return True
|
|
386
|
+
prev = _prev_element(prev)
|
|
387
|
+
return False
|
|
388
|
+
ancestor = node.parent
|
|
389
|
+
while ancestor is not None:
|
|
390
|
+
if _matches_chain(ancestor, chain, index - 1):
|
|
391
|
+
return True
|
|
392
|
+
ancestor = ancestor.parent
|
|
393
|
+
return False
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def _iter_scope(scope: Node):
|
|
397
|
+
if scope.tag not in (ROOT_TAG, COMMENT_TAG):
|
|
398
|
+
yield scope
|
|
399
|
+
yield from scope.iter_elements()
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def _split_selector_list(selector: str) -> List[str]:
|
|
403
|
+
parts: List[str] = []
|
|
404
|
+
buffer: List[str] = []
|
|
405
|
+
brackets = 0
|
|
406
|
+
parens = 0
|
|
407
|
+
quote: Optional[str] = None
|
|
408
|
+
for char in selector:
|
|
409
|
+
if quote:
|
|
410
|
+
buffer.append(char)
|
|
411
|
+
if char == quote:
|
|
412
|
+
quote = None
|
|
413
|
+
continue
|
|
414
|
+
if char in '"\'':
|
|
415
|
+
quote = char
|
|
416
|
+
elif char == '[':
|
|
417
|
+
brackets += 1
|
|
418
|
+
elif char == ']':
|
|
419
|
+
brackets = max(0, brackets - 1)
|
|
420
|
+
elif char == '(':
|
|
421
|
+
parens += 1
|
|
422
|
+
elif char == ')':
|
|
423
|
+
parens = max(0, parens - 1)
|
|
424
|
+
elif char == ',' and not brackets and not parens:
|
|
425
|
+
parts.append(''.join(buffer))
|
|
426
|
+
buffer = []
|
|
427
|
+
continue
|
|
428
|
+
buffer.append(char)
|
|
429
|
+
parts.append(''.join(buffer))
|
|
430
|
+
return parts
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def select_nodes(scope, selector: str) -> List[Node]:
|
|
434
|
+
"""Return all element nodes matching the selector, in document order."""
|
|
435
|
+
root = scope.root if isinstance(scope, ParsedDocument) else scope
|
|
436
|
+
chains = []
|
|
437
|
+
for part in _split_selector_list(selector):
|
|
438
|
+
part = part.strip()
|
|
439
|
+
if not part:
|
|
440
|
+
raise SelectorSyntaxError('empty selector in selector list')
|
|
441
|
+
chains.append(_split_chain(part))
|
|
442
|
+
results: List[Node] = []
|
|
443
|
+
seen = set()
|
|
444
|
+
for node in _iter_scope(root):
|
|
445
|
+
for chain in chains:
|
|
446
|
+
if _matches_chain(node, chain, len(chain) - 1):
|
|
447
|
+
if id(node) not in seen:
|
|
448
|
+
seen.add(id(node))
|
|
449
|
+
results.append(node)
|
|
450
|
+
break
|
|
451
|
+
return results
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
def find_all(scope, selector: str) -> List[Node]:
|
|
455
|
+
"""Return every element matching the selector."""
|
|
456
|
+
return select_nodes(scope, selector)
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def find(scope, selector: str) -> Optional[Node]:
|
|
460
|
+
"""Return the first element matching the selector, or None."""
|
|
461
|
+
nodes = select_nodes(scope, selector)
|
|
462
|
+
return nodes[0] if nodes else None
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: webreader
|
|
3
|
+
Version: 2.3.8
|
|
4
|
+
Summary: Tree-based HTML reader for Python.
|
|
5
|
+
Author-email: morichan <morichan@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Classifier: Programming Language :: Python :: 3
|
|
8
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
+
Classifier: Topic :: Software Development :: User Interfaces
|
|
16
|
+
Requires-Python: >=3.8
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Requires-Dist: pafer
|
|
20
|
+
Provides-Extra: dev
|
|
21
|
+
Requires-Dist: pytest; extra == "dev"
|
|
22
|
+
Requires-Dist: pytest-cov; extra == "dev"
|
|
23
|
+
Dynamic: license-file
|
|
24
|
+
|
|
25
|
+

|
|
26
|
+

|
|
27
|
+

|
|
28
|
+
|
|
29
|
+
## Features
|
|
30
|
+
|
|
31
|
+
- Tree-based parsing into an editable document tree
|
|
32
|
+
- CSS selectors: combinators, attributes, pseudo-classes, selector lists
|
|
33
|
+
- Extractors: metadata, links, tables, text, images, forms, headings,
|
|
34
|
+
scripts, stylesheets
|
|
35
|
+
- Sanitization with a removal report
|
|
36
|
+
- Minify, pretty-print, Markdown conversion
|
|
37
|
+
- JSON / CSV / Markdown export
|
|
38
|
+
- CLI tool
|
|
39
|
+
- Standard library only at runtime
|
|
40
|
+
|
|
41
|
+
## Installation
|
|
42
|
+
|
|
43
|
+
```
|
|
44
|
+
pip install webreader
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
## Quickstart
|
|
48
|
+
|
|
49
|
+
```python
|
|
50
|
+
import webreader
|
|
51
|
+
|
|
52
|
+
html = '<h1 class="title">Hello</h1><a href="/x">link</a>'
|
|
53
|
+
doc = webreader.parse(html)
|
|
54
|
+
|
|
55
|
+
webreader.select(doc, 'h1.title') # [{'tag': ..., 'attrs': ..., 'text': ..., 'html': ...}]
|
|
56
|
+
doc.find('a').get_attr('href') # '/x'
|
|
57
|
+
|
|
58
|
+
webreader.minify(html) # collapsed whitespace
|
|
59
|
+
webreader.pretty(html) # re-indented, readable markup
|
|
60
|
+
webreader.to_markdown(html) # convert to Markdown
|
|
61
|
+
|
|
62
|
+
clean, report = webreader.sanitize(html)
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
Bytes input is decoded using detected encoding: BOM sniffing first,
|
|
66
|
+
then `<meta charset>` in the first 2 KB, then the configured
|
|
67
|
+
`encoding` fallback.
|
|
68
|
+
|
|
69
|
+
In non-strict mode, malformed HTML does not raise; the warnings are
|
|
70
|
+
available on the document:
|
|
71
|
+
|
|
72
|
+
```python
|
|
73
|
+
doc = webreader.parse('<div><p>x</p>') # unclosed <div>
|
|
74
|
+
doc.errors # ['unclosed tag(s) at end of input: <div>']
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
## Extraction
|
|
78
|
+
|
|
79
|
+
```python
|
|
80
|
+
webreader.extract_metadata(html) # title, OG/Twitter, canonical, charset, JSON-LD
|
|
81
|
+
webreader.extract_links(html) # links and asset references
|
|
82
|
+
webreader.extract_tables(html) # matrices with colspan/rowspan expanded
|
|
83
|
+
webreader.extract_text(html) # readable text without boilerplate
|
|
84
|
+
webreader.extract_images(html) # src, alt, dimensions, srcset candidates
|
|
85
|
+
webreader.extract_forms(html) # form fields with types and defaults
|
|
86
|
+
webreader.extract_headings(html) # h1-h6 outline + word count + reading time
|
|
87
|
+
webreader.extract_scripts(html) # inline and external script inventory
|
|
88
|
+
webreader.extract_stylesheets(html) # link[rel~=stylesheet] and inline styles
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
## CSS selectors
|
|
92
|
+
|
|
93
|
+
Supported syntax: type selectors, `*`, `#id`, `.class`, attribute
|
|
94
|
+
selectors (`[attr]`, `[attr=value]` and the `^=`, `$=`, `*=`, `~=`,
|
|
95
|
+
`|=` operators), the pseudo-classes `:first-child`, `:last-child`,
|
|
96
|
+
`:nth-child(an+b)`, `:not(compound)` and `:contains(text)`, compounds
|
|
97
|
+
such as `div.content#main[href="x"]`, the combinators whitespace
|
|
98
|
+
(descendant), `>` (child), `+` (adjacent sibling) and `~` (general
|
|
99
|
+
sibling), and comma-separated selector lists.
|
|
100
|
+
|
|
101
|
+
```python
|
|
102
|
+
webreader.select(doc, 'a[href^="https://"]')
|
|
103
|
+
webreader.select(doc, 'ul > li:nth-child(2n+1):not(.skip)')
|
|
104
|
+
webreader.select(doc, 'h2 + p, blockquote p:contains("note")')
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
Invalid or unsupported selectors raise `SelectorSyntaxError`.
|
|
108
|
+
|
|
109
|
+
## Tree editing
|
|
110
|
+
|
|
111
|
+
Nodes support a mutation API, and the edited tree can be
|
|
112
|
+
re-serialized:
|
|
113
|
+
|
|
114
|
+
```python
|
|
115
|
+
doc = webreader.parse('<div><a href="/x">link</a></div>')
|
|
116
|
+
link = doc.find('a')
|
|
117
|
+
link.set_attr('rel', 'noopener').remove_attr('class')
|
|
118
|
+
|
|
119
|
+
new = webreader.Node('p')
|
|
120
|
+
new.children.append('added text')
|
|
121
|
+
doc.find('div').append_child(new)
|
|
122
|
+
|
|
123
|
+
link.unwrap() # replace the <a> with its inner text
|
|
124
|
+
doc.to_html() # '<div>link<p>added text</p></div>'
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
`Node` also provides `remove()` and `replace_with(*nodes)`.
|
|
128
|
+
|
|
129
|
+
## Markdown conversion
|
|
130
|
+
|
|
131
|
+
`to_markdown` renders headings, paragraphs, links, images,
|
|
132
|
+
emphasis/strong/strikethrough, inline and fenced code (with language
|
|
133
|
+
detection from `class="language-..."`), nested lists, blockquotes,
|
|
134
|
+
horizontal rules and tables.
|
|
135
|
+
|
|
136
|
+
```python
|
|
137
|
+
webreader.to_markdown('<h1>T</h1><p>Hi <b>bold</b></p>')
|
|
138
|
+
# '# T\n\nHi **bold**'
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
## Sanitization
|
|
142
|
+
|
|
143
|
+
`sanitize()` returns `(clean_html, report)`:
|
|
144
|
+
|
|
145
|
+
- tags outside the whitelist are unwrapped, inner content kept
|
|
146
|
+
- `script`, `style`, `noscript`, `template` are dropped entirely
|
|
147
|
+
- attributes filtered through a safe-list, `on*` handlers removed
|
|
148
|
+
- `javascript:`, `vbscript:` and non-image `data:` URLs blocked
|
|
149
|
+
- the report lists removed tags/attributes, blocked URLs and detected threats
|
|
150
|
+
|
|
151
|
+
## CLI
|
|
152
|
+
|
|
153
|
+
```
|
|
154
|
+
python -m webreader parse -f input.html
|
|
155
|
+
python -m webreader select -f input.html -s 'a[href]'
|
|
156
|
+
python -m webreader links -f input.html --base-url https://example.com
|
|
157
|
+
python -m webreader meta -f input.html
|
|
158
|
+
python -m webreader tables -f input.html --format csv
|
|
159
|
+
python -m webreader text -f input.html
|
|
160
|
+
python -m webreader images -f input.html
|
|
161
|
+
python -m webreader forms -f input.html
|
|
162
|
+
python -m webreader minify -f input.html --out min.html
|
|
163
|
+
python -m webreader pretty -f input.html --out pretty.html
|
|
164
|
+
python -m webreader md -f input.html --out page.md
|
|
165
|
+
python -m webreader sanitize -f input.html --out clean.html --allowed p,a,ul
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
Use `-f -` to read from stdin. Every output command accepts `--out`
|
|
169
|
+
to write to a file instead of stdout.
|
|
170
|
+
|
|
171
|
+
### Default settings
|
|
172
|
+
|
|
173
|
+
| Key | Default | Purpose |
|
|
174
|
+
|---|---|---|
|
|
175
|
+
| `strip_whitespace` | `True` | collapse whitespace in text nodes |
|
|
176
|
+
| `encoding` | `utf-8` | fallback when bytes input has no detectable encoding |
|
|
177
|
+
| `max_depth` | `100` | maximum element nesting |
|
|
178
|
+
| `preserve_comments` | `False` | keep comments in the tree |
|
|
179
|
+
| `strict_mode` | `False` | raise on malformed HTML |
|
|
180
|
+
| `remove_comments` | `True` | strip comments when minifying |
|
|
181
|
+
| `allowed_tags` | `p, br, b, i, a, ul, ol, li, div, span` | sanitize whitelist |
|
|
182
|
+
| `max_file_size_mb` | `10` | reject oversized input |
|
|
183
|
+
| `cache_size` | `50` | parse cache capacity |
|
|
184
|
+
|
|
185
|
+
## Exceptions
|
|
186
|
+
|
|
187
|
+
`HTMLPaferError` (base), `ConfigLoadError`, `SanitizationError`,
|
|
188
|
+
`SelectorSyntaxError`.
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
webreader/__init__.py,sha256=B_7GIR-PD0ovIEB4ohEIqMfg0H9BcgOY5d8aAEhkUfc,5939
|
|
2
|
+
webreader/__main__.py,sha256=IddtPdFVJkK7fQzXzTwQR53faIh0ou_fecp66SYwJZ8,118
|
|
3
|
+
webreader/cli.py,sha256=K0SSg5FGX3pRAeFDQT3eOSrvT0DxOIG0dum2xGwuQP0,6257
|
|
4
|
+
webreader/core.py,sha256=BGxJH-rdv7hU7x9tgd2gM_ZJ3wIMLeicLJgE-87eR3s,17135
|
|
5
|
+
webreader/default.config,sha256=u-MHw9NH_3pXs2MH6oD56T2o6H1qc404glYOwrho43I,3912
|
|
6
|
+
webreader/extractors.py,sha256=8SStkNjKBJOFlqZYkiw6d3tUCdvGXSaO_uCR5UZqDL0,14759
|
|
7
|
+
webreader/formatter.py,sha256=53BZwv7ITE76gI1q5fpoKMOZOk_4_VnYmmrZYxDlr2I,2758
|
|
8
|
+
webreader/markdown.py,sha256=rbtCGjh8YK-o5KRoIYAmM0CJbVd3I1YlsVPweqsNjtY,7059
|
|
9
|
+
webreader/sanitizer.py,sha256=DKq8COXQAMs1uTqi3KWojZV6ZXZ13dD3dTrJgtRayfo,4085
|
|
10
|
+
webreader/selectors.py,sha256=xSekLdo86iYo5Fp1YGj7rrtC_hLekMBzA8sReY9wdUw,15240
|
|
11
|
+
webreader-2.3.8.dist-info/licenses/LICENSE,sha256=WoBhv4IiumWxRgU4WdprWpi1RZavGc7_PC-GMwbHgtI,1087
|
|
12
|
+
webreader-2.3.8.dist-info/METADATA,sha256=UpnK17gZysOtnAMc-EEq4k1TWTAMoTfbiVhuXwk6auo,6753
|
|
13
|
+
webreader-2.3.8.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
14
|
+
webreader-2.3.8.dist-info/top_level.txt,sha256=InRfwO2sKc1fq2AmIyxTgfO2jCM5ivfqoDOZRRmejCg,10
|
|
15
|
+
webreader-2.3.8.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 webreader
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
webreader
|