resion 22.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- resion-22.0.1/LICENSE +21 -0
- resion-22.0.1/PKG-INFO +75 -0
- resion-22.0.1/README.md +52 -0
- resion-22.0.1/pyproject.toml +42 -0
- resion-22.0.1/resion/__init__.py +53 -0
- resion-22.0.1/resion/core.py +424 -0
- resion-22.0.1/resion/default.config +0 -0
- resion-22.0.1/resion/utils.py +69 -0
- resion-22.0.1/resion.egg-info/PKG-INFO +75 -0
- resion-22.0.1/resion.egg-info/SOURCES.txt +12 -0
- resion-22.0.1/resion.egg-info/dependency_links.txt +1 -0
- resion-22.0.1/resion.egg-info/requires.txt +5 -0
- resion-22.0.1/resion.egg-info/top_level.txt +1 -0
- resion-22.0.1/setup.cfg +4 -0
resion-22.0.1/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 resion
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
resion-22.0.1/PKG-INFO
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: resion
|
|
3
|
+
Version: 22.0.1
|
|
4
|
+
Summary: Minimal tree-based HTML parsing with basic selectors.
|
|
5
|
+
Author-email: hositel <hositel@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Classifier: Programming Language :: Python :: 3
|
|
8
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
+
Requires-Python: >=3.8
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
License-File: LICENSE
|
|
18
|
+
Requires-Dist: pafex
|
|
19
|
+
Provides-Extra: dev
|
|
20
|
+
Requires-Dist: pytest; extra == "dev"
|
|
21
|
+
Requires-Dist: pytest-cov; extra == "dev"
|
|
22
|
+
Dynamic: license-file
|
|
23
|
+
|
|
24
|
+
# resion
|
|
25
|
+
|
|
26
|
+
Minimal tree-based HTML parser built on the stdlib `html.parser`: an
|
|
27
|
+
editable `Node` tree, basic selectors (tag, `#id`, `.class`), readable-text
|
|
28
|
+
extraction and minification.
|
|
29
|
+
|
|
30
|
+
## Install
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
pip install resion
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## Usage
|
|
37
|
+
|
|
38
|
+
```python
|
|
39
|
+
import resion as rs
|
|
40
|
+
|
|
41
|
+
doc = rs.parse('<h1 id="t">Title</h1><p class="lead">Hi</p>')
|
|
42
|
+
|
|
43
|
+
doc.find('#t').text_content() # 'Title'
|
|
44
|
+
doc.find_all('p') # [Node p]
|
|
45
|
+
doc.find('.lead').get_attr('class') # 'lead'
|
|
46
|
+
rs.extract_text(doc) # readable text, boilerplate removed
|
|
47
|
+
rs.minify('<div> a </div><!-- x -->')
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
The tree is editable:
|
|
51
|
+
|
|
52
|
+
```python
|
|
53
|
+
from resion import Node
|
|
54
|
+
|
|
55
|
+
doc.find('p').replace_with(Node('span'))
|
|
56
|
+
node = doc.find('#t')
|
|
57
|
+
node.set_attr('id', 'new').unwrap()
|
|
58
|
+
doc.to_html()
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Settings are a `Config` or plain dict merged over defaults; a
|
|
62
|
+
`default.config` next to the package is auto-loaded at import (falling back
|
|
63
|
+
to built-in defaults when absent):
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
from resion import Config, Engine, get_config, load_config
|
|
67
|
+
|
|
68
|
+
engine = Engine({'strict_mode': True})
|
|
69
|
+
engine = Engine(load_config('my.config'))
|
|
70
|
+
get_config().get('strict_mode', False)
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
Bytes input is decoded via BOM sniffing and `<meta charset>` detection;
|
|
74
|
+
oversized input and malformed HTML are rejected in strict mode with
|
|
75
|
+
`PafexError`.
|
resion-22.0.1/README.md
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# resion
|
|
2
|
+
|
|
3
|
+
Minimal tree-based HTML parser built on the stdlib `html.parser`: an
|
|
4
|
+
editable `Node` tree, basic selectors (tag, `#id`, `.class`), readable-text
|
|
5
|
+
extraction and minification.
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
pip install resion
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## Usage
|
|
14
|
+
|
|
15
|
+
```python
|
|
16
|
+
import resion as rs
|
|
17
|
+
|
|
18
|
+
doc = rs.parse('<h1 id="t">Title</h1><p class="lead">Hi</p>')
|
|
19
|
+
|
|
20
|
+
doc.find('#t').text_content() # 'Title'
|
|
21
|
+
doc.find_all('p') # [Node p]
|
|
22
|
+
doc.find('.lead').get_attr('class') # 'lead'
|
|
23
|
+
rs.extract_text(doc) # readable text, boilerplate removed
|
|
24
|
+
rs.minify('<div> a </div><!-- x -->')
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
The tree is editable:
|
|
28
|
+
|
|
29
|
+
```python
|
|
30
|
+
from resion import Node
|
|
31
|
+
|
|
32
|
+
doc.find('p').replace_with(Node('span'))
|
|
33
|
+
node = doc.find('#t')
|
|
34
|
+
node.set_attr('id', 'new').unwrap()
|
|
35
|
+
doc.to_html()
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Settings are a `Config` or plain dict merged over defaults; a
|
|
39
|
+
`default.config` next to the package is auto-loaded at import (falling back
|
|
40
|
+
to built-in defaults when absent):
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
from resion import Config, Engine, get_config, load_config
|
|
44
|
+
|
|
45
|
+
engine = Engine({'strict_mode': True})
|
|
46
|
+
engine = Engine(load_config('my.config'))
|
|
47
|
+
get_config().get('strict_mode', False)
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Bytes input is decoded via BOM sniffing and `<meta charset>` detection;
|
|
51
|
+
oversized input and malformed HTML are rejected in strict mode with
|
|
52
|
+
`PafexError`.
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=69.5.1", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "resion"
|
|
7
|
+
version = "22.0.1"
|
|
8
|
+
description = "Minimal tree-based HTML parsing with basic selectors."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
authors = [
|
|
12
|
+
{name = "hositel", email = "hositel@gmail.com"}
|
|
13
|
+
]
|
|
14
|
+
requires-python = ">=3.8"
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Programming Language :: Python :: 3",
|
|
17
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
18
|
+
"Programming Language :: Python :: 3.8",
|
|
19
|
+
"Programming Language :: Python :: 3.9",
|
|
20
|
+
"Programming Language :: Python :: 3.10",
|
|
21
|
+
"Programming Language :: Python :: 3.11",
|
|
22
|
+
"Programming Language :: Python :: 3.12",
|
|
23
|
+
"Programming Language :: Python :: 3.13",
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
dependencies = [
|
|
27
|
+
"pafex"
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
[project.optional-dependencies]
|
|
31
|
+
dev = [
|
|
32
|
+
"pytest",
|
|
33
|
+
"pytest-cov",
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
[tool.setuptools]
|
|
37
|
+
packages = ["resion"]
|
|
38
|
+
|
|
39
|
+
[tool.setuptools.package-data]
|
|
40
|
+
resion = [
|
|
41
|
+
"*.config"
|
|
42
|
+
]
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""Minimal tree-based HTML parsing built on pafex configuration."""
|
|
2
|
+
|
|
3
|
+
from typing import Any, Dict, List, Union
|
|
4
|
+
|
|
5
|
+
from pafex.config import Config, load_config, save_config
|
|
6
|
+
from pafex.exceptions import ConfigLoadError, PafexError
|
|
7
|
+
|
|
8
|
+
from . import core as _core
|
|
9
|
+
from .core import Engine, Node, ParsedDocument, get_config, get_engine
|
|
10
|
+
|
|
11
|
+
def _as_doc(target: Union[str, bytes, ParsedDocument]) -> ParsedDocument:
|
|
12
|
+
if isinstance(target, ParsedDocument):
|
|
13
|
+
return target
|
|
14
|
+
return _core.get_engine().parse(target)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def parse(html: Union[str, bytes]) -> ParsedDocument:
|
|
18
|
+
"""Parse HTML into a ParsedDocument using the active configuration."""
|
|
19
|
+
return _core.get_engine().parse(html)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def select(target: Union[str, bytes, ParsedDocument],
|
|
23
|
+
selector: str) -> List[Dict[str, Any]]:
|
|
24
|
+
"""Run a basic selector (tag, #id, .class); returns dicts per match."""
|
|
25
|
+
return [node.to_dict() for node in _as_doc(target).find_all(selector)]
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def extract_text(target: Union[str, bytes, ParsedDocument]) -> str:
|
|
29
|
+
"""Extract readable text with boilerplate removal."""
|
|
30
|
+
return _as_doc(target).text()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def minify(html: str) -> str:
|
|
34
|
+
"""Minify HTML honoring remove_comments from the active configuration."""
|
|
35
|
+
return _core.get_engine().minify(html)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
__all__ = [
|
|
39
|
+
'Engine',
|
|
40
|
+
'Node',
|
|
41
|
+
'ParsedDocument',
|
|
42
|
+
'Config',
|
|
43
|
+
'load_config',
|
|
44
|
+
'save_config',
|
|
45
|
+
'get_config',
|
|
46
|
+
'get_engine',
|
|
47
|
+
'parse',
|
|
48
|
+
'select',
|
|
49
|
+
'extract_text',
|
|
50
|
+
'minify',
|
|
51
|
+
'PafexError',
|
|
52
|
+
'ConfigLoadError',
|
|
53
|
+
]
|
|
@@ -0,0 +1,424 @@
|
|
|
1
|
+
"""Minimal tree-based HTML parsing engine built on the stdlib html.parser."""
|
|
2
|
+
|
|
3
|
+
import codecs
|
|
4
|
+
import re
|
|
5
|
+
from html import escape as _escape
|
|
6
|
+
from html.parser import HTMLParser as _BaseHTMLParser
|
|
7
|
+
from typing import Any, Dict, Iterator, List, Optional, Union
|
|
8
|
+
|
|
9
|
+
from pafex.config import Config, load_config as _load_config
|
|
10
|
+
from pafex.exceptions import PafexError
|
|
11
|
+
|
|
12
|
+
from .utils import extract_doctype, minify_html
|
|
13
|
+
|
|
14
|
+
VOID_ELEMENTS = {
|
|
15
|
+
'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input',
|
|
16
|
+
'link', 'meta', 'param', 'source', 'track', 'wbr',
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
BLOCK_TAGS = {
|
|
20
|
+
'address', 'article', 'aside', 'blockquote', 'details', 'dialog', 'dd',
|
|
21
|
+
'div', 'dl', 'dt', 'fieldset', 'figcaption', 'figure', 'footer', 'form',
|
|
22
|
+
'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'header', 'hgroup', 'hr', 'li',
|
|
23
|
+
'main', 'nav', 'ol', 'p', 'pre', 'section', 'table', 'tbody', 'thead',
|
|
24
|
+
'tfoot', 'tr', 'ul',
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
ROOT_TAG = ':root'
|
|
28
|
+
COMMENT_TAG = ':comment'
|
|
29
|
+
|
|
30
|
+
PRESERVE_WHITESPACE_TAGS = {'pre', 'textarea'}
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _resolve_config(config: Optional[Any] = None) -> Config:
|
|
34
|
+
"""Normalize None, a plain dict or a Config into a Config instance."""
|
|
35
|
+
if config is None:
|
|
36
|
+
return Config()
|
|
37
|
+
if isinstance(config, Config):
|
|
38
|
+
return config
|
|
39
|
+
if isinstance(config, dict):
|
|
40
|
+
return Config(config)
|
|
41
|
+
return Config(config.to_dict())
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def detect_encoding(data: bytes, default: str = 'utf-8') -> str:
|
|
45
|
+
"""Detect the encoding of raw HTML: BOM sniff, then <meta charset>."""
|
|
46
|
+
if data.startswith(codecs.BOM_UTF8):
|
|
47
|
+
return 'utf-8-sig'
|
|
48
|
+
if data.startswith((codecs.BOM_UTF32_LE, codecs.BOM_UTF32_BE)):
|
|
49
|
+
return 'utf-32'
|
|
50
|
+
if data.startswith((codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)):
|
|
51
|
+
return 'utf-16'
|
|
52
|
+
head = data[:2048].decode('latin-1', errors='ignore')
|
|
53
|
+
match = re.search(
|
|
54
|
+
r'<meta[^>]+charset\s*=\s*["\']?\s*([\w.:+-]+)', head, re.IGNORECASE
|
|
55
|
+
)
|
|
56
|
+
if match:
|
|
57
|
+
try:
|
|
58
|
+
codecs.lookup(match.group(1))
|
|
59
|
+
return match.group(1)
|
|
60
|
+
except LookupError:
|
|
61
|
+
pass
|
|
62
|
+
return default
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class Node:
|
|
66
|
+
"""A single element or comment node in the parsed HTML tree."""
|
|
67
|
+
|
|
68
|
+
def __init__(self, tag: str, attrs: Optional[Dict[str, str]] = None,
|
|
69
|
+
parent: Optional['Node'] = None) -> None:
|
|
70
|
+
self.tag = tag
|
|
71
|
+
self.attrs: Dict[str, str] = dict(attrs or {})
|
|
72
|
+
self.children: List[Any] = []
|
|
73
|
+
self.parent: Optional[Node] = parent
|
|
74
|
+
|
|
75
|
+
@property
|
|
76
|
+
def is_comment(self) -> bool:
|
|
77
|
+
return self.tag == COMMENT_TAG
|
|
78
|
+
|
|
79
|
+
def get_attr(self, name: str, default: Optional[str] = None) -> Optional[str]:
|
|
80
|
+
return self.attrs.get(name, default)
|
|
81
|
+
|
|
82
|
+
def has_class(self, name: str) -> bool:
|
|
83
|
+
return name in (self.get_attr('class') or '').split()
|
|
84
|
+
|
|
85
|
+
def set_attr(self, name: str, value: str) -> 'Node':
|
|
86
|
+
self.attrs[name] = value
|
|
87
|
+
return self
|
|
88
|
+
|
|
89
|
+
def remove_attr(self, name: str) -> 'Node':
|
|
90
|
+
self.attrs.pop(name, None)
|
|
91
|
+
return self
|
|
92
|
+
|
|
93
|
+
def append_child(self, node: 'Node') -> 'Node':
|
|
94
|
+
if node.parent is not None:
|
|
95
|
+
node.parent.children.remove(node)
|
|
96
|
+
self.children.append(node)
|
|
97
|
+
node.parent = self
|
|
98
|
+
return self
|
|
99
|
+
|
|
100
|
+
def remove(self) -> 'Node':
|
|
101
|
+
if self.parent is not None:
|
|
102
|
+
self.parent.children.remove(self)
|
|
103
|
+
self.parent = None
|
|
104
|
+
return self
|
|
105
|
+
|
|
106
|
+
def replace_with(self, *nodes: 'Node') -> 'Node':
|
|
107
|
+
if self.parent is None:
|
|
108
|
+
raise PafexError('cannot replace a detached node')
|
|
109
|
+
siblings = self.parent.children
|
|
110
|
+
index = siblings.index(self)
|
|
111
|
+
replacements = list(nodes)
|
|
112
|
+
for node in replacements:
|
|
113
|
+
if node.parent is not None:
|
|
114
|
+
node.parent.children.remove(node)
|
|
115
|
+
siblings[index:index + 1] = replacements
|
|
116
|
+
for node in replacements:
|
|
117
|
+
node.parent = self.parent
|
|
118
|
+
self.parent = None
|
|
119
|
+
return self
|
|
120
|
+
|
|
121
|
+
def unwrap(self) -> 'Node':
|
|
122
|
+
if self.parent is None:
|
|
123
|
+
raise PafexError('cannot unwrap a detached node')
|
|
124
|
+
siblings = self.parent.children
|
|
125
|
+
index = siblings.index(self)
|
|
126
|
+
children = list(self.children)
|
|
127
|
+
siblings[index:index + 1] = children
|
|
128
|
+
for child in children:
|
|
129
|
+
if isinstance(child, Node):
|
|
130
|
+
child.parent = self.parent
|
|
131
|
+
self.children = []
|
|
132
|
+
self.parent = None
|
|
133
|
+
return self
|
|
134
|
+
|
|
135
|
+
def child_elements(self) -> List['Node']:
|
|
136
|
+
return [
|
|
137
|
+
child
|
|
138
|
+
for child in self.children
|
|
139
|
+
if isinstance(child, Node) and not child.is_comment
|
|
140
|
+
]
|
|
141
|
+
|
|
142
|
+
def text_content(self) -> str:
|
|
143
|
+
parts: List[str] = []
|
|
144
|
+
for child in self.children:
|
|
145
|
+
if isinstance(child, str):
|
|
146
|
+
parts.append(child)
|
|
147
|
+
else:
|
|
148
|
+
parts.append(child.text_content())
|
|
149
|
+
return ''.join(parts)
|
|
150
|
+
|
|
151
|
+
def iter_elements(self) -> Iterator['Node']:
|
|
152
|
+
for child in self.children:
|
|
153
|
+
if isinstance(child, Node) and not child.is_comment:
|
|
154
|
+
yield child
|
|
155
|
+
yield from child.iter_elements()
|
|
156
|
+
|
|
157
|
+
def to_html(self) -> str:
|
|
158
|
+
if self.is_comment:
|
|
159
|
+
return f'<!--{self.text_content()}-->'
|
|
160
|
+
attrs = ''.join(
|
|
161
|
+
f' {name}' if value == '' else f' {name}="{_escape(value, quote=True)}"'
|
|
162
|
+
for name, value in self.attrs.items()
|
|
163
|
+
)
|
|
164
|
+
if self.tag in VOID_ELEMENTS:
|
|
165
|
+
return f'<{self.tag}{attrs}>'
|
|
166
|
+
inner = ''.join(
|
|
167
|
+
_escape(child, quote=False) if isinstance(child, str) else child.to_html()
|
|
168
|
+
for child in self.children
|
|
169
|
+
)
|
|
170
|
+
return f'<{self.tag}{attrs}>{inner}</{self.tag}>'
|
|
171
|
+
|
|
172
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
173
|
+
return {
|
|
174
|
+
'tag': self.tag,
|
|
175
|
+
'attrs': dict(self.attrs),
|
|
176
|
+
'text': self.text_content().strip(),
|
|
177
|
+
'html': self.to_html(),
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
def __repr__(self) -> str:
|
|
181
|
+
return f'<Node {self.tag}>'
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
class _TreeBuilder(_BaseHTMLParser):
|
|
185
|
+
"""Builds a Node tree from HTML using the stdlib tokenizer."""
|
|
186
|
+
|
|
187
|
+
def __init__(self, settings: Any) -> None:
|
|
188
|
+
super().__init__(convert_charrefs=True)
|
|
189
|
+
self.settings = settings
|
|
190
|
+
self.root = Node(ROOT_TAG)
|
|
191
|
+
self._stack: List[Node] = [self.root]
|
|
192
|
+
self.comments: List[str] = []
|
|
193
|
+
self.errors: List[str] = []
|
|
194
|
+
self._max_depth = int(settings.get('max_depth', 100) or 100)
|
|
195
|
+
self._preserve_comments = bool(settings.get('preserve_comments', False))
|
|
196
|
+
self._strip_whitespace = bool(settings.get('strip_whitespace', True))
|
|
197
|
+
self._strict = bool(settings.get('strict_mode', False))
|
|
198
|
+
|
|
199
|
+
def _current(self) -> Node:
|
|
200
|
+
return self._stack[-1]
|
|
201
|
+
|
|
202
|
+
def _attach(self, node: Node) -> None:
|
|
203
|
+
parent = self._current()
|
|
204
|
+
if len(self._stack) <= self._max_depth:
|
|
205
|
+
parent.children.append(node)
|
|
206
|
+
node.parent = parent
|
|
207
|
+
elif self._strict:
|
|
208
|
+
self.errors.append(
|
|
209
|
+
f'<{node.tag}> exceeds max_depth={self._max_depth}'
|
|
210
|
+
)
|
|
211
|
+
self._stack.append(node)
|
|
212
|
+
|
|
213
|
+
def handle_starttag(self, tag, attrs) -> None:
|
|
214
|
+
node = Node(tag, {k: (v if v is not None else '') for k, v in attrs})
|
|
215
|
+
self._attach(node)
|
|
216
|
+
if tag in VOID_ELEMENTS:
|
|
217
|
+
self._stack.pop()
|
|
218
|
+
|
|
219
|
+
def handle_startendtag(self, tag, attrs) -> None:
|
|
220
|
+
node = Node(tag, {k: (v if v is not None else '') for k, v in attrs})
|
|
221
|
+
if len(self._stack) <= self._max_depth:
|
|
222
|
+
parent = self._current()
|
|
223
|
+
parent.children.append(node)
|
|
224
|
+
node.parent = parent
|
|
225
|
+
|
|
226
|
+
def handle_endtag(self, tag) -> None:
|
|
227
|
+
if tag in VOID_ELEMENTS:
|
|
228
|
+
return
|
|
229
|
+
for i in range(len(self._stack) - 1, 0, -1):
|
|
230
|
+
if self._stack[i].tag == tag:
|
|
231
|
+
if len(self._stack) - 1 > i:
|
|
232
|
+
unclosed = [n.tag for n in self._stack[i + 1:]]
|
|
233
|
+
self.errors.append(
|
|
234
|
+
f'unclosed tag(s) <{"> <".join(unclosed)}> before </{tag}>'
|
|
235
|
+
)
|
|
236
|
+
del self._stack[i:]
|
|
237
|
+
return
|
|
238
|
+
self.errors.append(f'closing tag </{tag}> has no matching open tag')
|
|
239
|
+
|
|
240
|
+
def handle_data(self, data) -> None:
|
|
241
|
+
if not data:
|
|
242
|
+
return
|
|
243
|
+
if self._strip_whitespace and not any(
|
|
244
|
+
node.tag in PRESERVE_WHITESPACE_TAGS for node in self._stack
|
|
245
|
+
):
|
|
246
|
+
data = re.sub(r'\s+', ' ', data)
|
|
247
|
+
self._current().children.append(data)
|
|
248
|
+
|
|
249
|
+
def handle_comment(self, data) -> None:
|
|
250
|
+
self.comments.append(data)
|
|
251
|
+
if self._preserve_comments:
|
|
252
|
+
comment = Node(COMMENT_TAG)
|
|
253
|
+
comment.children.append(data)
|
|
254
|
+
if len(self._stack) <= self._max_depth:
|
|
255
|
+
parent = self._current()
|
|
256
|
+
parent.children.append(comment)
|
|
257
|
+
comment.parent = parent
|
|
258
|
+
|
|
259
|
+
def close(self) -> None:
|
|
260
|
+
super().close()
|
|
261
|
+
if len(self._stack) > 1:
|
|
262
|
+
unclosed = [n.tag for n in self._stack[1:]]
|
|
263
|
+
self.errors.append(
|
|
264
|
+
f'unclosed tag(s) at end of input: <{"> <".join(unclosed)}>'
|
|
265
|
+
)
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def _match_basic(node: Node, selector: str) -> bool:
|
|
269
|
+
if selector.startswith('#'):
|
|
270
|
+
return node.get_attr('id') == selector[1:]
|
|
271
|
+
if selector.startswith('.'):
|
|
272
|
+
return node.has_class(selector[1:])
|
|
273
|
+
return node.tag.lower() == selector.lower()
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _find_all_basic(root: Node, selector: str) -> List[Node]:
|
|
277
|
+
selector = selector.strip()
|
|
278
|
+
if not selector or any(char.isspace() for char in selector):
|
|
279
|
+
raise PafexError(
|
|
280
|
+
f'unsupported selector {selector!r}; only tag, #id and .class are supported'
|
|
281
|
+
)
|
|
282
|
+
return [node for node in root.iter_elements() if _match_basic(node, selector)]
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
class ParsedDocument:
|
|
286
|
+
"""Structured, lazily-enriched result of parsing an HTML document."""
|
|
287
|
+
|
|
288
|
+
def __init__(self, doctype: Optional[str], root: Node,
|
|
289
|
+
comments: Optional[List[str]] = None,
|
|
290
|
+
config: Optional[Any] = None,
|
|
291
|
+
errors: Optional[List[str]] = None) -> None:
|
|
292
|
+
self.doctype = doctype
|
|
293
|
+
self.root = root
|
|
294
|
+
self.comments = comments or []
|
|
295
|
+
self.errors = errors or []
|
|
296
|
+
self.config: Config = _resolve_config(config)
|
|
297
|
+
self._text: Optional[str] = None
|
|
298
|
+
|
|
299
|
+
def text(self) -> str:
|
|
300
|
+
from .utils import extract_text
|
|
301
|
+
if self._text is None:
|
|
302
|
+
self._text = extract_text(self)
|
|
303
|
+
return self._text
|
|
304
|
+
|
|
305
|
+
def find_all(self, selector: str) -> List[Node]:
|
|
306
|
+
"""Return every element matching a basic selector (tag, #id, .class)."""
|
|
307
|
+
return _find_all_basic(self.root, selector)
|
|
308
|
+
|
|
309
|
+
def find(self, selector: str) -> Optional[Node]:
|
|
310
|
+
"""Return the first element matching a basic selector, or None."""
|
|
311
|
+
nodes = _find_all_basic(self.root, selector)
|
|
312
|
+
return nodes[0] if nodes else None
|
|
313
|
+
|
|
314
|
+
def to_html(self) -> str:
|
|
315
|
+
return ''.join(
|
|
316
|
+
_escape(child, quote=False) if isinstance(child, str) else child.to_html()
|
|
317
|
+
for child in self.root.children
|
|
318
|
+
)
|
|
319
|
+
|
|
320
|
+
def stats(self) -> Dict[str, int]:
|
|
321
|
+
element_count = 0
|
|
322
|
+
max_depth = 0
|
|
323
|
+
|
|
324
|
+
def walk(node: Node, depth: int) -> None:
|
|
325
|
+
nonlocal element_count, max_depth
|
|
326
|
+
for child in node.children:
|
|
327
|
+
if isinstance(child, Node) and not child.is_comment:
|
|
328
|
+
element_count += 1
|
|
329
|
+
if depth + 1 > max_depth:
|
|
330
|
+
max_depth = depth + 1
|
|
331
|
+
walk(child, depth + 1)
|
|
332
|
+
|
|
333
|
+
walk(self.root, 0)
|
|
334
|
+
return {
|
|
335
|
+
'elements': element_count,
|
|
336
|
+
'comments': len(self.comments),
|
|
337
|
+
'max_depth': max_depth,
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
341
|
+
return {
|
|
342
|
+
'doctype': self.doctype,
|
|
343
|
+
'stats': self.stats(),
|
|
344
|
+
'text': self.text(),
|
|
345
|
+
'comments': list(self.comments),
|
|
346
|
+
'errors': list(self.errors),
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
def __repr__(self) -> str:
|
|
350
|
+
return f'<ParsedDocument {self.stats()["elements"]} elements>'
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
class Engine:
|
|
354
|
+
"""High-level HTML processing engine configured by a Config or dict."""
|
|
355
|
+
|
|
356
|
+
def __init__(self, config: Optional[Any] = None) -> None:
|
|
357
|
+
self.config: Config = _resolve_config(config)
|
|
358
|
+
|
|
359
|
+
def parse(self, html: Union[str, bytes]) -> ParsedDocument:
|
|
360
|
+
"""Parse HTML (str or bytes) into a ParsedDocument."""
|
|
361
|
+
if isinstance(html, bytes):
|
|
362
|
+
encoding = detect_encoding(
|
|
363
|
+
html,
|
|
364
|
+
default=str(self.config.get('encoding', 'utf-8') or 'utf-8'),
|
|
365
|
+
)
|
|
366
|
+
html = html.decode(encoding, errors='replace')
|
|
367
|
+
self._check_size(html)
|
|
368
|
+
builder = _TreeBuilder(self.config)
|
|
369
|
+
try:
|
|
370
|
+
builder.feed(html)
|
|
371
|
+
builder.close()
|
|
372
|
+
except Exception as exc:
|
|
373
|
+
raise PafexError(f'failed to parse HTML: {exc}') from exc
|
|
374
|
+
if bool(self.config.get('strict_mode', False)) and builder.errors:
|
|
375
|
+
summary = '; '.join(builder.errors[:5])
|
|
376
|
+
raise PafexError(f'malformed HTML (strict mode): {summary}')
|
|
377
|
+
return ParsedDocument(
|
|
378
|
+
doctype=extract_doctype(html),
|
|
379
|
+
root=builder.root,
|
|
380
|
+
comments=builder.comments,
|
|
381
|
+
config=self.config,
|
|
382
|
+
errors=builder.errors,
|
|
383
|
+
)
|
|
384
|
+
|
|
385
|
+
def _check_size(self, html: str) -> None:
|
|
386
|
+
cap = self.config.get('max_file_size_mb', 10)
|
|
387
|
+
if not cap:
|
|
388
|
+
return
|
|
389
|
+
size_mb = len(html.encode('utf-8', errors='replace')) / (1024 * 1024)
|
|
390
|
+
if size_mb > float(cap):
|
|
391
|
+
raise PafexError(
|
|
392
|
+
f'input of {size_mb:.2f} MB exceeds max_file_size_mb={cap}'
|
|
393
|
+
)
|
|
394
|
+
|
|
395
|
+
def minify(self, html: str) -> str:
|
|
396
|
+
"""Minify HTML honoring remove_comments from the settings."""
|
|
397
|
+
return minify_html(
|
|
398
|
+
html, remove_comments=bool(self.config.get('remove_comments', True))
|
|
399
|
+
)
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def parse_raw(html: Union[str, bytes],
|
|
403
|
+
config: Optional[Any] = None) -> ParsedDocument:
|
|
404
|
+
"""Parse with whitespace preserved."""
|
|
405
|
+
cfg = _resolve_config(config).update({'strip_whitespace': False})
|
|
406
|
+
return Engine(cfg).parse(html)
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
try:
|
|
410
|
+
config = _load_config(None)
|
|
411
|
+
except Exception:
|
|
412
|
+
config = Config()
|
|
413
|
+
|
|
414
|
+
_engine = Engine(config)
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
def get_config() -> Config:
|
|
418
|
+
"""Return the active configuration object."""
|
|
419
|
+
return config
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
def get_engine() -> Engine:
|
|
423
|
+
"""Return the active parsing engine."""
|
|
424
|
+
return _engine
|
|
Binary file
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""Minimal HTML helpers: doctype extraction, minification and text extraction."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from typing import Optional
|
|
5
|
+
|
|
6
|
+
_SKIP_TEXT_TAGS = {'script', 'style', 'noscript', 'template', 'head'}
|
|
7
|
+
_BOILERPLATE_TAGS = {'nav', 'footer'}
|
|
8
|
+
_BOILERPLATE_NAME_RE = re.compile(
|
|
9
|
+
r'\b(nav|navbar|menu|sidebar|header|footer|banner|advert|ads?|promo|breadcrumb)\b',
|
|
10
|
+
re.IGNORECASE,
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def minify_html(html: str, remove_comments: bool = True) -> str:
|
|
15
|
+
"""Minify HTML by removing unnecessary whitespace and optional comments."""
|
|
16
|
+
if remove_comments:
|
|
17
|
+
html = re.sub(r'<!--(.*?)-->', '', html, flags=re.DOTALL)
|
|
18
|
+
|
|
19
|
+
html = re.sub(r'[ \t]+', ' ', html)
|
|
20
|
+
html = re.sub(r'\s+<(div|p|h[1-6]|br|hr|ul|ol|li|table|tr|td|th|head|body|html)', r'<\1', html)
|
|
21
|
+
html = re.sub(r'</(div|p|h[1-6]|ul|ol|li|table|tr|td|th|head|body|html)>\s+', r'</\1>', html)
|
|
22
|
+
return html.strip()
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def extract_doctype(html: str) -> Optional[str]:
|
|
26
|
+
"""Extract the DOCTYPE declaration from HTML."""
|
|
27
|
+
match = re.search(r'<!DOCTYPE\s+([^>]+)>', html, re.IGNORECASE)
|
|
28
|
+
return match.group(0) if match else None
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _is_boilerplate(node) -> bool:
|
|
32
|
+
if node.tag in _BOILERPLATE_TAGS:
|
|
33
|
+
return True
|
|
34
|
+
identity = f"{node.get_attr('class') or ''} {node.get_attr('id') or ''}"
|
|
35
|
+
return bool(_BOILERPLATE_NAME_RE.search(identity))
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def extract_text(doc) -> str:
|
|
39
|
+
"""Extract readable text, dropping scripts/styles and common boilerplate
|
|
40
|
+
elements while preserving block-level line structure."""
|
|
41
|
+
from .core import BLOCK_TAGS, Node
|
|
42
|
+
|
|
43
|
+
parts = []
|
|
44
|
+
|
|
45
|
+
def walk(node) -> None:
|
|
46
|
+
for child in node.children:
|
|
47
|
+
if isinstance(child, str):
|
|
48
|
+
parts.append(child)
|
|
49
|
+
continue
|
|
50
|
+
if child.is_comment:
|
|
51
|
+
continue
|
|
52
|
+
if child.tag in _SKIP_TEXT_TAGS or _is_boilerplate(child):
|
|
53
|
+
continue
|
|
54
|
+
if child.tag == 'br':
|
|
55
|
+
parts.append('\n')
|
|
56
|
+
continue
|
|
57
|
+
block = child.tag in BLOCK_TAGS
|
|
58
|
+
if block:
|
|
59
|
+
parts.append('\n')
|
|
60
|
+
walk(child)
|
|
61
|
+
if block:
|
|
62
|
+
parts.append('\n')
|
|
63
|
+
|
|
64
|
+
walk(doc.root)
|
|
65
|
+
text = ''.join(parts)
|
|
66
|
+
text = re.sub(r'[ \t]+', ' ', text)
|
|
67
|
+
text = re.sub(r' ?\n ?', '\n', text)
|
|
68
|
+
text = re.sub(r'\n{3,}', '\n\n', text)
|
|
69
|
+
return text.strip()
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: resion
|
|
3
|
+
Version: 22.0.1
|
|
4
|
+
Summary: Minimal tree-based HTML parsing with basic selectors.
|
|
5
|
+
Author-email: hositel <hositel@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Classifier: Programming Language :: Python :: 3
|
|
8
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
+
Requires-Python: >=3.8
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
License-File: LICENSE
|
|
18
|
+
Requires-Dist: pafex
|
|
19
|
+
Provides-Extra: dev
|
|
20
|
+
Requires-Dist: pytest; extra == "dev"
|
|
21
|
+
Requires-Dist: pytest-cov; extra == "dev"
|
|
22
|
+
Dynamic: license-file
|
|
23
|
+
|
|
24
|
+
# resion
|
|
25
|
+
|
|
26
|
+
Minimal tree-based HTML parser built on the stdlib `html.parser`: an
|
|
27
|
+
editable `Node` tree, basic selectors (tag, `#id`, `.class`), readable-text
|
|
28
|
+
extraction and minification.
|
|
29
|
+
|
|
30
|
+
## Install
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
pip install resion
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## Usage
|
|
37
|
+
|
|
38
|
+
```python
|
|
39
|
+
import resion as rs
|
|
40
|
+
|
|
41
|
+
doc = rs.parse('<h1 id="t">Title</h1><p class="lead">Hi</p>')
|
|
42
|
+
|
|
43
|
+
doc.find('#t').text_content() # 'Title'
|
|
44
|
+
doc.find_all('p') # [Node p]
|
|
45
|
+
doc.find('.lead').get_attr('class') # 'lead'
|
|
46
|
+
rs.extract_text(doc) # readable text, boilerplate removed
|
|
47
|
+
rs.minify('<div> a </div><!-- x -->')
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
The tree is editable:
|
|
51
|
+
|
|
52
|
+
```python
|
|
53
|
+
from resion import Node
|
|
54
|
+
|
|
55
|
+
doc.find('p').replace_with(Node('span'))
|
|
56
|
+
node = doc.find('#t')
|
|
57
|
+
node.set_attr('id', 'new').unwrap()
|
|
58
|
+
doc.to_html()
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Settings are a `Config` or plain dict merged over defaults; a
|
|
62
|
+
`default.config` next to the package is auto-loaded at import (falling back
|
|
63
|
+
to built-in defaults when absent):
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
from resion import Config, Engine, get_config, load_config
|
|
67
|
+
|
|
68
|
+
engine = Engine({'strict_mode': True})
|
|
69
|
+
engine = Engine(load_config('my.config'))
|
|
70
|
+
get_config().get('strict_mode', False)
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
Bytes input is decoded via BOM sniffing and `<meta charset>` detection;
|
|
74
|
+
oversized input and malformed HTML are rejected in strict mode with
|
|
75
|
+
`PafexError`.
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
resion/__init__.py
|
|
5
|
+
resion/core.py
|
|
6
|
+
resion/default.config
|
|
7
|
+
resion/utils.py
|
|
8
|
+
resion.egg-info/PKG-INFO
|
|
9
|
+
resion.egg-info/SOURCES.txt
|
|
10
|
+
resion.egg-info/dependency_links.txt
|
|
11
|
+
resion.egg-info/requires.txt
|
|
12
|
+
resion.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
resion
|
resion-22.0.1/setup.cfg
ADDED