purehtml-purescraps 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- purehtml_purescraps-0.1.1/LICENSE +21 -0
- purehtml_purescraps-0.1.1/PKG-INFO +77 -0
- purehtml_purescraps-0.1.1/README.md +59 -0
- purehtml_purescraps-0.1.1/purehtml/__init__.py +7 -0
- purehtml_purescraps-0.1.1/purehtml/backend/__init__.py +0 -0
- purehtml_purescraps-0.1.1/purehtml/backend/backend.py +89 -0
- purehtml_purescraps-0.1.1/purehtml/config_factory.py +139 -0
- purehtml_purescraps-0.1.1/purehtml/configs/__init__.py +1 -0
- purehtml_purescraps-0.1.1/purehtml/configs/configs.py +15 -0
- purehtml_purescraps-0.1.1/purehtml/configs/extract_params.py +35 -0
- purehtml_purescraps-0.1.1/purehtml/configs/selector_match_params.py +25 -0
- purehtml_purescraps-0.1.1/purehtml/configs/types/__init__.py +0 -0
- purehtml_purescraps-0.1.1/purehtml/configs/types/array_config.py +58 -0
- purehtml_purescraps-0.1.1/purehtml/configs/types/config_with_selector.py +79 -0
- purehtml_purescraps-0.1.1/purehtml/configs/types/constant_config.py +38 -0
- purehtml_purescraps-0.1.1/purehtml/configs/types/object_config.py +62 -0
- purehtml_purescraps-0.1.1/purehtml/configs/types/primitive_value_config.py +63 -0
- purehtml_purescraps-0.1.1/purehtml/configs/types/union_config.py +60 -0
- purehtml_purescraps-0.1.1/purehtml/purehtml.py +24 -0
- purehtml_purescraps-0.1.1/purehtml/purehtml_validator.py +164 -0
- purehtml_purescraps-0.1.1/purehtml/transformers/__init__.py +0 -0
- purehtml_purescraps-0.1.1/purehtml/transformers/transform_params.py +34 -0
- purehtml_purescraps-0.1.1/purehtml/transformers/transformer.py +33 -0
- purehtml_purescraps-0.1.1/purehtml/transformers/transformer_factory.py +62 -0
- purehtml_purescraps-0.1.1/purehtml/transformers/transformers.py +44 -0
- purehtml_purescraps-0.1.1/purehtml/transformers/types/__init__.py +0 -0
- purehtml_purescraps-0.1.1/purehtml/transformers/types/attribute_transformer.py +56 -0
- purehtml_purescraps-0.1.1/purehtml/transformers/types/exists_transformer.py +39 -0
- purehtml_purescraps-0.1.1/purehtml/transformers/types/html_transformer.py +42 -0
- purehtml_purescraps-0.1.1/purehtml/transformers/types/length_transformer.py +52 -0
- purehtml_purescraps-0.1.1/purehtml/transformers/types/number_transformer.py +54 -0
- purehtml_purescraps-0.1.1/purehtml/transformers/types/resolve_transformer.py +50 -0
- purehtml_purescraps-0.1.1/purehtml/transformers/types/trim_transformer.py +48 -0
- purehtml_purescraps-0.1.1/purehtml/transformers/types/url_query_params_transformer.py +76 -0
- purehtml_purescraps-0.1.1/purehtml_purescraps.egg-info/PKG-INFO +77 -0
- purehtml_purescraps-0.1.1/purehtml_purescraps.egg-info/SOURCES.txt +39 -0
- purehtml_purescraps-0.1.1/purehtml_purescraps.egg-info/dependency_links.txt +1 -0
- purehtml_purescraps-0.1.1/purehtml_purescraps.egg-info/requires.txt +4 -0
- purehtml_purescraps-0.1.1/purehtml_purescraps.egg-info/top_level.txt +1 -0
- purehtml_purescraps-0.1.1/pyproject.toml +24 -0
- purehtml_purescraps-0.1.1/setup.cfg +4 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024 Onur C. Güven
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: purehtml-purescraps
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: HTML data extraction library
|
|
5
|
+
Author-email: "Onur C. Güven" <onurcguven09@gmail.com>
|
|
6
|
+
Project-URL: Homepage, https://github.com/purescraps/purehtml
|
|
7
|
+
Project-URL: Issues, https://github.com/purescraps/purehtml/issues
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Requires-Python: >=3.8
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Requires-Dist: PyYAML~=6.0.2
|
|
15
|
+
Requires-Dist: jsonschema~=4.23.0
|
|
16
|
+
Requires-Dist: beautifulsoup4~=4.12.3
|
|
17
|
+
Requires-Dist: html5lib~=1.1
|
|
18
|
+
|
|
19
|
+
# PureHTML
|
|
20
|
+
|
|
21
|
+
PureHTML is a parsing specification for extracting JSON from HTML.
|
|
22
|
+
|
|
23
|
+
- [Documentation](https://purescraps.github.io/purehtml/)
|
|
24
|
+
|
|
25
|
+
**Installation**:
|
|
26
|
+
|
|
27
|
+
To install **PureHTML**, you can use `pip` with following command:
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
pip install purehtml
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
## Usage
|
|
34
|
+
|
|
35
|
+
```python
|
|
36
|
+
from purehtml import ConfigFactory, extract
|
|
37
|
+
|
|
38
|
+
yaml_string = """
|
|
39
|
+
selector: span
|
|
40
|
+
type: array
|
|
41
|
+
items: {}
|
|
42
|
+
"""
|
|
43
|
+
html_string = """
|
|
44
|
+
<div>
|
|
45
|
+
<span>a</span>
|
|
46
|
+
<span>b</span>
|
|
47
|
+
<span>c</span>
|
|
48
|
+
</div>
|
|
49
|
+
"""
|
|
50
|
+
expected_output = "['a', 'b', 'c']"
|
|
51
|
+
|
|
52
|
+
config = ConfigFactory.from_yaml(yaml_string)
|
|
53
|
+
|
|
54
|
+
print(f"Extracted output : {extract(config, html_string, 'http://example.com')}")
|
|
55
|
+
print(f"Expected output : {expected_output}")
|
|
56
|
+
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## Development
|
|
60
|
+
|
|
61
|
+
For testing, we use test specifications defined in `<purehtml root>/specs`.
|
|
62
|
+
You may run `python3 run_tests.py` to check if the python implementation
|
|
63
|
+
is working correctly or has some incompatibilities.
|
|
64
|
+
|
|
65
|
+
#### Building
|
|
66
|
+
|
|
67
|
+
1. Install `build` pip with this command: `python3 -m pip install --upgrade build`
|
|
68
|
+
2. Execute `make clean build` to create a clean build output. The output will be placed in `dist/`
|
|
69
|
+
|
|
70
|
+
#### Publishing to Python Package Index
|
|
71
|
+
|
|
72
|
+
1. Install the `twine` package: `python3 -m pip install --upgrade twine`
|
|
73
|
+
2. Publish to pypi: `python3 -m twine upload dist/*` or with `make publish`
|
|
74
|
+
|
|
75
|
+
## License
|
|
76
|
+
|
|
77
|
+
[MIT](https://choosealicense.com/licenses/mit/)
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# PureHTML
|
|
2
|
+
|
|
3
|
+
PureHTML is a parsing specification for extracting JSON from HTML.
|
|
4
|
+
|
|
5
|
+
- [Documentation](https://purescraps.github.io/purehtml/)
|
|
6
|
+
|
|
7
|
+
**Installation**:
|
|
8
|
+
|
|
9
|
+
To install **PureHTML**, you can use `pip` with following command:
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
pip install purehtml
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
## Usage
|
|
16
|
+
|
|
17
|
+
```python
|
|
18
|
+
from purehtml import ConfigFactory, extract
|
|
19
|
+
|
|
20
|
+
yaml_string = """
|
|
21
|
+
selector: span
|
|
22
|
+
type: array
|
|
23
|
+
items: {}
|
|
24
|
+
"""
|
|
25
|
+
html_string = """
|
|
26
|
+
<div>
|
|
27
|
+
<span>a</span>
|
|
28
|
+
<span>b</span>
|
|
29
|
+
<span>c</span>
|
|
30
|
+
</div>
|
|
31
|
+
"""
|
|
32
|
+
expected_output = "['a', 'b', 'c']"
|
|
33
|
+
|
|
34
|
+
config = ConfigFactory.from_yaml(yaml_string)
|
|
35
|
+
|
|
36
|
+
print(f"Extracted output : {extract(config, html_string, 'http://example.com')}")
|
|
37
|
+
print(f"Expected output : {expected_output}")
|
|
38
|
+
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
## Development
|
|
42
|
+
|
|
43
|
+
For testing, we use test specifications defined in `<purehtml root>/specs`.
|
|
44
|
+
You may run `python3 run_tests.py` to check if the python implementation
|
|
45
|
+
is working correctly or has some incompatibilities.
|
|
46
|
+
|
|
47
|
+
#### Building
|
|
48
|
+
|
|
49
|
+
1. Install `build` pip with this command: `python3 -m pip install --upgrade build`
|
|
50
|
+
2. Execute `make clean build` to create a clean build output. The output will be placed in `dist/`
|
|
51
|
+
|
|
52
|
+
#### Publishing to Python Package Index
|
|
53
|
+
|
|
54
|
+
1. Install the `twine` package: `python3 -m pip install --upgrade twine`
|
|
55
|
+
2. Publish to pypi: `python3 -m twine upload dist/*` or with `make publish`
|
|
56
|
+
|
|
57
|
+
## License
|
|
58
|
+
|
|
59
|
+
[MIT](https://choosealicense.com/licenses/mit/)
|
|
File without changes
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
from bs4 import BeautifulSoup
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class BeautifulSoupBackend:
|
|
5
|
+
"""
|
|
6
|
+
A backend implementation using BeautifulSoup.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
@staticmethod
|
|
10
|
+
def load(html):
|
|
11
|
+
"""
|
|
12
|
+
Load an HTML string or buffer into a PureHTMLBeautifulSoupDocument.
|
|
13
|
+
"""
|
|
14
|
+
return PureHTMLDocument(BeautifulSoup(html, "html5lib"))
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class PureHTMLDocument:
|
|
18
|
+
"""
|
|
19
|
+
A document implementation using BeautifulSoup.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
def __init__(self, soup):
|
|
23
|
+
self._soup = soup
|
|
24
|
+
|
|
25
|
+
def select(self, selector):
|
|
26
|
+
"""
|
|
27
|
+
Select elements based on a CSS selector and wrap them in PureHTMLBeautifulSoupNode objects.
|
|
28
|
+
"""
|
|
29
|
+
elements = self._soup.select(selector)
|
|
30
|
+
return [PureHTMLNode(self._soup, el) for el in elements]
|
|
31
|
+
|
|
32
|
+
def root(self):
|
|
33
|
+
"""
|
|
34
|
+
Return the root element of the document.
|
|
35
|
+
"""
|
|
36
|
+
return PureHTMLNode(self._soup, self._soup)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class PureHTMLNode:
|
|
40
|
+
"""
|
|
41
|
+
A node implementation using BeautifulSoup.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
def __init__(self, soup, element):
|
|
45
|
+
self._soup = soup
|
|
46
|
+
self._element = element
|
|
47
|
+
|
|
48
|
+
def attr(self, name=None):
|
|
49
|
+
"""
|
|
50
|
+
Get an attribute or all attributes of the node.
|
|
51
|
+
"""
|
|
52
|
+
if name:
|
|
53
|
+
return self._element.get(name, None)
|
|
54
|
+
return self._element.attrs
|
|
55
|
+
|
|
56
|
+
def find(self, selector):
|
|
57
|
+
"""
|
|
58
|
+
Find child elements based on a CSS selector and wrap them in PureHTMLBeautifulSoupNode objects.
|
|
59
|
+
"""
|
|
60
|
+
# If `self._element` is the root, use it globally
|
|
61
|
+
if self._element == self._soup:
|
|
62
|
+
elements = self._soup.select(selector)
|
|
63
|
+
else:
|
|
64
|
+
# Otherwise, restrict to children
|
|
65
|
+
elements = self._element.select(selector)
|
|
66
|
+
return [PureHTMLNode(self._soup, el) for el in elements]
|
|
67
|
+
|
|
68
|
+
def html(self):
|
|
69
|
+
"""
|
|
70
|
+
Get the HTML content of the node.
|
|
71
|
+
"""
|
|
72
|
+
return self._element.decode_contents()
|
|
73
|
+
|
|
74
|
+
def is_selector(self, selector):
|
|
75
|
+
"""
|
|
76
|
+
Check if the node matches the selector.
|
|
77
|
+
"""
|
|
78
|
+
matched_elements = self._soup.select(
|
|
79
|
+
selector
|
|
80
|
+
) # Store the matched elements in a variable
|
|
81
|
+
return (
|
|
82
|
+
self._element in matched_elements
|
|
83
|
+
) # Check if current element is in the matched list
|
|
84
|
+
|
|
85
|
+
def text(self):
|
|
86
|
+
"""
|
|
87
|
+
Get the text content of the node.
|
|
88
|
+
"""
|
|
89
|
+
return self._element.get_text()
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import Any
|
|
3
|
+
from typing import Dict
|
|
4
|
+
from typing import List
|
|
5
|
+
from typing import Optional
|
|
6
|
+
from typing import Union
|
|
7
|
+
|
|
8
|
+
import yaml
|
|
9
|
+
|
|
10
|
+
from purehtml.configs.configs import Config
|
|
11
|
+
from purehtml.configs.types.array_config import ArrayConfig
|
|
12
|
+
from purehtml.configs.types.constant_config import ConstantConfig
|
|
13
|
+
from purehtml.configs.types.object_config import ObjectConfig
|
|
14
|
+
from purehtml.configs.types.primitive_value_config import PrimitiveValueConfig
|
|
15
|
+
from purehtml.configs.types.union_config import UnionConfig
|
|
16
|
+
from purehtml.transformers.transformer_factory import TransformerFactory
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class ConfigFactory:
|
|
20
|
+
@staticmethod
|
|
21
|
+
def from_yaml(yaml_input: Union[str, dict]) -> Config:
|
|
22
|
+
"""
|
|
23
|
+
Parses a YAML string or dictionary and generates a Config object.
|
|
24
|
+
"""
|
|
25
|
+
if isinstance(yaml_input, str):
|
|
26
|
+
plain = yaml.safe_load(yaml_input)
|
|
27
|
+
return ConfigFactory.from_dict(plain)
|
|
28
|
+
|
|
29
|
+
elif isinstance(yaml_input, dict):
|
|
30
|
+
return ConfigFactory.from_dict(yaml_input)
|
|
31
|
+
|
|
32
|
+
else:
|
|
33
|
+
logging.error(
|
|
34
|
+
f"ConfigFactory.extract --> ValueError : yaml_input is not dict or str"
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
@staticmethod
|
|
38
|
+
def from_dict(
|
|
39
|
+
plain: Dict[str, Any]
|
|
40
|
+
) -> (
|
|
41
|
+
ConstantConfig | ObjectConfig | ArrayConfig | UnionConfig | PrimitiveValueConfig
|
|
42
|
+
):
|
|
43
|
+
"""
|
|
44
|
+
Generating a Config object based on the input dictionary.
|
|
45
|
+
"""
|
|
46
|
+
constant = plain.get("constant")
|
|
47
|
+
selector_orig = plain.get("selector")
|
|
48
|
+
transform_orig = plain.get("transform")
|
|
49
|
+
properties = plain.get("properties")
|
|
50
|
+
items = plain.get("items")
|
|
51
|
+
union = plain.get("union")
|
|
52
|
+
|
|
53
|
+
selector = ConfigFactory.generate_selector(selector_orig)
|
|
54
|
+
expected_type = ConfigFactory.detect_expected_type(plain)
|
|
55
|
+
transformers = ConfigFactory.generate_transform(transform_orig)
|
|
56
|
+
|
|
57
|
+
if expected_type == "constant":
|
|
58
|
+
return ConstantConfig(constant, selector)
|
|
59
|
+
|
|
60
|
+
elif expected_type == "object":
|
|
61
|
+
|
|
62
|
+
prop_configs = (
|
|
63
|
+
{
|
|
64
|
+
key: ConfigFactory.from_dict(value)
|
|
65
|
+
for key, value in properties.items()
|
|
66
|
+
}
|
|
67
|
+
if isinstance(properties, dict)
|
|
68
|
+
else None
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
return ObjectConfig(selector, prop_configs)
|
|
72
|
+
|
|
73
|
+
elif expected_type == "array":
|
|
74
|
+
|
|
75
|
+
return ArrayConfig(
|
|
76
|
+
selector, ConfigFactory.from_dict(items) if items else None, transformers
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
elif expected_type == "union":
|
|
80
|
+
|
|
81
|
+
return UnionConfig([ConfigFactory.from_dict(u) for u in union])
|
|
82
|
+
|
|
83
|
+
else:
|
|
84
|
+
|
|
85
|
+
return PrimitiveValueConfig(selector, transformers)
|
|
86
|
+
|
|
87
|
+
@staticmethod
|
|
88
|
+
def generate_selector(selector_orig: Any) -> Optional[str]:
|
|
89
|
+
"""
|
|
90
|
+
Generates a selector string from various input formats.
|
|
91
|
+
"""
|
|
92
|
+
if isinstance(selector_orig, str):
|
|
93
|
+
return selector_orig
|
|
94
|
+
|
|
95
|
+
if isinstance(selector_orig, list):
|
|
96
|
+
return ", ".join(map(str, selector_orig))
|
|
97
|
+
|
|
98
|
+
if isinstance(selector_orig, dict):
|
|
99
|
+
return ConfigFactory.generate_selector(selector_orig.get("selector"))
|
|
100
|
+
|
|
101
|
+
if selector_orig is None:
|
|
102
|
+
return None
|
|
103
|
+
|
|
104
|
+
raise ValueError(f"Unexpected selector type: {type(selector_orig).__name__}")
|
|
105
|
+
|
|
106
|
+
@staticmethod
|
|
107
|
+
def generate_transform(transform_orig: Any) -> Optional[List[Any]]:
|
|
108
|
+
"""
|
|
109
|
+
Generates a list of transformers from input data.
|
|
110
|
+
"""
|
|
111
|
+
if transform_orig is None:
|
|
112
|
+
return None
|
|
113
|
+
|
|
114
|
+
if isinstance(transform_orig, str):
|
|
115
|
+
return [TransformerFactory.create(transform_orig)]
|
|
116
|
+
|
|
117
|
+
if isinstance(transform_orig, list):
|
|
118
|
+
return [TransformerFactory.create(t) for t in transform_orig]
|
|
119
|
+
|
|
120
|
+
raise ValueError(f"Unexpected transform type: {type(transform_orig).__name__}")
|
|
121
|
+
|
|
122
|
+
@staticmethod
|
|
123
|
+
def detect_expected_type(conf: Dict[str, Any]) -> str:
|
|
124
|
+
"""
|
|
125
|
+
Detects the expected type of the configuration based on its keys or type field.
|
|
126
|
+
"""
|
|
127
|
+
if "properties" in conf or conf.get("type") == "object":
|
|
128
|
+
return "object"
|
|
129
|
+
|
|
130
|
+
if "items" in conf or conf.get("type") == "array":
|
|
131
|
+
return "array"
|
|
132
|
+
|
|
133
|
+
if "union" in conf:
|
|
134
|
+
return "union"
|
|
135
|
+
|
|
136
|
+
if "constant" in conf:
|
|
137
|
+
return "constant"
|
|
138
|
+
|
|
139
|
+
return "primitive"
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# __init__.py
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
from abc import ABC
|
|
2
|
+
from abc import abstractmethod
|
|
3
|
+
|
|
4
|
+
from purehtml.configs.extract_params import ExtractParams
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class Config(ABC):
|
|
8
|
+
@abstractmethod
|
|
9
|
+
def extract(self, params: ExtractParams) -> str:
|
|
10
|
+
"""
|
|
11
|
+
Abstract method to be implemented by subclasses.
|
|
12
|
+
:param params: ExtractParams object
|
|
13
|
+
:return: result
|
|
14
|
+
"""
|
|
15
|
+
pass
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
from typing import List
|
|
2
|
+
|
|
3
|
+
from purehtml.backend.backend import PureHTMLDocument
|
|
4
|
+
from purehtml.backend.backend import PureHTMLNode
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class ExtractParams:
|
|
8
|
+
"""
|
|
9
|
+
Implementation of the ExtractParams Class. It is used by configs for extraction.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
def __init__(
|
|
13
|
+
self,
|
|
14
|
+
document: PureHTMLDocument,
|
|
15
|
+
nodes: List[PureHTMLNode],
|
|
16
|
+
url: str,
|
|
17
|
+
element_already_matched: bool,
|
|
18
|
+
|
|
19
|
+
):
|
|
20
|
+
self._document = document
|
|
21
|
+
self._nodes = nodes
|
|
22
|
+
self._url = url
|
|
23
|
+
self._element_already_matched = element_already_matched
|
|
24
|
+
|
|
25
|
+
def document(self) -> PureHTMLDocument:
|
|
26
|
+
return self._document
|
|
27
|
+
|
|
28
|
+
def node(self) -> List[PureHTMLNode]:
|
|
29
|
+
return self._nodes
|
|
30
|
+
|
|
31
|
+
def url(self) -> str:
|
|
32
|
+
return self._url
|
|
33
|
+
|
|
34
|
+
def get_element_already_matched(self) -> bool:
|
|
35
|
+
return self._element_already_matched
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
class GetSelectorMatchesParams:
|
|
2
|
+
"""
|
|
3
|
+
A concrete implementation of GetSelectorMatchesParams.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
def __init__(self, already_matched: bool, include_root: bool):
|
|
7
|
+
"""
|
|
8
|
+
Initializes the GetSelectorMatchesParams with required parameters.
|
|
9
|
+
:param already_matched: Whether the selector is already matched.
|
|
10
|
+
:param include_root: Whether the root element should be included.
|
|
11
|
+
"""
|
|
12
|
+
self._already_matched = already_matched
|
|
13
|
+
self._include_root = include_root
|
|
14
|
+
|
|
15
|
+
def is_already_matched(self) -> bool:
|
|
16
|
+
"""
|
|
17
|
+
Returns whether the selector is already matched.
|
|
18
|
+
"""
|
|
19
|
+
return self._already_matched
|
|
20
|
+
|
|
21
|
+
def is_include_root(self) -> bool:
|
|
22
|
+
"""
|
|
23
|
+
Returns whether the root element should be included.
|
|
24
|
+
"""
|
|
25
|
+
return self._include_root
|
|
File without changes
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
from typing import Any
|
|
2
|
+
from typing import List
|
|
3
|
+
|
|
4
|
+
from purehtml.configs.configs import Config
|
|
5
|
+
from purehtml.configs.extract_params import ExtractParams
|
|
6
|
+
from purehtml.configs.selector_match_params import GetSelectorMatchesParams
|
|
7
|
+
from purehtml.configs.types.config_with_selector import ConfigWithSelector
|
|
8
|
+
from purehtml.configs.types.primitive_value_config import PrimitiveValueConfig
|
|
9
|
+
from purehtml.transformers.transformer import Transformer
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class ArrayConfig(ConfigWithSelector):
|
|
13
|
+
"""
|
|
14
|
+
Concrete class representing an array configuration with selector and transformers.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
def __init__(self, selector: str, items: Config, transform: List[Transformer]):
|
|
18
|
+
"""
|
|
19
|
+
Initializes the ArrayConfig with selector, item config, and transformers.
|
|
20
|
+
:param selector: The selector string.
|
|
21
|
+
:param items: The configuration for the items in the array.
|
|
22
|
+
:param transform: A list of transformers to apply.
|
|
23
|
+
"""
|
|
24
|
+
super().__init__()
|
|
25
|
+
self.selector = selector
|
|
26
|
+
self.items = items
|
|
27
|
+
self.transform = transform
|
|
28
|
+
|
|
29
|
+
def extract(self, params: ExtractParams) -> Any:
|
|
30
|
+
"""
|
|
31
|
+
Extract values from the array based on the provided parameters.
|
|
32
|
+
:param params: The ExtractParams object containing context for extraction.
|
|
33
|
+
:return: A list of extracted values.
|
|
34
|
+
"""
|
|
35
|
+
# Get all matches for the element
|
|
36
|
+
|
|
37
|
+
selector_params = GetSelectorMatchesParams(
|
|
38
|
+
already_matched=params.get_element_already_matched() or False,
|
|
39
|
+
include_root=False
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
elements = self.get_all_matches(params.node(), selector_params, params.document())
|
|
43
|
+
|
|
44
|
+
# Use the provided `items` config, or a default primitive config
|
|
45
|
+
conf = self.items if self.items is not None else PrimitiveValueConfig(None, self.transform)
|
|
46
|
+
|
|
47
|
+
result = []
|
|
48
|
+
|
|
49
|
+
for el in elements:
|
|
50
|
+
extract_params = ExtractParams(document=params.document(),
|
|
51
|
+
nodes=[el],
|
|
52
|
+
url=params.url(),
|
|
53
|
+
element_already_matched=params.get_element_already_matched())
|
|
54
|
+
|
|
55
|
+
# Extract the value
|
|
56
|
+
result.append(conf.extract(extract_params))
|
|
57
|
+
|
|
58
|
+
return result
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
from abc import ABC
|
|
2
|
+
from typing import Any
|
|
3
|
+
from typing import List
|
|
4
|
+
from typing import Optional
|
|
5
|
+
from typing import Union
|
|
6
|
+
|
|
7
|
+
from purehtml.backend.backend import PureHTMLDocument
|
|
8
|
+
from purehtml.backend.backend import PureHTMLNode
|
|
9
|
+
from purehtml.configs.configs import Config
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class ConfigWithSelector(Config, ABC):
|
|
13
|
+
|
|
14
|
+
def __init__(self, selector: Optional[str] = None):
|
|
15
|
+
"""
|
|
16
|
+
Initialize ConfigWithSelector with an optional CSS selector.
|
|
17
|
+
:param selector: The CSS selector to use for matching.
|
|
18
|
+
"""
|
|
19
|
+
super().__init__()
|
|
20
|
+
self.selector = selector
|
|
21
|
+
|
|
22
|
+
def get_all_matches(self, element: List, params: Any, doc: PureHTMLDocument) \
|
|
23
|
+
-> List[PureHTMLNode]:
|
|
24
|
+
"""
|
|
25
|
+
Get the all matches according to below conditions. It will return the given
|
|
26
|
+
element if it is already matched, no selector is provided.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
if self.selector is None:
|
|
30
|
+
# If no selector is provided, return the element as is.
|
|
31
|
+
return element
|
|
32
|
+
|
|
33
|
+
if params.is_already_matched():
|
|
34
|
+
return element
|
|
35
|
+
|
|
36
|
+
if params.is_include_root():
|
|
37
|
+
if len(element) == 0:
|
|
38
|
+
root = doc.select(self.selector)
|
|
39
|
+
|
|
40
|
+
for nod in root:
|
|
41
|
+
if nod.is_selector(self.selector):
|
|
42
|
+
return [nod]
|
|
43
|
+
|
|
44
|
+
else:
|
|
45
|
+
return root
|
|
46
|
+
else:
|
|
47
|
+
for node in element:
|
|
48
|
+
if node.is_selector(self.selector):
|
|
49
|
+
return [node]
|
|
50
|
+
return []
|
|
51
|
+
|
|
52
|
+
if len(element) == 0:
|
|
53
|
+
|
|
54
|
+
nodes = doc.select(self.selector)
|
|
55
|
+
results = []
|
|
56
|
+
|
|
57
|
+
for node in nodes:
|
|
58
|
+
|
|
59
|
+
if params.is_include_root():
|
|
60
|
+
results.extend(node.find(self.selector))
|
|
61
|
+
|
|
62
|
+
return results if params.is_include_root() else nodes
|
|
63
|
+
|
|
64
|
+
else:
|
|
65
|
+
nodes = element[0].find(self.selector)
|
|
66
|
+
return nodes
|
|
67
|
+
|
|
68
|
+
def get_first_match(self,
|
|
69
|
+
element: Union[List[PureHTMLNode], PureHTMLNode],
|
|
70
|
+
params: Any,
|
|
71
|
+
doc: PureHTMLDocument) \
|
|
72
|
+
-> Optional[PureHTMLNode]:
|
|
73
|
+
"""
|
|
74
|
+
Get the first match for the selector.
|
|
75
|
+
"""
|
|
76
|
+
matches = self.get_all_matches(element, params, doc)
|
|
77
|
+
if matches:
|
|
78
|
+
return matches[0]
|
|
79
|
+
return None
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
from typing import Any
|
|
2
|
+
from typing import Optional
|
|
3
|
+
|
|
4
|
+
from purehtml.configs.selector_match_params import GetSelectorMatchesParams
|
|
5
|
+
from purehtml.configs.types.config_with_selector import ConfigWithSelector
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class ConstantConfig(ConfigWithSelector):
|
|
9
|
+
def __init__(self, val: Any, selector: Optional[str]):
|
|
10
|
+
"""
|
|
11
|
+
Initialize ConstantConfig with a value and an optional selector.
|
|
12
|
+
:param val: The constant value to be returned.
|
|
13
|
+
:param selector: The CSS selector to match nodes.
|
|
14
|
+
"""
|
|
15
|
+
super().__init__()
|
|
16
|
+
self.val = val
|
|
17
|
+
self.selector = selector
|
|
18
|
+
|
|
19
|
+
def extract(self, params: Any) -> Optional[str]:
|
|
20
|
+
"""
|
|
21
|
+
Extracts data based on the selector and parameters provided.
|
|
22
|
+
:param params: An object implementing ExtractParams interface.
|
|
23
|
+
:return: A JSON string of the value or None if no match is found.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
if self.selector is not None:
|
|
27
|
+
# Create a local equivalent of GetSelectorMatchesParams
|
|
28
|
+
selector_params = GetSelectorMatchesParams(False, True)
|
|
29
|
+
|
|
30
|
+
# Attempt to get matches using the selector
|
|
31
|
+
matches = self.get_all_matches(params.node(), selector_params, params.document)
|
|
32
|
+
if matches is None:
|
|
33
|
+
return None
|
|
34
|
+
|
|
35
|
+
return self.val
|
|
36
|
+
|
|
37
|
+
# Default return if no selector
|
|
38
|
+
return self.val
|