scrapling 0.2.7__py3-none-any.whl → 0.2.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scrapling/__init__.py +5 -4
- scrapling/core/_types.py +2 -3
- scrapling/core/custom_types.py +93 -11
- scrapling/core/storage_adaptors.py +9 -10
- scrapling/core/translator.py +6 -7
- scrapling/core/utils.py +35 -30
- scrapling/defaults.py +2 -1
- scrapling/engines/__init__.py +2 -2
- scrapling/engines/camo.py +96 -26
- scrapling/engines/constants.py +4 -4
- scrapling/engines/pw.py +166 -96
- scrapling/engines/static.py +94 -50
- scrapling/engines/toolbelt/__init__.py +6 -20
- scrapling/engines/toolbelt/custom.py +22 -23
- scrapling/engines/toolbelt/fingerprints.py +7 -7
- scrapling/engines/toolbelt/navigation.py +25 -12
- scrapling/fetchers.py +233 -17
- scrapling/parser.py +63 -28
- {scrapling-0.2.7.dist-info → scrapling-0.2.9.dist-info}/METADATA +41 -25
- scrapling-0.2.9.dist-info/RECORD +47 -0
- tests/fetchers/async/__init__.py +0 -0
- tests/fetchers/async/test_camoufox.py +95 -0
- tests/fetchers/async/test_httpx.py +83 -0
- tests/fetchers/async/test_playwright.py +99 -0
- tests/fetchers/sync/__init__.py +0 -0
- tests/fetchers/sync/test_camoufox.py +68 -0
- tests/fetchers/sync/test_httpx.py +82 -0
- tests/fetchers/sync/test_playwright.py +87 -0
- tests/fetchers/test_utils.py +90 -122
- tests/parser/test_automatch.py +64 -9
- tests/parser/test_general.py +263 -219
- scrapling-0.2.7.dist-info/RECORD +0 -42
- tests/fetchers/test_camoufox.py +0 -64
- tests/fetchers/test_httpx.py +0 -67
- tests/fetchers/test_playwright.py +0 -76
- {scrapling-0.2.7.dist-info → scrapling-0.2.9.dist-info}/LICENSE +0 -0
- {scrapling-0.2.7.dist-info → scrapling-0.2.9.dist-info}/WHEEL +0 -0
- {scrapling-0.2.7.dist-info → scrapling-0.2.9.dist-info}/top_level.txt +0 -0
tests/parser/test_automatch.py
CHANGED
@@ -1,10 +1,11 @@
|
|
1
|
-
import
|
1
|
+
import asyncio
|
2
2
|
|
3
|
-
|
3
|
+
import pytest
|
4
4
|
|
5
|
+
from scrapling import Adaptor
|
5
6
|
|
6
|
-
class TestParserAutoMatch(unittest.TestCase):
|
7
7
|
|
8
|
+
class TestParserAutoMatch:
|
8
9
|
def test_element_relocation(self):
|
9
10
|
"""Test relocating element after structure change"""
|
10
11
|
original_html = '''
|
@@ -42,15 +43,69 @@ class TestParserAutoMatch(unittest.TestCase):
|
|
42
43
|
</div>
|
43
44
|
'''
|
44
45
|
|
45
|
-
old_page = Adaptor(original_html, url='example.com', auto_match=True
|
46
|
-
new_page = Adaptor(changed_html, url='example.com', auto_match=True
|
46
|
+
old_page = Adaptor(original_html, url='example.com', auto_match=True)
|
47
|
+
new_page = Adaptor(changed_html, url='example.com', auto_match=True)
|
48
|
+
|
49
|
+
# 'p1' was used as ID and now it's not and all the path elements have changes
|
50
|
+
# Also at the same time testing auto-match vs combined selectors
|
51
|
+
_ = old_page.css('#p1, #p2', auto_save=True)[0]
|
52
|
+
relocated = new_page.css('#p1', auto_match=True)
|
53
|
+
|
54
|
+
assert relocated is not None
|
55
|
+
assert relocated[0].attrib['data-id'] == 'p1'
|
56
|
+
assert relocated[0].has_class('new-class')
|
57
|
+
assert relocated[0].css('.new-description')[0].text == 'Description 1'
|
58
|
+
|
59
|
+
@pytest.mark.asyncio
|
60
|
+
async def test_element_relocation_async(self):
|
61
|
+
"""Test relocating element after structure change in async mode"""
|
62
|
+
original_html = '''
|
63
|
+
<div class="container">
|
64
|
+
<section class="products">
|
65
|
+
<article class="product" id="p1">
|
66
|
+
<h3>Product 1</h3>
|
67
|
+
<p class="description">Description 1</p>
|
68
|
+
</article>
|
69
|
+
<article class="product" id="p2">
|
70
|
+
<h3>Product 2</h3>
|
71
|
+
<p class="description">Description 2</p>
|
72
|
+
</article>
|
73
|
+
</section>
|
74
|
+
</div>
|
75
|
+
'''
|
76
|
+
changed_html = '''
|
77
|
+
<div class="new-container">
|
78
|
+
<div class="product-wrapper">
|
79
|
+
<section class="products">
|
80
|
+
<article class="product new-class" data-id="p1">
|
81
|
+
<div class="product-info">
|
82
|
+
<h3>Product 1</h3>
|
83
|
+
<p class="new-description">Description 1</p>
|
84
|
+
</div>
|
85
|
+
</article>
|
86
|
+
<article class="product new-class" data-id="p2">
|
87
|
+
<div class="product-info">
|
88
|
+
<h3>Product 2</h3>
|
89
|
+
<p class="new-description">Description 2</p>
|
90
|
+
</div>
|
91
|
+
</article>
|
92
|
+
</section>
|
93
|
+
</div>
|
94
|
+
</div>
|
95
|
+
'''
|
96
|
+
|
97
|
+
# Simulate async operation
|
98
|
+
await asyncio.sleep(0.1) # Minimal async operation
|
99
|
+
|
100
|
+
old_page = Adaptor(original_html, url='example.com', auto_match=True)
|
101
|
+
new_page = Adaptor(changed_html, url='example.com', auto_match=True)
|
47
102
|
|
48
103
|
# 'p1' was used as ID and now it's not and all the path elements have changes
|
49
104
|
# Also at the same time testing auto-match vs combined selectors
|
50
105
|
_ = old_page.css('#p1, #p2', auto_save=True)[0]
|
51
106
|
relocated = new_page.css('#p1', auto_match=True)
|
52
107
|
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
108
|
+
assert relocated is not None
|
109
|
+
assert relocated[0].attrib['data-id'] == 'p1'
|
110
|
+
assert relocated[0].has_class('new-class')
|
111
|
+
assert relocated[0].css('.new-description')[0].text == 'Description 1'
|
tests/parser/test_general.py
CHANGED
@@ -1,286 +1,330 @@
|
|
1
|
-
|
2
1
|
import pickle
|
3
|
-
import
|
4
|
-
|
2
|
+
import time
|
3
|
+
|
4
|
+
import pytest
|
5
5
|
from cssselect import SelectorError, SelectorSyntaxError
|
6
6
|
|
7
|
+
from scrapling import Adaptor
|
8
|
+
|
7
9
|
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
10
|
+
@pytest.fixture
|
11
|
+
def html_content():
|
12
|
+
return '''
|
13
|
+
<html>
|
14
|
+
<head>
|
15
|
+
<title>Complex Web Page</title>
|
16
|
+
<style>
|
17
|
+
.hidden { display: none; }
|
18
|
+
</style>
|
19
|
+
</head>
|
20
|
+
<body>
|
21
|
+
<header>
|
22
|
+
<nav>
|
23
|
+
<ul>
|
24
|
+
<li><a href="#home">Home</a></li>
|
25
|
+
<li><a href="#about">About</a></li>
|
26
|
+
<li><a href="#contact">Contact</a></li>
|
27
|
+
</ul>
|
28
|
+
</nav>
|
29
|
+
</header>
|
30
|
+
<main>
|
31
|
+
<section id="products" schema='{"jsonable": "data"}'>
|
32
|
+
<h2>Products</h2>
|
33
|
+
<div class="product-list">
|
34
|
+
<article class="product" data-id="1">
|
35
|
+
<h3>Product 1</h3>
|
36
|
+
<p class="description">This is product 1</p>
|
37
|
+
<span class="price">$10.99</span>
|
38
|
+
<div class="hidden stock">In stock: 5</div>
|
39
|
+
</article>
|
40
|
+
<article class="product" data-id="2">
|
41
|
+
<h3>Product 2</h3>
|
42
|
+
<p class="description">This is product 2</p>
|
43
|
+
<span class="price">$20.99</span>
|
44
|
+
<div class="hidden stock">In stock: 3</div>
|
45
|
+
</article>
|
46
|
+
<article class="product" data-id="3">
|
47
|
+
<h3>Product 3</h3>
|
48
|
+
<p class="description">This is product 3</p>
|
49
|
+
<span class="price">$15.99</span>
|
50
|
+
<div class="hidden stock">Out of stock</div>
|
51
|
+
</article>
|
52
|
+
</div>
|
53
|
+
</section>
|
54
|
+
<section id="reviews">
|
55
|
+
<h2>Customer Reviews</h2>
|
56
|
+
<div class="review-list">
|
57
|
+
<div class="review" data-rating="5">
|
58
|
+
<p class="review-text">Great product!</p>
|
59
|
+
<span class="reviewer">John Doe</span>
|
50
60
|
</div>
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
<div class="review-list">
|
55
|
-
<div class="review" data-rating="5">
|
56
|
-
<p class="review-text">Great product!</p>
|
57
|
-
<span class="reviewer">John Doe</span>
|
58
|
-
</div>
|
59
|
-
<div class="review" data-rating="4">
|
60
|
-
<p class="review-text">Good value for money.</p>
|
61
|
-
<span class="reviewer">Jane Smith</span>
|
62
|
-
</div>
|
61
|
+
<div class="review" data-rating="4">
|
62
|
+
<p class="review-text">Good value for money.</p>
|
63
|
+
<span class="reviewer">Jane Smith</span>
|
63
64
|
</div>
|
64
|
-
</
|
65
|
-
</
|
66
|
-
|
67
|
-
|
68
|
-
</
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
</
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
65
|
+
</div>
|
66
|
+
</section>
|
67
|
+
</main>
|
68
|
+
<footer>
|
69
|
+
<p>© 2024 Our Company</p>
|
70
|
+
</footer>
|
71
|
+
<script id="page-data" type="application/json">
|
72
|
+
{"lastUpdated": "2024-09-22T10:30:00Z", "totalProducts": 3}
|
73
|
+
</script>
|
74
|
+
</body>
|
75
|
+
</html>
|
76
|
+
'''
|
77
|
+
|
78
|
+
|
79
|
+
@pytest.fixture
|
80
|
+
def page(html_content):
|
81
|
+
return Adaptor(html_content, auto_match=False)
|
82
|
+
|
83
|
+
|
84
|
+
# CSS Selector Tests
|
85
|
+
class TestCSSSelectors:
|
86
|
+
def test_basic_product_selection(self, page):
|
87
|
+
"""Test selecting all product elements"""
|
88
|
+
elements = page.css('main #products .product-list article.product')
|
89
|
+
assert len(elements) == 3
|
90
|
+
|
91
|
+
def test_in_stock_product_selection(self, page):
|
92
|
+
"""Test selecting in-stock products"""
|
93
|
+
in_stock_products = page.css(
|
83
94
|
'main #products .product-list article.product:not(:contains("Out of stock"))')
|
84
|
-
|
95
|
+
assert len(in_stock_products) == 2
|
96
|
+
|
85
97
|
|
86
|
-
|
87
|
-
|
88
|
-
|
98
|
+
# XPath Selector Tests
|
99
|
+
class TestXPathSelectors:
|
100
|
+
def test_high_rating_reviews(self, page):
|
101
|
+
"""Test selecting reviews with high ratings"""
|
102
|
+
reviews = page.xpath(
|
89
103
|
'//section[@id="reviews"]//div[contains(@class, "review") and @data-rating >= 4]'
|
90
104
|
)
|
91
|
-
|
105
|
+
assert len(reviews) == 2
|
92
106
|
|
93
|
-
|
107
|
+
def test_high_priced_products(self, page):
|
108
|
+
"""Test selecting products above a certain price"""
|
109
|
+
high_priced_products = page.xpath(
|
94
110
|
'//article[contains(@class, "product")]'
|
95
111
|
'[number(translate(substring-after(.//span[@class="price"], "$"), ",", "")) > 15]'
|
96
112
|
)
|
97
|
-
|
113
|
+
assert len(high_priced_products) == 2
|
114
|
+
|
115
|
+
|
116
|
+
# Text Matching Tests
|
117
|
+
class TestTextMatching:
|
118
|
+
def test_regex_multiple_matches(self, page):
|
119
|
+
"""Test finding multiple matches with regex"""
|
120
|
+
stock_info = page.find_by_regex(r'In stock: \d+', first_match=False)
|
121
|
+
assert len(stock_info) == 2
|
98
122
|
|
99
|
-
def
|
100
|
-
"""Test
|
101
|
-
stock_info =
|
102
|
-
|
123
|
+
def test_regex_first_match(self, page):
|
124
|
+
"""Test finding the first match with regex"""
|
125
|
+
stock_info = page.find_by_regex(r'In stock: \d+', first_match=True, case_sensitive=True)
|
126
|
+
assert stock_info.text == 'In stock: 5'
|
103
127
|
|
104
|
-
|
105
|
-
|
128
|
+
def test_partial_text_match(self, page):
|
129
|
+
"""Test finding elements with partial text match"""
|
130
|
+
stock_info = page.find_by_text(r'In stock:', partial=True, first_match=False)
|
131
|
+
assert len(stock_info) == 2
|
106
132
|
|
107
|
-
|
108
|
-
|
133
|
+
def test_exact_text_match(self, page):
|
134
|
+
"""Test finding elements with exact text match"""
|
135
|
+
out_of_stock = page.find_by_text('Out of stock', partial=False, first_match=False)
|
136
|
+
assert len(out_of_stock) == 1
|
109
137
|
|
110
|
-
out_of_stock = self.page.find_by_text('Out of stock', partial=False, first_match=False)
|
111
|
-
self.assertEqual(len(out_of_stock), 1)
|
112
138
|
|
113
|
-
|
114
|
-
|
115
|
-
|
139
|
+
# Similar Elements Tests
|
140
|
+
class TestSimilarElements:
|
141
|
+
def test_finding_similar_products(self, page):
|
142
|
+
"""Test finding similar product elements"""
|
143
|
+
first_product = page.css_first('.product')
|
116
144
|
similar_products = first_product.find_similar()
|
117
|
-
|
145
|
+
assert len(similar_products) == 2
|
118
146
|
|
119
|
-
|
147
|
+
def test_finding_similar_reviews(self, page):
|
148
|
+
"""Test finding similar review elements with additional filtering"""
|
149
|
+
first_review = page.find('div', class_='review')
|
120
150
|
similar_high_rated_reviews = [
|
121
151
|
review
|
122
152
|
for review in first_review.find_similar()
|
123
153
|
if int(review.attrib.get('data-rating', 0)) >= 4
|
124
154
|
]
|
125
|
-
|
155
|
+
assert len(similar_high_rated_reviews) == 1
|
126
156
|
|
127
|
-
|
128
|
-
|
129
|
-
|
157
|
+
|
158
|
+
# Error Handling Tests
|
159
|
+
class TestErrorHandling:
|
160
|
+
def test_invalid_adaptor_initialization(self):
|
161
|
+
"""Test various invalid Adaptor initializations"""
|
162
|
+
# No arguments
|
163
|
+
with pytest.raises(ValueError):
|
130
164
|
_ = Adaptor(auto_match=False)
|
131
165
|
|
132
|
-
|
166
|
+
# Invalid argument types
|
167
|
+
with pytest.raises(TypeError):
|
133
168
|
_ = Adaptor(root="ayo", auto_match=False)
|
134
169
|
|
135
|
-
with
|
170
|
+
with pytest.raises(TypeError):
|
136
171
|
_ = Adaptor(text=1, auto_match=False)
|
137
172
|
|
138
|
-
with
|
173
|
+
with pytest.raises(TypeError):
|
139
174
|
_ = Adaptor(body=1, auto_match=False)
|
140
175
|
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
"""Test that objects aren't pickleable"""
|
146
|
-
table = self.page.css('.product-list')[0]
|
147
|
-
with self.assertRaises(TypeError): # Adaptors
|
148
|
-
pickle.dumps(table)
|
149
|
-
|
150
|
-
with self.assertRaises(TypeError): # Adaptor
|
151
|
-
pickle.dumps(table[0])
|
152
|
-
|
153
|
-
def test_overridden(self):
|
154
|
-
"""Test overridden functions"""
|
155
|
-
table = self.page.css('.product-list')[0]
|
156
|
-
self.assertTrue(issubclass(type(table.__str__()), str))
|
157
|
-
self.assertTrue(issubclass(type(table.__repr__()), str))
|
158
|
-
self.assertTrue(issubclass(type(table.attrib.__str__()), str))
|
159
|
-
self.assertTrue(issubclass(type(table.attrib.__repr__()), str))
|
160
|
-
|
161
|
-
def test_bad_selector(self):
|
162
|
-
"""Test object can handle bad selector"""
|
163
|
-
with self.assertRaises((SelectorError, SelectorSyntaxError,)):
|
164
|
-
self.page.css('4 ayo')
|
165
|
-
|
166
|
-
with self.assertRaises((SelectorError, SelectorSyntaxError,)):
|
167
|
-
self.page.xpath('4 ayo')
|
176
|
+
def test_invalid_storage(self, page, html_content):
|
177
|
+
"""Test invalid storage parameter"""
|
178
|
+
with pytest.raises(ValueError):
|
179
|
+
_ = Adaptor(html_content, storage=object, auto_match=True)
|
168
180
|
|
169
|
-
def
|
170
|
-
"""
|
171
|
-
|
172
|
-
|
173
|
-
self.assertTrue(type(element.generate_xpath_selector) is str)
|
174
|
-
for branch in element.children:
|
175
|
-
_traverse(branch)
|
181
|
+
def test_bad_selectors(self, page):
|
182
|
+
"""Test handling of invalid selectors"""
|
183
|
+
with pytest.raises((SelectorError, SelectorSyntaxError)):
|
184
|
+
page.css('4 ayo')
|
176
185
|
|
177
|
-
|
186
|
+
with pytest.raises((SelectorError, SelectorSyntaxError)):
|
187
|
+
page.xpath('4 ayo')
|
178
188
|
|
179
|
-
def test_getting_all_text(self):
|
180
|
-
"""Test getting all text"""
|
181
|
-
self.assertNotEqual(self.page.get_all_text(), '')
|
182
189
|
|
183
|
-
|
184
|
-
|
185
|
-
|
190
|
+
# Pickling and Object Representation Tests
|
191
|
+
class TestPicklingAndRepresentation:
|
192
|
+
def test_unpickleable_objects(self, page):
|
193
|
+
"""Test that Adaptor objects cannot be pickled"""
|
194
|
+
table = page.css('.product-list')[0]
|
195
|
+
with pytest.raises(TypeError):
|
196
|
+
pickle.dumps(table)
|
186
197
|
|
187
|
-
|
188
|
-
|
189
|
-
self.assertNotEqual(table.prettify(), '')
|
198
|
+
with pytest.raises(TypeError):
|
199
|
+
pickle.dumps(table[0])
|
190
200
|
|
201
|
+
def test_string_representations(self, page):
|
202
|
+
"""Test custom string representations of objects"""
|
203
|
+
table = page.css('.product-list')[0]
|
204
|
+
assert issubclass(type(table.__str__()), str)
|
205
|
+
assert issubclass(type(table.__repr__()), str)
|
206
|
+
assert issubclass(type(table.attrib.__str__()), str)
|
207
|
+
assert issubclass(type(table.attrib.__repr__()), str)
|
208
|
+
|
209
|
+
|
210
|
+
# Navigation and Traversal Tests
|
211
|
+
class TestElementNavigation:
|
212
|
+
def test_basic_navigation_properties(self, page):
|
213
|
+
"""Test basic navigation properties of elements"""
|
214
|
+
table = page.css('.product-list')[0]
|
215
|
+
assert table.path is not None
|
216
|
+
assert table.html_content != ''
|
217
|
+
assert table.prettify() != ''
|
218
|
+
|
219
|
+
def test_parent_and_sibling_navigation(self, page):
|
220
|
+
"""Test parent and sibling navigation"""
|
221
|
+
table = page.css('.product-list')[0]
|
191
222
|
parent = table.parent
|
192
|
-
|
193
|
-
|
194
|
-
children = table.children
|
195
|
-
self.assertEqual(len(children), 3)
|
223
|
+
assert parent.attrib['id'] == 'products'
|
196
224
|
|
197
225
|
parent_siblings = parent.siblings
|
198
|
-
|
226
|
+
assert len(parent_siblings) == 1
|
199
227
|
|
200
|
-
|
228
|
+
def test_child_navigation(self, page):
|
229
|
+
"""Test child navigation"""
|
230
|
+
table = page.css('.product-list')[0]
|
231
|
+
children = table.children
|
232
|
+
assert len(children) == 3
|
233
|
+
|
234
|
+
def test_next_and_previous_navigation(self, page):
|
235
|
+
"""Test next and previous element navigation"""
|
236
|
+
child = page.css('.product-list')[0].find({'data-id': "1"})
|
201
237
|
next_element = child.next
|
202
|
-
|
238
|
+
assert next_element.attrib['data-id'] == '2'
|
203
239
|
|
204
240
|
prev_element = next_element.previous
|
205
|
-
|
241
|
+
assert prev_element.tag == child.tag
|
206
242
|
|
207
|
-
|
243
|
+
def test_ancestor_finding(self, page):
|
244
|
+
"""Test finding ancestors of elements"""
|
245
|
+
all_prices = page.css('.price')
|
208
246
|
products_with_prices = [
|
209
247
|
price.find_ancestor(lambda p: p.has_class('product'))
|
210
248
|
for price in all_prices
|
211
249
|
]
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
soup = Adaptor(test_html, auto_match=False, keep_comments=False)
|
222
|
-
html_tag = soup.css('html')[0]
|
223
|
-
self.assertEqual(html_tag.path, [])
|
224
|
-
self.assertEqual(html_tag.siblings, [])
|
225
|
-
self.assertEqual(html_tag.parent, None)
|
226
|
-
self.assertEqual(html_tag.find_ancestor(lambda e: e), None)
|
227
|
-
|
228
|
-
self.assertEqual(soup.css('#a a')[0].next, None)
|
229
|
-
self.assertEqual(soup.css('#b a')[0].previous, None)
|
230
|
-
|
231
|
-
def test_text_to_json(self):
|
232
|
-
"""Test converting text to json"""
|
233
|
-
script_content = self.page.css('#page-data::text')[0]
|
234
|
-
self.assertTrue(issubclass(type(script_content.sort()), str))
|
250
|
+
assert len(products_with_prices) == 3
|
251
|
+
|
252
|
+
|
253
|
+
# JSON and Attribute Tests
|
254
|
+
class TestJSONAndAttributes:
|
255
|
+
def test_json_conversion(self, page):
|
256
|
+
"""Test converting content to JSON"""
|
257
|
+
script_content = page.css('#page-data::text')[0]
|
258
|
+
assert issubclass(type(script_content.sort()), str)
|
235
259
|
page_data = script_content.json()
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
def
|
240
|
-
"""Test
|
241
|
-
|
242
|
-
|
243
|
-
self.assertEqual(match, '10.99')
|
244
|
-
match = element.text.re(r'(\d+)', replace_entities=False)
|
245
|
-
self.assertEqual(len(match), 2)
|
246
|
-
|
247
|
-
def test_attribute_operations(self):
|
248
|
-
"""Test operations on elements attributes"""
|
249
|
-
products = self.page.css('.product')
|
260
|
+
assert page_data['totalProducts'] == 3
|
261
|
+
assert 'lastUpdated' in page_data
|
262
|
+
|
263
|
+
def test_attribute_operations(self, page):
|
264
|
+
"""Test various attribute-related operations"""
|
265
|
+
# Product ID extraction
|
266
|
+
products = page.css('.product')
|
250
267
|
product_ids = [product.attrib['data-id'] for product in products]
|
251
|
-
|
252
|
-
|
268
|
+
assert product_ids == ['1', '2', '3']
|
269
|
+
assert 'data-id' in products[0].attrib
|
253
270
|
|
254
|
-
|
271
|
+
# Review rating calculations
|
272
|
+
reviews = page.css('.review')
|
255
273
|
review_ratings = [int(review.attrib['data-rating']) for review in reviews]
|
256
|
-
|
274
|
+
assert sum(review_ratings) / len(review_ratings) == 4.5
|
257
275
|
|
276
|
+
# Attribute searching
|
258
277
|
key_value = list(products[0].attrib.search_values('1', partial=False))
|
259
|
-
|
278
|
+
assert list(key_value[0].keys()) == ['data-id']
|
260
279
|
|
261
280
|
key_value = list(products[0].attrib.search_values('1', partial=True))
|
262
|
-
|
281
|
+
assert list(key_value[0].keys()) == ['data-id']
|
282
|
+
|
283
|
+
# JSON attribute conversion
|
284
|
+
attr_json = page.css_first('#products').attrib['schema'].json()
|
285
|
+
assert attr_json == {'jsonable': 'data'}
|
286
|
+
assert isinstance(page.css('#products')[0].attrib.json_string, bytes)
|
287
|
+
|
288
|
+
|
289
|
+
# Performance Test
|
290
|
+
def test_large_html_parsing_performance():
|
291
|
+
"""Test parsing and selecting performance on large HTML"""
|
292
|
+
large_html = '<html><body>' + '<div class="item">' * 5000 + '</div>' * 5000 + '</body></html>'
|
293
|
+
|
294
|
+
start_time = time.time()
|
295
|
+
parsed = Adaptor(large_html, auto_match=False)
|
296
|
+
elements = parsed.css('.item')
|
297
|
+
end_time = time.time()
|
298
|
+
|
299
|
+
assert len(elements) == 5000
|
300
|
+
# Converting 5000 elements to a class and doing operations on them will take time
|
301
|
+
# Based on my tests with 100 runs, 1 loop each Scrapling (given the extra work/features) takes 10.4ms on average
|
302
|
+
assert end_time - start_time < 0.5 # Locally I test on 0.1 but on GitHub actions with browsers and threading sometimes closing adds fractions of seconds
|
303
|
+
|
304
|
+
|
305
|
+
# Selector Generation Test
|
306
|
+
def test_selectors_generation(page):
|
307
|
+
"""Try to create selectors for all elements in the page"""
|
263
308
|
|
264
|
-
|
265
|
-
|
266
|
-
|
309
|
+
def _traverse(element: Adaptor):
|
310
|
+
assert isinstance(element.generate_css_selector, str)
|
311
|
+
assert isinstance(element.generate_xpath_selector, str)
|
312
|
+
for branch in element.children:
|
313
|
+
_traverse(branch)
|
267
314
|
|
268
|
-
|
269
|
-
"""Test parsing and selecting speed"""
|
270
|
-
import time
|
271
|
-
large_html = '<html><body>' + '<div class="item">' * 5000 + '</div>' * 5000 + '</body></html>'
|
315
|
+
_traverse(page)
|
272
316
|
|
273
|
-
start_time = time.time()
|
274
|
-
parsed = Adaptor(large_html, auto_match=False, debug=False)
|
275
|
-
elements = parsed.css('.item')
|
276
|
-
end_time = time.time()
|
277
317
|
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
318
|
+
# Miscellaneous Tests
|
319
|
+
def test_getting_all_text(page):
|
320
|
+
"""Test getting all text from the page"""
|
321
|
+
assert page.get_all_text() != ''
|
282
322
|
|
283
323
|
|
284
|
-
|
285
|
-
|
286
|
-
|
324
|
+
def test_regex_on_text(page):
|
325
|
+
"""Test regex operations on text"""
|
326
|
+
element = page.css('[data-id="1"] .price')[0]
|
327
|
+
match = element.re_first(r'[\.\d]+')
|
328
|
+
assert match == '10.99'
|
329
|
+
match = element.text.re(r'(\d+)', replace_entities=False)
|
330
|
+
assert len(match) == 2
|