scrapling 0.2.98__py3-none-any.whl → 0.3__py3-none-any.whl

tests/fetchers/test_utils.py

@@ -1,97 +0,0 @@
-import pytest
-
-from scrapling.engines.toolbelt.custom import ResponseEncoding, StatusText
-
-
-@pytest.fixture
-def content_type_map():
-    return {
-        # A map generated by ChatGPT for most possible `content_type` values and the expected outcome
-        'text/html; charset=UTF-8': 'UTF-8',
-        'text/html; charset=ISO-8859-1': 'ISO-8859-1',
-        'text/html': 'ISO-8859-1',
-        'application/json; charset=UTF-8': 'UTF-8',
-        'application/json': 'utf-8',
-        'text/json': 'utf-8',
-        'application/javascript; charset=UTF-8': 'UTF-8',
-        'application/javascript': 'utf-8',
-        'text/plain; charset=UTF-8': 'UTF-8',
-        'text/plain; charset=ISO-8859-1': 'ISO-8859-1',
-        'text/plain': 'ISO-8859-1',
-        'application/xhtml+xml; charset=UTF-8': 'UTF-8',
-        'application/xhtml+xml': 'utf-8',
-        'text/html; charset=windows-1252': 'windows-1252',
-        'application/json; charset=windows-1252': 'windows-1252',
-        'text/plain; charset=windows-1252': 'windows-1252',
-        'text/html; charset="UTF-8"': 'UTF-8',
-        'text/html; charset="ISO-8859-1"': 'ISO-8859-1',
-        'text/html; charset="windows-1252"': 'windows-1252',
-        'application/json; charset="UTF-8"': 'UTF-8',
-        'application/json; charset="ISO-8859-1"': 'ISO-8859-1',
-        'application/json; charset="windows-1252"': 'windows-1252',
-        'text/json; charset="UTF-8"': 'UTF-8',
-        'application/javascript; charset="UTF-8"': 'UTF-8',
-        'application/javascript; charset="ISO-8859-1"': 'ISO-8859-1',
-        'text/plain; charset="UTF-8"': 'UTF-8',
-        'text/plain; charset="ISO-8859-1"': 'ISO-8859-1',
-        'text/plain; charset="windows-1252"': 'windows-1252',
-        'application/xhtml+xml; charset="UTF-8"': 'UTF-8',
-        'application/xhtml+xml; charset="ISO-8859-1"': 'ISO-8859-1',
-        'application/xhtml+xml; charset="windows-1252"': 'windows-1252',
-        'text/html; charset="US-ASCII"': 'US-ASCII',
-        'application/json; charset="US-ASCII"': 'US-ASCII',
-        'text/plain; charset="US-ASCII"': 'US-ASCII',
-        'text/html; charset="Shift_JIS"': 'Shift_JIS',
-        'application/json; charset="Shift_JIS"': 'Shift_JIS',
-        'text/plain; charset="Shift_JIS"': 'Shift_JIS',
-        'application/xml; charset="UTF-8"': 'UTF-8',
-        'application/xml; charset="ISO-8859-1"': 'ISO-8859-1',
-        'application/xml': 'utf-8',
-        'text/xml; charset="UTF-8"': 'UTF-8',
-        'text/xml; charset="ISO-8859-1"': 'ISO-8859-1',
-        'text/xml': 'utf-8'
-    }
-
-
-@pytest.fixture
-def status_map():
-    return {
-        100: "Continue", 101: "Switching Protocols", 102: "Processing", 103: "Early Hints",
-        200: "OK", 201: "Created", 202: "Accepted", 203: "Non-Authoritative Information",
-        204: "No Content", 205: "Reset Content", 206: "Partial Content", 207: "Multi-Status",
-        208: "Already Reported", 226: "IM Used", 300: "Multiple Choices",
-        301: "Moved Permanently", 302: "Found", 303: "See Other", 304: "Not Modified",
-        305: "Use Proxy", 307: "Temporary Redirect", 308: "Permanent Redirect",
-        400: "Bad Request", 401: "Unauthorized", 402: "Payment Required", 403: "Forbidden",
-        404: "Not Found", 405: "Method Not Allowed", 406: "Not Acceptable",
-        407: "Proxy Authentication Required", 408: "Request Timeout", 409: "Conflict",
-        410: "Gone", 411: "Length Required", 412: "Precondition Failed",
-        413: "Payload Too Large", 414: "URI Too Long", 415: "Unsupported Media Type",
-        416: "Range Not Satisfiable", 417: "Expectation Failed", 418: "I'm a teapot",
-        421: "Misdirected Request", 422: "Unprocessable Entity", 423: "Locked",
-        424: "Failed Dependency", 425: "Too Early", 426: "Upgrade Required",
-        428: "Precondition Required", 429: "Too Many Requests",
-        431: "Request Header Fields Too Large", 451: "Unavailable For Legal Reasons",
-        500: "Internal Server Error", 501: "Not Implemented", 502: "Bad Gateway",
-        503: "Service Unavailable", 504: "Gateway Timeout",
-        505: "HTTP Version Not Supported", 506: "Variant Also Negotiates",
-        507: "Insufficient Storage", 508: "Loop Detected", 510: "Not Extended",
-        511: "Network Authentication Required"
-    }
-
-
-def test_parsing_content_type(content_type_map):
-    """Test if parsing different types of content-type returns the expected result"""
-    for header_value, expected_encoding in content_type_map.items():
-        assert ResponseEncoding.get_value(header_value) == expected_encoding
-
-
-def test_parsing_response_status(status_map):
-    """Test if using different http responses' status codes returns the expected result"""
-    for status_code, expected_status_text in status_map.items():
-        assert StatusText.get(status_code) == expected_status_text
-
-
-def test_unknown_status_code():
-    """Test handling of an unknown status code"""
-    assert StatusText.get(1000) == "Unknown Status Code"

tests/parser/test_automatch.py

@@ -1,111 +0,0 @@
-import asyncio
-
-import pytest
-
-from scrapling import Adaptor
-
-
-class TestParserAutoMatch:
-    def test_element_relocation(self):
-        """Test relocating element after structure change"""
-        original_html = '''
-                <div class="container">
-                    <section class="products">
-                        <article class="product" id="p1">
-                            <h3>Product 1</h3>
-                            <p class="description">Description 1</p>
-                        </article>
-                        <article class="product" id="p2">
-                            <h3>Product 2</h3>
-                            <p class="description">Description 2</p>
-                        </article>
-                    </section>
-                </div>
-                '''
-        changed_html = '''
-                <div class="new-container">
-                    <div class="product-wrapper">
-                        <section class="products">
-                            <article class="product new-class" data-id="p1">
-                                <div class="product-info">
-                                    <h3>Product 1</h3>
-                                    <p class="new-description">Description 1</p>
-                                </div>
-                            </article>
-                            <article class="product new-class" data-id="p2">
-                                <div class="product-info">
-                                    <h3>Product 2</h3>
-                                    <p class="new-description">Description 2</p>
-                                </div>
-                            </article>
-                        </section>
-                    </div>
-                </div>
-                '''
-
-        old_page = Adaptor(original_html, url='example.com', auto_match=True)
-        new_page = Adaptor(changed_html, url='example.com', auto_match=True)
-
-        # 'p1' was used as ID and now it's not and all the path elements have changes
-        # Also at the same time testing auto-match vs combined selectors
-        _ = old_page.css('#p1, #p2', auto_save=True)[0]
-        relocated = new_page.css('#p1', auto_match=True)
-
-        assert relocated is not None
-        assert relocated[0].attrib['data-id'] == 'p1'
-        assert relocated[0].has_class('new-class')
-        assert relocated[0].css('.new-description')[0].text == 'Description 1'
-
-    @pytest.mark.asyncio
-    async def test_element_relocation_async(self):
-        """Test relocating element after structure change in async mode"""
-        original_html = '''
-                <div class="container">
-                    <section class="products">
-                        <article class="product" id="p1">
-                            <h3>Product 1</h3>
-                            <p class="description">Description 1</p>
-                        </article>
-                        <article class="product" id="p2">
-                            <h3>Product 2</h3>
-                            <p class="description">Description 2</p>
-                        </article>
-                    </section>
-                </div>
-                '''
-        changed_html = '''
-                <div class="new-container">
-                    <div class="product-wrapper">
-                        <section class="products">
-                            <article class="product new-class" data-id="p1">
-                                <div class="product-info">
-                                    <h3>Product 1</h3>
-                                    <p class="new-description">Description 1</p>
-                                </div>
-                            </article>
-                            <article class="product new-class" data-id="p2">
-                                <div class="product-info">
-                                    <h3>Product 2</h3>
-                                    <p class="new-description">Description 2</p>
-                                </div>
-                            </article>
-                        </section>
-                    </div>
-                </div>
-                '''
-
-        # Simulate async operation
-        await asyncio.sleep(0.1)  # Minimal async operation
-
-        old_page = Adaptor(original_html, url='example.com', auto_match=True)
-        new_page = Adaptor(changed_html, url='example.com', auto_match=True)
-
-        # 'p1' was used as ID and now it's not and all the path elements have changes
-        # Also at the same time testing auto-match vs combined selectors
-        _ = old_page.css('#p1, #p2', auto_save=True)[0]
-        relocated = new_page.css('#p1', auto_match=True)
-
-        assert relocated is not None
-        assert relocated[0].attrib['data-id'] == 'p1'
-        assert relocated[0].has_class('new-class')
-        assert relocated[0].css('.new-description')[0].text == 'Description 1'

tests/parser/test_general.py

@@ -1,330 +0,0 @@
-import pickle
-import time
-
-import pytest
-from cssselect import SelectorError, SelectorSyntaxError
-
-from scrapling import Adaptor
-
-
-@pytest.fixture
-def html_content():
-    return '''
-    <html>
-    <head>
-        <title>Complex Web Page</title>
-        <style>
-            .hidden { display: none; }
-        </style>
-    </head>
-    <body>
-        <header>
-            <nav>
-                <ul>
-                    <li><a href="#home">Home</a></li>
-                    <li><a href="#about">About</a></li>
-                    <li><a href="#contact">Contact</a></li>
-                </ul>
-            </nav>
-        </header>
-        <main>
-            <section id="products" schema='{"jsonable": "data"}'>
-                <h2>Products</h2>
-                <div class="product-list">
-                    <article class="product" data-id="1">
-                        <h3>Product 1</h3>
-                        <p class="description">This is product 1</p>
-                        <span class="price">$10.99</span>
-                        <div class="hidden stock">In stock: 5</div>
-                    </article>
-                    <article class="product" data-id="2">
-                        <h3>Product 2</h3>
-                        <p class="description">This is product 2</p>
-                        <span class="price">$20.99</span>
-                        <div class="hidden stock">In stock: 3</div>
-                    </article>
-                    <article class="product" data-id="3">
-                        <h3>Product 3</h3>
-                        <p class="description">This is product 3</p>
-                        <span class="price">$15.99</span>
-                        <div class="hidden stock">Out of stock</div>
-                    </article>
-                </div>
-            </section>
-            <section id="reviews">
-                <h2>Customer Reviews</h2>
-                <div class="review-list">
-                    <div class="review" data-rating="5">
-                        <p class="review-text">Great product!</p>
-                        <span class="reviewer">John Doe</span>
-                    </div>
-                    <div class="review" data-rating="4">
-                        <p class="review-text">Good value for money.</p>
-                        <span class="reviewer">Jane Smith</span>
-                    </div>
-                </div>
-            </section>
-        </main>
-        <footer>
-            <p>&copy; 2024 Our Company</p>
-        </footer>
-        <script id="page-data" type="application/json">
-            {"lastUpdated": "2024-09-22T10:30:00Z", "totalProducts": 3}
-        </script>
-    </body>
-    </html>
-    '''
-
-
-@pytest.fixture
-def page(html_content):
-    return Adaptor(html_content, auto_match=False)
-
-
-# CSS Selector Tests
-class TestCSSSelectors:
-    def test_basic_product_selection(self, page):
-        """Test selecting all product elements"""
-        elements = page.css('main #products .product-list article.product')
-        assert len(elements) == 3
-
-    def test_in_stock_product_selection(self, page):
-        """Test selecting in-stock products"""
-        in_stock_products = page.css(
-            'main #products .product-list article.product:not(:contains("Out of stock"))')
-        assert len(in_stock_products) == 2
-
-
-# XPath Selector Tests
-class TestXPathSelectors:
-    def test_high_rating_reviews(self, page):
-        """Test selecting reviews with high ratings"""
-        reviews = page.xpath(
-            '//section[@id="reviews"]//div[contains(@class, "review") and @data-rating >= 4]'
-        )
-        assert len(reviews) == 2
-
-    def test_high_priced_products(self, page):
-        """Test selecting products above a certain price"""
-        high_priced_products = page.xpath(
-            '//article[contains(@class, "product")]'
-            '[number(translate(substring-after(.//span[@class="price"], "$"), ",", "")) > 15]'
-        )
-        assert len(high_priced_products) == 2
-
-
-# Text Matching Tests
-class TestTextMatching:
-    def test_regex_multiple_matches(self, page):
-        """Test finding multiple matches with regex"""
-        stock_info = page.find_by_regex(r'In stock: \d+', first_match=False)
-        assert len(stock_info) == 2
-
-    def test_regex_first_match(self, page):
-        """Test finding the first match with regex"""
-        stock_info = page.find_by_regex(r'In stock: \d+', first_match=True, case_sensitive=True)
-        assert stock_info.text == 'In stock: 5'
-
-    def test_partial_text_match(self, page):
-        """Test finding elements with partial text match"""
-        stock_info = page.find_by_text(r'In stock:', partial=True, first_match=False)
-        assert len(stock_info) == 2
-
-    def test_exact_text_match(self, page):
-        """Test finding elements with exact text match"""
-        out_of_stock = page.find_by_text('Out of stock', partial=False, first_match=False)
-        assert len(out_of_stock) == 1
-
-
-# Similar Elements Tests
-class TestSimilarElements:
-    def test_finding_similar_products(self, page):
-        """Test finding similar product elements"""
-        first_product = page.css_first('.product')
-        similar_products = first_product.find_similar()
-        assert len(similar_products) == 2
-
-    def test_finding_similar_reviews(self, page):
-        """Test finding similar review elements with additional filtering"""
-        first_review = page.find('div', class_='review')
-        similar_high_rated_reviews = [
-            review
-            for review in first_review.find_similar()
-            if int(review.attrib.get('data-rating', 0)) >= 4
-        ]
-        assert len(similar_high_rated_reviews) == 1
-
-
-# Error Handling Tests
-class TestErrorHandling:
-    def test_invalid_adaptor_initialization(self):
-        """Test various invalid Adaptor initializations"""
-        # No arguments
-        with pytest.raises(ValueError):
-            _ = Adaptor(auto_match=False)
-
-        # Invalid argument types
-        with pytest.raises(TypeError):
-            _ = Adaptor(root="ayo", auto_match=False)
-
-        with pytest.raises(TypeError):
-            _ = Adaptor(text=1, auto_match=False)
-
-        with pytest.raises(TypeError):
-            _ = Adaptor(body=1, auto_match=False)
-
-    def test_invalid_storage(self, page, html_content):
-        """Test invalid storage parameter"""
-        with pytest.raises(ValueError):
-            _ = Adaptor(html_content, storage=object, auto_match=True)
-
-    def test_bad_selectors(self, page):
-        """Test handling of invalid selectors"""
-        with pytest.raises((SelectorError, SelectorSyntaxError)):
-            page.css('4 ayo')
-
-        with pytest.raises((SelectorError, SelectorSyntaxError)):
-            page.xpath('4 ayo')
-
-
-# Pickling and Object Representation Tests
-class TestPicklingAndRepresentation:
-    def test_unpickleable_objects(self, page):
-        """Test that Adaptor objects cannot be pickled"""
-        table = page.css('.product-list')[0]
-        with pytest.raises(TypeError):
-            pickle.dumps(table)
-
-        with pytest.raises(TypeError):
-            pickle.dumps(table[0])
-
-    def test_string_representations(self, page):
-        """Test custom string representations of objects"""
-        table = page.css('.product-list')[0]
-        assert issubclass(type(table.__str__()), str)
-        assert issubclass(type(table.__repr__()), str)
-        assert issubclass(type(table.attrib.__str__()), str)
-        assert issubclass(type(table.attrib.__repr__()), str)
-
-
-# Navigation and Traversal Tests
-class TestElementNavigation:
-    def test_basic_navigation_properties(self, page):
-        """Test basic navigation properties of elements"""
-        table = page.css('.product-list')[0]
-        assert table.path is not None
-        assert table.html_content != ''
-        assert table.prettify() != ''
-
-    def test_parent_and_sibling_navigation(self, page):
-        """Test parent and sibling navigation"""
-        table = page.css('.product-list')[0]
-        parent = table.parent
-        assert parent.attrib['id'] == 'products'
-
-        parent_siblings = parent.siblings
-        assert len(parent_siblings) == 1
-
-    def test_child_navigation(self, page):
-        """Test child navigation"""
-        table = page.css('.product-list')[0]
-        children = table.children
-        assert len(children) == 3
-
-    def test_next_and_previous_navigation(self, page):
-        """Test next and previous element navigation"""
-        child = page.css('.product-list')[0].find({'data-id': "1"})
-        next_element = child.next
-        assert next_element.attrib['data-id'] == '2'
-
-        prev_element = next_element.previous
-        assert prev_element.tag == child.tag
-
-    def test_ancestor_finding(self, page):
-        """Test finding ancestors of elements"""
-        all_prices = page.css('.price')
-        products_with_prices = [
-            price.find_ancestor(lambda p: p.has_class('product'))
-            for price in all_prices
-        ]
-        assert len(products_with_prices) == 3
-
-
-# JSON and Attribute Tests
-class TestJSONAndAttributes:
-    def test_json_conversion(self, page):
-        """Test converting content to JSON"""
-        script_content = page.css('#page-data::text')[0]
-        assert issubclass(type(script_content.sort()), str)
-        page_data = script_content.json()
-        assert page_data['totalProducts'] == 3
-        assert 'lastUpdated' in page_data
-
-    def test_attribute_operations(self, page):
-        """Test various attribute-related operations"""
-        # Product ID extraction
-        products = page.css('.product')
-        product_ids = [product.attrib['data-id'] for product in products]
-        assert product_ids == ['1', '2', '3']
-        assert 'data-id' in products[0].attrib
-
-        # Review rating calculations
-        reviews = page.css('.review')
-        review_ratings = [int(review.attrib['data-rating']) for review in reviews]
-        assert sum(review_ratings) / len(review_ratings) == 4.5
-
-        # Attribute searching
-        key_value = list(products[0].attrib.search_values('1', partial=False))
-        assert list(key_value[0].keys()) == ['data-id']
-
-        key_value = list(products[0].attrib.search_values('1', partial=True))
-        assert list(key_value[0].keys()) == ['data-id']
-
-        # JSON attribute conversion
-        attr_json = page.css_first('#products').attrib['schema'].json()
-        assert attr_json == {'jsonable': 'data'}
-        assert isinstance(page.css('#products')[0].attrib.json_string, bytes)
-
-
-# Performance Test
-def test_large_html_parsing_performance():
-    """Test parsing and selecting performance on large HTML"""
-    large_html = '<html><body>' + '<div class="item">' * 5000 + '</div>' * 5000 + '</body></html>'
-
-    start_time = time.time()
-    parsed = Adaptor(large_html, auto_match=False)
-    elements = parsed.css('.item')
-    end_time = time.time()
-
-    assert len(elements) == 5000
-    # Converting 5000 elements to a class and doing operations on them will take time
-    # Based on my tests with 100 runs, 1 loop each Scrapling (given the extra work/features) takes 10.4ms on average
-    assert end_time - start_time < 0.5  # Locally I test on 0.1 but on GitHub actions with browsers and threading sometimes closing adds fractions of seconds
-
-
-# Selector Generation Test
-def test_selectors_generation(page):
-    """Try to create selectors for all elements in the page"""
-
-    def _traverse(element: Adaptor):
-        assert isinstance(element.generate_css_selector, str)
-        assert isinstance(element.generate_xpath_selector, str)
-        for branch in element.children:
-            _traverse(branch)
-
-    _traverse(page)
-
-
-# Miscellaneous Tests
-def test_getting_all_text(page):
-    """Test getting all text from the page"""
-    assert page.get_all_text() != ''
-
-
-def test_regex_on_text(page):
-    """Test regex operations on text"""
-    element = page.css('[data-id="1"] .price')[0]
-    match = element.re_first(r'[\.\d]+')
-    assert match == '10.99'
-    match = element.text.re(r'(\d+)', replace_entities=False)
-    assert len(match) == 2