scrapling 0.2.8__py3-none-any.whl → 0.2.9__py3-none-any.whl

tests/fetchers/async/test_playwright.py

@@ -0,0 +1,99 @@
+import pytest
+import pytest_httpbin
+
+from scrapling import PlayWrightFetcher
+
+
+@pytest_httpbin.use_class_based_httpbin
+class TestPlayWrightFetcherAsync:
+    @pytest.fixture
+    def fetcher(self):
+        return PlayWrightFetcher(auto_match=False)
+
+    @pytest.fixture
+    def urls(self, httpbin):
+        return {
+            'status_200': f'{httpbin.url}/status/200',
+            'status_404': f'{httpbin.url}/status/404',
+            'status_501': f'{httpbin.url}/status/501',
+            'basic_url': f'{httpbin.url}/get',
+            'html_url': f'{httpbin.url}/html',
+            'delayed_url': f'{httpbin.url}/delay/10',
+            'cookies_url': f"{httpbin.url}/cookies/set/test/value"
+        }
+
+    @pytest.mark.asyncio
+    async def test_basic_fetch(self, fetcher, urls):
+        """Test doing basic fetch request with multiple statuses"""
+        response = await fetcher.async_fetch(urls['status_200'])
+        assert response.status == 200
+
+    @pytest.mark.asyncio
+    async def test_networkidle(self, fetcher, urls):
+        """Test if waiting for `networkidle` make page does not finish loading or not"""
+        response = await fetcher.async_fetch(urls['basic_url'], network_idle=True)
+        assert response.status == 200
+
+    @pytest.mark.asyncio
+    async def test_blocking_resources(self, fetcher, urls):
+        """Test if blocking resources make page does not finish loading or not"""
+        response = await fetcher.async_fetch(urls['basic_url'], disable_resources=True)
+        assert response.status == 200
+
+    @pytest.mark.asyncio
+    async def test_waiting_selector(self, fetcher, urls):
+        """Test if waiting for a selector make page does not finish loading or not"""
+        response1 = await fetcher.async_fetch(urls['html_url'], wait_selector='h1')
+        assert response1.status == 200
+
+        response2 = await fetcher.async_fetch(urls['html_url'], wait_selector='h1', wait_selector_state='visible')
+        assert response2.status == 200
+
+    @pytest.mark.asyncio
+    async def test_cookies_loading(self, fetcher, urls):
+        """Test if cookies are set after the request"""
+        response = await fetcher.async_fetch(urls['cookies_url'])
+        assert response.cookies == {'test': 'value'}
+
+    @pytest.mark.asyncio
+    async def test_automation(self, fetcher, urls):
+        """Test if automation break the code or not"""
+        async def scroll_page(page):
+            await page.mouse.wheel(10, 0)
+            await page.mouse.move(100, 400)
+            await page.mouse.up()
+            return page
+
+        response = await fetcher.async_fetch(urls['html_url'], page_action=scroll_page)
+        assert response.status == 200
+
+    @pytest.mark.parametrize("kwargs", [
+        {"disable_webgl": True, "hide_canvas": False},
+        {"disable_webgl": False, "hide_canvas": True},
+        {"stealth": True},
+        {"useragent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0'},
+        {"extra_headers": {'ayo': ''}}
+    ])
+    @pytest.mark.asyncio
+    async def test_properties(self, fetcher, urls, kwargs):
+        """Test if different arguments breaks the code or not"""
+        response = await fetcher.async_fetch(urls['html_url'], **kwargs)
+        assert response.status == 200
+
+    @pytest.mark.asyncio
+    async def test_cdp_url_invalid(self, fetcher, urls):
+        """Test if invalid CDP URLs raise appropriate exceptions"""
+        with pytest.raises(ValueError):
+            await fetcher.async_fetch(urls['html_url'], cdp_url='blahblah')
+
+        with pytest.raises(ValueError):
+            await fetcher.async_fetch(urls['html_url'], cdp_url='blahblah', nstbrowser_mode=True)
+
+        with pytest.raises(Exception):
+            await fetcher.async_fetch(urls['html_url'], cdp_url='ws://blahblah')
+
+    @pytest.mark.asyncio
+    async def test_infinite_timeout(self, fetcher, urls):
+        """Test if infinite timeout breaks the code or not"""
+        response = await fetcher.async_fetch(urls['delayed_url'], timeout=None)
+        assert response.status == 200

tests/fetchers/sync/test_camoufox.py

@@ -0,0 +1,68 @@
+import pytest
+import pytest_httpbin
+
+from scrapling import StealthyFetcher
+
+
+@pytest_httpbin.use_class_based_httpbin
+class TestStealthyFetcher:
+    @pytest.fixture(scope="class")
+    def fetcher(self):
+        """Fixture to create a StealthyFetcher instance for the entire test class"""
+        return StealthyFetcher(auto_match=False)
+
+    @pytest.fixture(autouse=True)
+    def setup_urls(self, httpbin):
+        """Fixture to set up URLs for testing"""
+        self.status_200 = f'{httpbin.url}/status/200'
+        self.status_404 = f'{httpbin.url}/status/404'
+        self.status_501 = f'{httpbin.url}/status/501'
+        self.basic_url = f'{httpbin.url}/get'
+        self.html_url = f'{httpbin.url}/html'
+        self.delayed_url = f'{httpbin.url}/delay/10'  # 10 Seconds delay response
+        self.cookies_url = f"{httpbin.url}/cookies/set/test/value"
+
+    def test_basic_fetch(self, fetcher):
+        """Test doing basic fetch request with multiple statuses"""
+        assert fetcher.fetch(self.status_200).status == 200
+        assert fetcher.fetch(self.status_404).status == 404
+        assert fetcher.fetch(self.status_501).status == 501
+
+    def test_networkidle(self, fetcher):
+        """Test if waiting for `networkidle` make page does not finish loading or not"""
+        assert fetcher.fetch(self.basic_url, network_idle=True).status == 200
+
+    def test_blocking_resources(self, fetcher):
+        """Test if blocking resources make page does not finish loading or not"""
+        assert fetcher.fetch(self.basic_url, block_images=True).status == 200
+        assert fetcher.fetch(self.basic_url, disable_resources=True).status == 200
+
+    def test_waiting_selector(self, fetcher):
+        """Test if waiting for a selector make page does not finish loading or not"""
+        assert fetcher.fetch(self.html_url, wait_selector='h1').status == 200
+        assert fetcher.fetch(self.html_url, wait_selector='h1', wait_selector_state='visible').status == 200
+
+    def test_cookies_loading(self, fetcher):
+        """Test if cookies are set after the request"""
+        assert fetcher.fetch(self.cookies_url).cookies == {'test': 'value'}
+
+    def test_automation(self, fetcher):
+        """Test if automation break the code or not"""
+        def scroll_page(page):
+            page.mouse.wheel(10, 0)
+            page.mouse.move(100, 400)
+            page.mouse.up()
+            return page
+
+        assert fetcher.fetch(self.html_url, page_action=scroll_page).status == 200
+
+    def test_properties(self, fetcher):
+        """Test if different arguments breaks the code or not"""
+        assert fetcher.fetch(self.html_url, block_webrtc=True, allow_webgl=True).status == 200
+        assert fetcher.fetch(self.html_url, block_webrtc=False, allow_webgl=True).status == 200
+        assert fetcher.fetch(self.html_url, block_webrtc=True, allow_webgl=False).status == 200
+        assert fetcher.fetch(self.html_url, extra_headers={'ayo': ''}, os_randomize=True).status == 200
+
+    def test_infinite_timeout(self, fetcher):
+        """Test if infinite timeout breaks the code or not"""
+        assert fetcher.fetch(self.delayed_url, timeout=None).status == 200

tests/fetchers/sync/test_httpx.py

@@ -0,0 +1,82 @@
+import pytest
+import pytest_httpbin
+
+from scrapling import Fetcher
+
+
+@pytest_httpbin.use_class_based_httpbin
+class TestFetcher:
+    @pytest.fixture(scope="class")
+    def fetcher(self):
+        """Fixture to create a Fetcher instance for the entire test class"""
+        return Fetcher(auto_match=False)
+
+    @pytest.fixture(autouse=True)
+    def setup_urls(self, httpbin):
+        """Fixture to set up URLs for testing"""
+        self.status_200 = f'{httpbin.url}/status/200'
+        self.status_404 = f'{httpbin.url}/status/404'
+        self.status_501 = f'{httpbin.url}/status/501'
+        self.basic_url = f'{httpbin.url}/get'
+        self.post_url = f'{httpbin.url}/post'
+        self.put_url = f'{httpbin.url}/put'
+        self.delete_url = f'{httpbin.url}/delete'
+        self.html_url = f'{httpbin.url}/html'
+
+    def test_basic_get(self, fetcher):
+        """Test doing basic get request with multiple statuses"""
+        assert fetcher.get(self.status_200).status == 200
+        assert fetcher.get(self.status_404).status == 404
+        assert fetcher.get(self.status_501).status == 501
+
+    def test_get_properties(self, fetcher):
+        """Test if different arguments with GET request breaks the code or not"""
+        assert fetcher.get(self.status_200, stealthy_headers=True).status == 200
+        assert fetcher.get(self.status_200, follow_redirects=True).status == 200
+        assert fetcher.get(self.status_200, timeout=None).status == 200
+        assert fetcher.get(
+            self.status_200,
+            stealthy_headers=True,
+            follow_redirects=True,
+            timeout=None
+        ).status == 200
+
+    def test_post_properties(self, fetcher):
+        """Test if different arguments with POST request breaks the code or not"""
+        assert fetcher.post(self.post_url, data={'key': 'value'}).status == 200
+        assert fetcher.post(self.post_url, data={'key': 'value'}, stealthy_headers=True).status == 200
+        assert fetcher.post(self.post_url, data={'key': 'value'}, follow_redirects=True).status == 200
+        assert fetcher.post(self.post_url, data={'key': 'value'}, timeout=None).status == 200
+        assert fetcher.post(
+            self.post_url,
+            data={'key': 'value'},
+            stealthy_headers=True,
+            follow_redirects=True,
+            timeout=None
+        ).status == 200
+
+    def test_put_properties(self, fetcher):
+        """Test if different arguments with PUT request breaks the code or not"""
+        assert fetcher.put(self.put_url, data={'key': 'value'}).status == 200
+        assert fetcher.put(self.put_url, data={'key': 'value'}, stealthy_headers=True).status == 200
+        assert fetcher.put(self.put_url, data={'key': 'value'}, follow_redirects=True).status == 200
+        assert fetcher.put(self.put_url, data={'key': 'value'}, timeout=None).status == 200
+        assert fetcher.put(
+            self.put_url,
+            data={'key': 'value'},
+            stealthy_headers=True,
+            follow_redirects=True,
+            timeout=None
+        ).status == 200
+
+    def test_delete_properties(self, fetcher):
+        """Test if different arguments with DELETE request breaks the code or not"""
+        assert fetcher.delete(self.delete_url, stealthy_headers=True).status == 200
+        assert fetcher.delete(self.delete_url, follow_redirects=True).status == 200
+        assert fetcher.delete(self.delete_url, timeout=None).status == 200
+        assert fetcher.delete(
+            self.delete_url,
+            stealthy_headers=True,
+            follow_redirects=True,
+            timeout=None
+        ).status == 200

tests/fetchers/sync/test_playwright.py

@@ -0,0 +1,87 @@
+import pytest
+import pytest_httpbin
+
+from scrapling import PlayWrightFetcher
+
+
+@pytest_httpbin.use_class_based_httpbin
+class TestPlayWrightFetcher:
+
+    @pytest.fixture(scope="class")
+    def fetcher(self):
+        """Fixture to create a StealthyFetcher instance for the entire test class"""
+        return PlayWrightFetcher(auto_match=False)
+
+    @pytest.fixture(autouse=True)
+    def setup_urls(self, httpbin):
+        """Fixture to set up URLs for testing"""
+        self.status_200 = f'{httpbin.url}/status/200'
+        self.status_404 = f'{httpbin.url}/status/404'
+        self.status_501 = f'{httpbin.url}/status/501'
+        self.basic_url = f'{httpbin.url}/get'
+        self.html_url = f'{httpbin.url}/html'
+        self.delayed_url = f'{httpbin.url}/delay/10'  # 10 Seconds delay response
+        self.cookies_url = f"{httpbin.url}/cookies/set/test/value"
+
+    def test_basic_fetch(self, fetcher):
+        """Test doing basic fetch request with multiple statuses"""
+        assert fetcher.fetch(self.status_200).status == 200
+        # There's a bug with playwright makes it crashes if a URL returns status code 4xx/5xx without body, let's disable this till they reply to my issue report
+        # assert fetcher.fetch(self.status_404).status == 404
+        # assert fetcher.fetch(self.status_501).status == 501
+
+    def test_networkidle(self, fetcher):
+        """Test if waiting for `networkidle` make page does not finish loading or not"""
+        assert fetcher.fetch(self.basic_url, network_idle=True).status == 200
+
+    def test_blocking_resources(self, fetcher):
+        """Test if blocking resources make page does not finish loading or not"""
+        assert fetcher.fetch(self.basic_url, disable_resources=True).status == 200
+
+    def test_waiting_selector(self, fetcher):
+        """Test if waiting for a selector make page does not finish loading or not"""
+        assert fetcher.fetch(self.html_url, wait_selector='h1').status == 200
+        assert fetcher.fetch(self.html_url, wait_selector='h1', wait_selector_state='visible').status == 200
+
+    def test_cookies_loading(self, fetcher):
+        """Test if cookies are set after the request"""
+        assert fetcher.fetch(self.cookies_url).cookies == {'test': 'value'}
+
+    def test_automation(self, fetcher):
+        """Test if automation break the code or not"""
+
+        def scroll_page(page):
+            page.mouse.wheel(10, 0)
+            page.mouse.move(100, 400)
+            page.mouse.up()
+            return page
+
+        assert fetcher.fetch(self.html_url, page_action=scroll_page).status == 200
+
+    @pytest.mark.parametrize("kwargs", [
+        {"disable_webgl": True, "hide_canvas": False},
+        {"disable_webgl": False, "hide_canvas": True},
+        {"stealth": True},
+        {"useragent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0'},
+        {"extra_headers": {'ayo': ''}}
+    ])
+    def test_properties(self, fetcher, kwargs):
+        """Test if different arguments breaks the code or not"""
+        response = fetcher.fetch(self.html_url, **kwargs)
+        assert response.status == 200
+
+    def test_cdp_url_invalid(self, fetcher):
+        """Test if invalid CDP URLs raise appropriate exceptions"""
+        with pytest.raises(ValueError):
+            fetcher.fetch(self.html_url, cdp_url='blahblah')
+
+        with pytest.raises(ValueError):
+            fetcher.fetch(self.html_url, cdp_url='blahblah', nstbrowser_mode=True)
+
+        with pytest.raises(Exception):
+            fetcher.fetch(self.html_url, cdp_url='ws://blahblah')
+
+    def test_infinite_timeout(self, fetcher, ):
+        """Test if infinite timeout breaks the code or not"""
+        response = fetcher.fetch(self.delayed_url, timeout=None)
+        assert response.status == 200

tests/fetchers/test_utils.py

@@ -1,129 +1,97 @@
-import unittest
+import pytest
 
 from scrapling.engines.toolbelt.custom import ResponseEncoding, StatusText
 
 
-class TestPlayWrightFetcher(unittest.TestCase):
-    def setUp(self):
-        self.content_type_map = {
-            # A map generated by ChatGPT for most possible `content_type` values and the expected outcome
-            'text/html; charset=UTF-8': 'UTF-8',
-            'text/html; charset=ISO-8859-1': 'ISO-8859-1',
-            'text/html': 'ISO-8859-1',
-            'application/json; charset=UTF-8': 'UTF-8',
-            'application/json': 'utf-8',
-            'text/json': 'utf-8',
-            'application/javascript; charset=UTF-8': 'UTF-8',
-            'application/javascript': 'utf-8',
-            'text/plain; charset=UTF-8': 'UTF-8',
-            'text/plain; charset=ISO-8859-1': 'ISO-8859-1',
-            'text/plain': 'ISO-8859-1',
-            'application/xhtml+xml; charset=UTF-8': 'UTF-8',
-            'application/xhtml+xml': 'utf-8',
-            'text/html; charset=windows-1252': 'windows-1252',
-            'application/json; charset=windows-1252': 'windows-1252',
-            'text/plain; charset=windows-1252': 'windows-1252',
-            'text/html; charset="UTF-8"': 'UTF-8',
-            'text/html; charset="ISO-8859-1"': 'ISO-8859-1',
-            'text/html; charset="windows-1252"': 'windows-1252',
-            'application/json; charset="UTF-8"': 'UTF-8',
-            'application/json; charset="ISO-8859-1"': 'ISO-8859-1',
-            'application/json; charset="windows-1252"': 'windows-1252',
-            'text/json; charset="UTF-8"': 'UTF-8',
-            'application/javascript; charset="UTF-8"': 'UTF-8',
-            'application/javascript; charset="ISO-8859-1"': 'ISO-8859-1',
-            'text/plain; charset="UTF-8"': 'UTF-8',
-            'text/plain; charset="ISO-8859-1"': 'ISO-8859-1',
-            'text/plain; charset="windows-1252"': 'windows-1252',
-            'application/xhtml+xml; charset="UTF-8"': 'UTF-8',
-            'application/xhtml+xml; charset="ISO-8859-1"': 'ISO-8859-1',
-            'application/xhtml+xml; charset="windows-1252"': 'windows-1252',
-            'text/html; charset="US-ASCII"': 'US-ASCII',
-            'application/json; charset="US-ASCII"': 'US-ASCII',
-            'text/plain; charset="US-ASCII"': 'US-ASCII',
-            'text/html; charset="Shift_JIS"': 'Shift_JIS',
-            'application/json; charset="Shift_JIS"': 'Shift_JIS',
-            'text/plain; charset="Shift_JIS"': 'Shift_JIS',
-            'application/xml; charset="UTF-8"': 'UTF-8',
-            'application/xml; charset="ISO-8859-1"': 'ISO-8859-1',
-            'application/xml': 'utf-8',
-            'text/xml; charset="UTF-8"': 'UTF-8',
-            'text/xml; charset="ISO-8859-1"': 'ISO-8859-1',
-            'text/xml': 'utf-8'
-        }
-        self.status_map = {
-            100: "Continue",
-            101: "Switching Protocols",
-            102: "Processing",
-            103: "Early Hints",
-            200: "OK",
-            201: "Created",
-            202: "Accepted",
-            203: "Non-Authoritative Information",
-            204: "No Content",
-            205: "Reset Content",
-            206: "Partial Content",
-            207: "Multi-Status",
-            208: "Already Reported",
-            226: "IM Used",
-            300: "Multiple Choices",
-            301: "Moved Permanently",
-            302: "Found",
-            303: "See Other",
-            304: "Not Modified",
-            305: "Use Proxy",
-            307: "Temporary Redirect",
-            308: "Permanent Redirect",
-            400: "Bad Request",
-            401: "Unauthorized",
-            402: "Payment Required",
-            403: "Forbidden",
-            404: "Not Found",
-            405: "Method Not Allowed",
-            406: "Not Acceptable",
-            407: "Proxy Authentication Required",
-            408: "Request Timeout",
-            409: "Conflict",
-            410: "Gone",
-            411: "Length Required",
-            412: "Precondition Failed",
-            413: "Payload Too Large",
-            414: "URI Too Long",
-            415: "Unsupported Media Type",
-            416: "Range Not Satisfiable",
-            417: "Expectation Failed",
-            418: "I'm a teapot",
-            421: "Misdirected Request",
-            422: "Unprocessable Entity",
-            423: "Locked",
-            424: "Failed Dependency",
-            425: "Too Early",
-            426: "Upgrade Required",
-            428: "Precondition Required",
-            429: "Too Many Requests",
-            431: "Request Header Fields Too Large",
-            451: "Unavailable For Legal Reasons",
-            500: "Internal Server Error",
-            501: "Not Implemented",
-            502: "Bad Gateway",
-            503: "Service Unavailable",
-            504: "Gateway Timeout",
-            505: "HTTP Version Not Supported",
-            506: "Variant Also Negotiates",
-            507: "Insufficient Storage",
-            508: "Loop Detected",
-            510: "Not Extended",
-            511: "Network Authentication Required"
-        }
+@pytest.fixture
+def content_type_map():
+    return {
+        # A map generated by ChatGPT for most possible `content_type` values and the expected outcome
+        'text/html; charset=UTF-8': 'UTF-8',
+        'text/html; charset=ISO-8859-1': 'ISO-8859-1',
+        'text/html': 'ISO-8859-1',
+        'application/json; charset=UTF-8': 'UTF-8',
+        'application/json': 'utf-8',
+        'text/json': 'utf-8',
+        'application/javascript; charset=UTF-8': 'UTF-8',
+        'application/javascript': 'utf-8',
+        'text/plain; charset=UTF-8': 'UTF-8',
+        'text/plain; charset=ISO-8859-1': 'ISO-8859-1',
+        'text/plain': 'ISO-8859-1',
+        'application/xhtml+xml; charset=UTF-8': 'UTF-8',
+        'application/xhtml+xml': 'utf-8',
+        'text/html; charset=windows-1252': 'windows-1252',
+        'application/json; charset=windows-1252': 'windows-1252',
+        'text/plain; charset=windows-1252': 'windows-1252',
+        'text/html; charset="UTF-8"': 'UTF-8',
+        'text/html; charset="ISO-8859-1"': 'ISO-8859-1',
+        'text/html; charset="windows-1252"': 'windows-1252',
+        'application/json; charset="UTF-8"': 'UTF-8',
+        'application/json; charset="ISO-8859-1"': 'ISO-8859-1',
+        'application/json; charset="windows-1252"': 'windows-1252',
+        'text/json; charset="UTF-8"': 'UTF-8',
+        'application/javascript; charset="UTF-8"': 'UTF-8',
+        'application/javascript; charset="ISO-8859-1"': 'ISO-8859-1',
+        'text/plain; charset="UTF-8"': 'UTF-8',
+        'text/plain; charset="ISO-8859-1"': 'ISO-8859-1',
+        'text/plain; charset="windows-1252"': 'windows-1252',
+        'application/xhtml+xml; charset="UTF-8"': 'UTF-8',
+        'application/xhtml+xml; charset="ISO-8859-1"': 'ISO-8859-1',
+        'application/xhtml+xml; charset="windows-1252"': 'windows-1252',
+        'text/html; charset="US-ASCII"': 'US-ASCII',
+        'application/json; charset="US-ASCII"': 'US-ASCII',
+        'text/plain; charset="US-ASCII"': 'US-ASCII',
+        'text/html; charset="Shift_JIS"': 'Shift_JIS',
+        'application/json; charset="Shift_JIS"': 'Shift_JIS',
+        'text/plain; charset="Shift_JIS"': 'Shift_JIS',
+        'application/xml; charset="UTF-8"': 'UTF-8',
+        'application/xml; charset="ISO-8859-1"': 'ISO-8859-1',
+        'application/xml': 'utf-8',
+        'text/xml; charset="UTF-8"': 'UTF-8',
+        'text/xml; charset="ISO-8859-1"': 'ISO-8859-1',
+        'text/xml': 'utf-8'
+    }
 
-    def test_parsing_content_type(self):
-        """Test if parsing different types of content-type returns the expected result"""
-        for header_value, expected_encoding in self.content_type_map.items():
-            self.assertEqual(ResponseEncoding.get_value(header_value), expected_encoding)
 
-    def test_parsing_response_status(self):
-        """Test if using different http responses' status codes returns the expected result"""
-        for status_code, expected_status_text in self.status_map.items():
-            self.assertEqual(StatusText.get(status_code), expected_status_text)
+@pytest.fixture
+def status_map():
+    return {
+        100: "Continue", 101: "Switching Protocols", 102: "Processing", 103: "Early Hints",
+        200: "OK", 201: "Created", 202: "Accepted", 203: "Non-Authoritative Information",
+        204: "No Content", 205: "Reset Content", 206: "Partial Content", 207: "Multi-Status",
+        208: "Already Reported", 226: "IM Used", 300: "Multiple Choices",
+        301: "Moved Permanently", 302: "Found", 303: "See Other", 304: "Not Modified",
+        305: "Use Proxy", 307: "Temporary Redirect", 308: "Permanent Redirect",
+        400: "Bad Request", 401: "Unauthorized", 402: "Payment Required", 403: "Forbidden",
+        404: "Not Found", 405: "Method Not Allowed", 406: "Not Acceptable",
+        407: "Proxy Authentication Required", 408: "Request Timeout", 409: "Conflict",
+        410: "Gone", 411: "Length Required", 412: "Precondition Failed",
+        413: "Payload Too Large", 414: "URI Too Long", 415: "Unsupported Media Type",
+        416: "Range Not Satisfiable", 417: "Expectation Failed", 418: "I'm a teapot",
+        421: "Misdirected Request", 422: "Unprocessable Entity", 423: "Locked",
+        424: "Failed Dependency", 425: "Too Early", 426: "Upgrade Required",
+        428: "Precondition Required", 429: "Too Many Requests",
+        431: "Request Header Fields Too Large", 451: "Unavailable For Legal Reasons",
+        500: "Internal Server Error", 501: "Not Implemented", 502: "Bad Gateway",
+        503: "Service Unavailable", 504: "Gateway Timeout",
+        505: "HTTP Version Not Supported", 506: "Variant Also Negotiates",
+        507: "Insufficient Storage", 508: "Loop Detected", 510: "Not Extended",
+        511: "Network Authentication Required"
+    }
 
-        self.assertEqual(StatusText.get(1000), "Unknown Status Code")
+
+def test_parsing_content_type(content_type_map):
+    """Test if parsing different types of content-type returns the expected result"""
+    for header_value, expected_encoding in content_type_map.items():
+        assert ResponseEncoding.get_value(header_value) == expected_encoding
+
+
+def test_parsing_response_status(status_map):
+    """Test if using different http responses' status codes returns the expected result"""
+    for status_code, expected_status_text in status_map.items():
+        assert StatusText.get(status_code) == expected_status_text
+
+
+def test_unknown_status_code():
+    """Test handling of an unknown status code"""
+    assert StatusText.get(1000) == "Unknown Status Code"

tests/parser/test_automatch.py

@@ -1,10 +1,11 @@
-import unittest
+import asyncio
 
-from scrapling import Adaptor
+import pytest
 
+from scrapling import Adaptor
 
-class TestParserAutoMatch(unittest.TestCase):
 
+class TestParserAutoMatch:
     def test_element_relocation(self):
         """Test relocating element after structure change"""
         original_html = '''
@@ -42,15 +43,69 @@ class TestParserAutoMatch(unittest.TestCase):
                 </div>
                 '''
 
-        old_page = Adaptor(original_html, url='example.com', auto_match=True, debug=True)
-        new_page = Adaptor(changed_html, url='example.com', auto_match=True, debug=True)
+        old_page = Adaptor(original_html, url='example.com', auto_match=True)
+        new_page = Adaptor(changed_html, url='example.com', auto_match=True)
+
+        # 'p1' was used as ID and now it's not and all the path elements have changes
+        # Also at the same time testing auto-match vs combined selectors
+        _ = old_page.css('#p1, #p2', auto_save=True)[0]
+        relocated = new_page.css('#p1', auto_match=True)
+
+        assert relocated is not None
+        assert relocated[0].attrib['data-id'] == 'p1'
+        assert relocated[0].has_class('new-class')
+        assert relocated[0].css('.new-description')[0].text == 'Description 1'
+
+    @pytest.mark.asyncio
+    async def test_element_relocation_async(self):
+        """Test relocating element after structure change in async mode"""
+        original_html = '''
+                <div class="container">
+                    <section class="products">
+                        <article class="product" id="p1">
+                            <h3>Product 1</h3>
+                            <p class="description">Description 1</p>
+                        </article>
+                        <article class="product" id="p2">
+                            <h3>Product 2</h3>
+                            <p class="description">Description 2</p>
+                        </article>
+                    </section>
+                </div>
+                '''
+        changed_html = '''
+                <div class="new-container">
+                    <div class="product-wrapper">
+                        <section class="products">
+                            <article class="product new-class" data-id="p1">
+                                <div class="product-info">
+                                    <h3>Product 1</h3>
+                                    <p class="new-description">Description 1</p>
+                                </div>
+                            </article>
+                            <article class="product new-class" data-id="p2">
+                                <div class="product-info">
+                                    <h3>Product 2</h3>
+                                    <p class="new-description">Description 2</p>
+                                </div>
+                            </article>
+                        </section>
+                    </div>
+                </div>
+                '''
+
+        # Simulate async operation
+        await asyncio.sleep(0.1)  # Minimal async operation
+
+        old_page = Adaptor(original_html, url='example.com', auto_match=True)
+        new_page = Adaptor(changed_html, url='example.com', auto_match=True)
 
         # 'p1' was used as ID and now it's not and all the path elements have changes
         # Also at the same time testing auto-match vs combined selectors
         _ = old_page.css('#p1, #p2', auto_save=True)[0]
         relocated = new_page.css('#p1', auto_match=True)
 
-        self.assertIsNotNone(relocated)
-        self.assertEqual(relocated[0].attrib['data-id'], 'p1')
-        self.assertTrue(relocated[0].has_class('new-class'))
-        self.assertEqual(relocated[0].css('.new-description')[0].text, 'Description 1')
+        assert relocated is not None
+        assert relocated[0].attrib['data-id'] == 'p1'
+        assert relocated[0].has_class('new-class')
+        assert relocated[0].css('.new-description')[0].text == 'Description 1'