scrapling 0.2.4__py3-none-any.whl → 0.2.6__py3-none-any.whl

scrapling/__init__.py

@@ -4,7 +4,7 @@ from scrapling.parser import Adaptor, Adaptors
 from scrapling.core.custom_types import TextHandler, AttributesHandler
 
 __author__ = "Karim Shoair (karim.shoair@pm.me)"
-__version__ = "0.2.4"
+__version__ = "0.2.6"
 __copyright__ = "Copyright (c) 2024 Karim Shoair"
 
 

scrapling/core/custom_types.py

@@ -129,9 +129,8 @@ class TextHandlers(List[TextHandler]):
 
 
 class AttributesHandler(Mapping):
-    """A read-only mapping to use instead of the standard dictionary for the speed boost but
-     at the same time I use it to add more functionalities.
-    If standard dictionary is needed, just convert this class to dictionary with `dict` function
+    """A read-only mapping to use instead of the standard dictionary for the speed boost but at the same time I use it to add more functionalities.
+        If standard dictionary is needed, just convert this class to dictionary with `dict` function
     """
     __slots__ = ('_data',)
 

scrapling/core/translator.py

@@ -1,9 +1,11 @@
 """
 Most of this file is adapted version of the translator of parsel library with some modifications simply for 1 important reason...
-To add pseudo-elements ``::text`` and ``::attr(ATTR_NAME)`` so we match Parsel/Scrapy selectors format
-which will be important in future releases but most importantly...
-    so you don't have to learn a new selectors/api method like what bs4 done with soupsieve :)
-> if you want to learn about this, head to https://cssselect.readthedocs.io/en/latest/#cssselect.FunctionalPseudoElement
+
+To add pseudo-elements ``::text`` and ``::attr(ATTR_NAME)`` so we match Parsel/Scrapy selectors format which will be important in future releases but most importantly...
+
+So you don't have to learn a new selectors/api method like what bs4 done with soupsieve :)
+
+    if you want to learn about this, head to https://cssselect.readthedocs.io/en/latest/#cssselect.FunctionalPseudoElement
 """
 
 import re

scrapling/engines/camo.py

@@ -104,13 +104,10 @@ class CamoufoxEngine:
 
             if self.wait_selector and type(self.wait_selector) is str:
                 waiter = page.locator(self.wait_selector)
-                waiter.wait_for(state=self.wait_selector_state)
+                waiter.first.wait_for(state=self.wait_selector_state)
 
-            content_type = res.headers.get('content-type', '')
-            # Parse charset from content-type
-            encoding = 'utf-8'  # default encoding
-            if 'charset=' in content_type.lower():
-                encoding = content_type.lower().split('charset=')[-1].split(';')[0].strip()
+            # This will be parsed inside `Response`
+            encoding = res.headers.get('content-type', '') or 'utf-8'  # default encoding
 
             status_text = res.status_text
             # PlayWright API sometimes give empty status text for some reason!

scrapling/engines/pw.py

@@ -27,11 +27,12 @@ class PlaywrightEngine:
             page_action: Callable = do_nothing,
             wait_selector: Optional[str] = None,
             wait_selector_state: Optional[str] = 'attached',
-            stealth: bool = False,
-            hide_canvas: bool = True,
-            disable_webgl: bool = False,
+            stealth: Optional[bool] = False,
+            real_chrome: Optional[bool] = False,
+            hide_canvas: Optional[bool] = False,
+            disable_webgl: Optional[bool] = False,
             cdp_url: Optional[str] = None,
-            nstbrowser_mode: bool = False,
+            nstbrowser_mode: Optional[bool] = False,
             nstbrowser_config: Optional[Dict] = None,
             google_search: Optional[bool] = True,
             extra_headers: Optional[Dict[str, str]] = None,
@@ -51,6 +52,7 @@ class PlaywrightEngine:
         :param wait_selector: Wait for a specific css selector to be in a specific state.
         :param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
         :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
+        :param real_chrome: If you have chrome browser installed on your device, enable this and the Fetcher will launch an instance of your browser and use it.
         :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
         :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
         :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.
@@ -67,6 +69,7 @@ class PlaywrightEngine:
         self.stealth = bool(stealth)
         self.hide_canvas = bool(hide_canvas)
         self.disable_webgl = bool(disable_webgl)
+        self.real_chrome = bool(real_chrome)
         self.google_search = bool(google_search)
         self.extra_headers = extra_headers or {}
         self.proxy = construct_proxy_dict(proxy)
@@ -119,7 +122,8 @@ class PlaywrightEngine:
         :param url: Target url.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
-        if not self.stealth:
+        if not self.stealth or self.real_chrome:
+            # Because rebrowser_playwright doesn't play well with real browsers
             from playwright.sync_api import sync_playwright
         else:
             from rebrowser_playwright.sync_api import sync_playwright
@@ -130,8 +134,8 @@ class PlaywrightEngine:
                 extra_headers = {}
                 useragent = self.useragent
             else:
-                extra_headers = generate_headers(browser_mode=True)
-                useragent = extra_headers.get('User-Agent')
+                extra_headers = {}
+                useragent = generate_headers(browser_mode=True).get('User-Agent')
 
             # Prepare the flags before diving
             flags = DEFAULT_STEALTH_FLAGS
@@ -146,9 +150,11 @@ class PlaywrightEngine:
                 browser = p.chromium.connect_over_cdp(endpoint_url=cdp_url)
             else:
                 if self.stealth:
-                    browser = p.chromium.launch(headless=self.headless, args=flags, ignore_default_args=['--enable-automation'], chromium_sandbox=True)
+                    browser = p.chromium.launch(
+                        headless=self.headless, args=flags, ignore_default_args=['--enable-automation'], chromium_sandbox=True, channel='chrome' if self.real_chrome else 'chromium'
+                    )
                 else:
-                    browser = p.chromium.launch(headless=self.headless, ignore_default_args=['--enable-automation'])
+                    browser = p.chromium.launch(headless=self.headless, ignore_default_args=['--enable-automation'], channel='chrome' if self.real_chrome else 'chromium')
 
             # Creating the context
             if self.stealth:
@@ -214,13 +220,10 @@ class PlaywrightEngine:
 
             if self.wait_selector and type(self.wait_selector) is str:
                 waiter = page.locator(self.wait_selector)
-                waiter.wait_for(state=self.wait_selector_state)
+                waiter.first.wait_for(state=self.wait_selector_state)
 
-            content_type = res.headers.get('content-type', '')
-            # Parse charset from content-type
-            encoding = 'utf-8'  # default encoding
-            if 'charset=' in content_type.lower():
-                encoding = content_type.lower().split('charset=')[-1].split(';')[0].strip()
+            # This will be parsed inside `Response`
+            encoding = res.headers.get('content-type', '') or 'utf-8'  # default encoding
 
             status_text = res.status_text
             # PlayWright API sometimes give empty status text for some reason!

scrapling/engines/static.py

@@ -23,7 +23,7 @@ class StaticEngine:
     @staticmethod
     def _headers_job(headers: Optional[Dict], url: str, stealth: bool) -> Dict:
         """Adds useragent to headers if it doesn't exist, generates real headers and append it to current headers, and
-        finally generates a referer header that looks like if this request came from Google's search of the current URL's domain.
+            finally generates a referer header that looks like if this request came from Google's search of the current URL's domain.
 
         :param headers: Current headers in the request if the user passed any
         :param url: The Target URL.
@@ -65,6 +65,7 @@ class StaticEngine:
 
     def get(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
         """Make basic HTTP GET request for you but with some added flavors.
+
         :param url: Target url.
         :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
             create a referer header as if this request had came from Google's search of this URL's domain.
@@ -77,6 +78,7 @@ class StaticEngine:
 
     def post(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
         """Make basic HTTP POST request for you but with some added flavors.
+
         :param url: Target url.
         :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
             create a referer header as if this request had came from Google's search of this URL's domain.
@@ -89,6 +91,7 @@ class StaticEngine:
 
     def delete(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
         """Make basic HTTP DELETE request for you but with some added flavors.
+
         :param url: Target url.
         :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
             create a referer header as if this request had came from Google's search of this URL's domain.
@@ -101,6 +104,7 @@ class StaticEngine:
 
     def put(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
         """Make basic HTTP PUT request for you but with some added flavors.
+
         :param url: Target url.
         :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
             create a referer header as if this request had came from Google's search of this URL's domain.

scrapling/engines/toolbelt/custom.py

@@ -3,11 +3,78 @@ Functions related to custom types or type checking
 """
 import inspect
 import logging
+from email.message import Message
 
 from scrapling.core.custom_types import MappingProxyType
 from scrapling.parser import Adaptor, SQLiteStorageSystem
 from scrapling.core.utils import setup_basic_logging, cache
-from scrapling.core._types import Any, List, Type, Union, Optional, Dict, Callable
+from scrapling.core._types import Any, List, Type, Union, Optional, Dict, Callable, Tuple
+
+
+class ResponseEncoding:
+    __DEFAULT_ENCODING = "utf-8"
+    __ISO_8859_1_CONTENT_TYPES = {"text/plain", "text/html", "text/css", "text/javascript"}
+
+    @classmethod
+    @cache(maxsize=None)
+    def __parse_content_type(cls, header_value: str) -> Tuple[str, Dict[str, str]]:
+        """Parse content type and parameters from a content-type header value.
+
+            Uses `email.message.Message` for robust header parsing according to RFC 2045.
+
+        :param header_value: Raw content-type header string
+        :return: Tuple of (content_type, parameters_dict)
+        """
+        # Create a Message object and set the Content-Type header then get the content type and parameters
+        msg = Message()
+        msg['content-type'] = header_value
+
+        content_type = msg.get_content_type()
+        params = dict(msg.get_params(failobj=[]))
+
+        # Remove the content-type from params if present somehow
+        params.pop('content-type', None)
+
+        return content_type, params
+
+    @classmethod
+    @cache(maxsize=None)
+    def get_value(cls, content_type: Optional[str]) -> str:
+        """Determine the appropriate character encoding from a content-type header.
+
+        The encoding is determined by these rules in order:
+            1. If no content-type is provided, use UTF-8
+            2. If charset parameter is present, use that encoding
+            3. If content-type is `text/*`, use ISO-8859-1 per HTTP/1.1 spec
+            4. If content-type is application/json, use UTF-8 per RFC 4627
+            5. Default to UTF-8 if nothing else matches
+
+        :param content_type: Content-Type header value or None
+        :return: String naming the character encoding
+        """
+        if not content_type:
+            return cls.__DEFAULT_ENCODING
+
+        try:
+            content_type, params = cls.__parse_content_type(content_type)
+
+            # First check for explicit charset parameter
+            if "charset" in params:
+                encoding = params["charset"].strip("'\"")
+                "test".encode(encoding)  # Validate encoding
+                return encoding
+
+            # Apply content-type specific rules
+            if content_type in cls.__ISO_8859_1_CONTENT_TYPES:
+                return "ISO-8859-1"
+
+            if content_type == "application/json":
+                return cls.__DEFAULT_ENCODING
+
+            return cls.__DEFAULT_ENCODING
+
+        except (ValueError, LookupError, UnicodeEncodeError):
+            return cls.__DEFAULT_ENCODING
 
 
 class Response(Adaptor):
@@ -20,6 +87,7 @@ class Response(Adaptor):
         self.cookies = cookies
         self.headers = headers
         self.request_headers = request_headers
+        encoding = ResponseEncoding.get_value(encoding)
         super().__init__(text=text, body=body, url=automatch_domain or url, encoding=encoding, **adaptor_arguments)
         # For back-ward compatibility
         self.adaptor = self

scrapling/engines/toolbelt/fingerprints.py

@@ -67,7 +67,7 @@ def generate_headers(browser_mode: bool = False) -> Dict:
         # So we don't raise any inconsistency red flags while websites fingerprinting us
         os_name = get_os_name()
         return HeaderGenerator(
-            browser=[Browser(name='chrome', min_version=128)],
+            browser=[Browser(name='chrome', min_version=130)],
             os=os_name,  # None is ignored
             device='desktop'
         ).generate()

scrapling/fetchers.py

@@ -11,6 +11,7 @@ class Fetcher(BaseFetcher):
     """
     def get(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
         """Make basic HTTP GET request for you but with some added flavors.
+
         :param url: Target url.
         :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
         :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
@@ -24,6 +25,7 @@ class Fetcher(BaseFetcher):
 
     def post(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
         """Make basic HTTP POST request for you but with some added flavors.
+
         :param url: Target url.
         :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
         :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
@@ -37,12 +39,14 @@ class Fetcher(BaseFetcher):
 
     def put(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
         """Make basic HTTP PUT request for you but with some added flavors.
+
         :param url: Target url
         :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
         :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
         :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
-        create a referer header as if this request came from Google's search of this URL's domain.
+            create a referer header as if this request came from Google's search of this URL's domain.
         :param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
+
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
         response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).put(url, stealthy_headers, **kwargs)
@@ -50,6 +54,7 @@ class Fetcher(BaseFetcher):
 
     def delete(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
         """Make basic HTTP DELETE request for you but with some added flavors.
+
         :param url: Target url
         :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
         :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
@@ -77,6 +82,7 @@ class StealthyFetcher(BaseFetcher):
     ) -> Response:
         """
         Opens up a browser and do your request based on your chosen options below.
+
         :param url: Target url.
         :param headless: Run the browser in headless/hidden (default), 'virtual' screen mode, or headful/visible mode.
         :param block_images: Prevent the loading of images through Firefox preferences.
@@ -127,26 +133,28 @@ class PlayWrightFetcher(BaseFetcher):
      Using this Fetcher class, you can do requests with:
         - Vanilla Playwright without any modifications other than the ones you chose.
         - Stealthy Playwright with the stealth mode I wrote for it. It's still a work in progress but it bypasses many online tests like bot.sannysoft.com
-        Some of the things stealth mode does include:
-            1) Patches the CDP runtime fingerprint.
-            2) Mimics some of the real browsers' properties by injecting several JS files and using custom options.
-            3) Using custom flags on launch to hide Playwright even more and make it faster.
-            4) Generates real browser's headers of the same type and same user OS then append it to the request.
-        - Real browsers by passing the CDP URL of your browser to be controlled by the Fetcher and most of the options can be enabled on it.
+            Some of the things stealth mode does include:
+                1) Patches the CDP runtime fingerprint.
+                2) Mimics some of the real browsers' properties by injecting several JS files and using custom options.
+                3) Using custom flags on launch to hide Playwright even more and make it faster.
+                4) Generates real browser's headers of the same type and same user OS then append it to the request.
+        - Real browsers by passing the `real_chrome` argument or the CDP URL of your browser to be controlled by the Fetcher and most of the options can be enabled on it.
         - NSTBrowser's docker browserless option by passing the CDP URL and enabling `nstbrowser_mode` option.
-        > Note that these are the main options with PlayWright but it can be mixed together.
+
+    > Note that these are the main options with PlayWright but it can be mixed together.
     """
     def fetch(
             self, url: str, headless: Union[bool, str] = True, disable_resources: bool = None,
             useragent: Optional[str] = None, network_idle: Optional[bool] = False, timeout: Optional[float] = 30000,
-            page_action: Callable = do_nothing, wait_selector: Optional[str] = None, wait_selector_state: Optional[str] = 'attached',
-            hide_canvas: bool = True, disable_webgl: bool = False, extra_headers: Optional[Dict[str, str]] = None, google_search: Optional[bool] = True,
+            page_action: Optional[Callable] = do_nothing, wait_selector: Optional[str] = None, wait_selector_state: Optional[str] = 'attached',
+            hide_canvas: Optional[bool] = False, disable_webgl: Optional[bool] = False, extra_headers: Optional[Dict[str, str]] = None, google_search: Optional[bool] = True,
             proxy: Optional[Union[str, Dict[str, str]]] = None,
-            stealth: bool = False,
+            stealth: Optional[bool] = False, real_chrome: Optional[bool] = False,
             cdp_url: Optional[str] = None,
-            nstbrowser_mode: bool = False, nstbrowser_config: Optional[Dict] = None,
+            nstbrowser_mode: Optional[bool] = False, nstbrowser_config: Optional[Dict] = None,
     ) -> Response:
         """Opens up a browser and do your request based on your chosen options below.
+
         :param url: Target url.
         :param headless: Run the browser in headless/hidden (default), or headful/visible mode.
         :param disable_resources: Drop requests of unnecessary resources for speed boost. It depends but it made requests ~25% faster in my tests for some websites.
@@ -159,6 +167,7 @@ class PlayWrightFetcher(BaseFetcher):
         :param wait_selector: Wait for a specific css selector to be in a specific state.
         :param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
         :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
+        :param real_chrome: If you have chrome browser installed on your device, enable this and the Fetcher will launch an instance of your browser and use it.
         :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
         :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
         :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
@@ -176,6 +185,7 @@ class PlayWrightFetcher(BaseFetcher):
             cdp_url=cdp_url,
             headless=headless,
             useragent=useragent,
+            real_chrome=real_chrome,
             page_action=page_action,
             hide_canvas=hide_canvas,
             network_idle=network_idle,

{scrapling-0.2.4.dist-info → scrapling-0.2.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scrapling
-Version: 0.2.4
+Version: 0.2.6
 Summary: Scrapling is a powerful, flexible, and high-performance web scraping library for Python. It 
 Home-page: https://github.com/D4Vinci/Scrapling
 Author: Karim Shoair
@@ -32,16 +32,16 @@ Classifier: Typing :: Typed
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: requests >=2.3
-Requires-Dist: lxml >=4.5
-Requires-Dist: cssselect >=1.2
+Requires-Dist: requests>=2.3
+Requires-Dist: lxml>=4.5
+Requires-Dist: cssselect>=1.2
 Requires-Dist: w3lib
-Requires-Dist: orjson >=3
+Requires-Dist: orjson>=3
 Requires-Dist: tldextract
 Requires-Dist: httpx[brotli,zstd]
-Requires-Dist: playwright
+Requires-Dist: playwright==1.48
 Requires-Dist: rebrowser-playwright
-Requires-Dist: camoufox >=0.3.10
+Requires-Dist: camoufox>=0.3.10
 Requires-Dist: browserforge
 
 # 🕷️ Scrapling: Undetectable, Lightning-Fast, and Adaptive Web Scraping for Python
@@ -336,9 +336,11 @@ Using this Fetcher class, you can make requests with:
      * Mimics some of the real browsers' properties by injecting several JS files and using custom options.
      * Using custom flags on launch to hide Playwright even more and make it faster.
      * Generates real browser's headers of the same type and same user OS then append it to the request's headers.
-  3) Real browsers by passing the CDP URL of your browser to be controlled by the Fetcher and most of the options can be enabled on it.
+  3) Real browsers by passing the `real_chrome` argument or the CDP URL of your browser to be controlled by the Fetcher and most of the options can be enabled on it.
   4) [NSTBrowser](https://app.nstbrowser.io/r/1vO5e5)'s [docker browserless](https://hub.docker.com/r/nstbrowser/browserless) option by passing the CDP URL and enabling `nstbrowser_mode` option.
 
+> Hence using the `real_chrome` argument requires that you have chrome browser installed on your device
+
 Add that to a lot of controlling/hiding options as you will see in the arguments list below.
 
 <details><summary><strong>Expand this for the complete list of arguments</strong></summary>
@@ -360,6 +362,7 @@ Add that to a lot of controlling/hiding options as you will see in the arguments
 |     hide_canvas     | Add random noise to canvas operations to prevent fingerprinting.                                                                                                                                                                                                                                                                                                                                                |    ✔️    |
 |    disable_webgl    | Disables WebGL and WebGL 2.0 support entirely.                                                                                                                                                                                                                                                                                                                                                                  |    ✔️    |
 |       stealth       | Enables stealth mode, always check the documentation to see what stealth mode does currently.                                                                                                                                                                                                                                                                                                                   |    ✔️    |
+|     real_chrome     | If you have chrome browser installed on your device, enable this and the Fetcher will launch an instance of your browser and use it.                                                                                                                                                                                                                                                                            |    ✔️    |
 |       cdp_url       | Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.                                                                                                                                                                                                                                                                                           |    ✔️    |
 |   nstbrowser_mode   | Enables NSTBrowser mode, **it have to be used with `cdp_url` argument or it will get completely ignored.**                                                                                                                                                                                                                                                                                                      |    ✔️    |
 |  nstbrowser_config  | The config you want to send with requests to the NSTBrowser. _If left empty, Scrapling defaults to an optimized NSTBrowser's docker browserless config._                                                                                                                                                                                                                                                        |    ✔️    |

{scrapling-0.2.4.dist-info → scrapling-0.2.6.dist-info}/RECORD

@@ -1,23 +1,23 @@
-scrapling/__init__.py,sha256=Tj_pDeN1yhufhlxQ0bY7Qnuxntq_JaqBUCQZrz01EFA,435
+scrapling/__init__.py,sha256=NnIpEZcBGs5Pu2TjqPCacC7N6LN37SbnniBU1AhgdXs,435
 scrapling/defaults.py,sha256=blYDLiuI5DgDSLRWnUgpp21WtFOsv1BsCRCmPeg8Xc4,287
-scrapling/fetchers.py,sha256=_6mL7XSTZE1fHXBqbxE2bBHnlQP1lH-4MCiQHQd5hQs,16017
+scrapling/fetchers.py,sha256=-gc-Yo1MjF_4cdJ-5rxZqNC0owxFXTFoEBj08BFEYPs,16361
 scrapling/parser.py,sha256=d2n00uF5i7W5lf0afLNRdk17ZFcNyiF9EzXLRQGA0NM,54111
 scrapling/py.typed,sha256=frcCV1k9oG9oKj3dpUqdJg1PxRT2RSN_XKdLCPjaYaY,2
 scrapling/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 scrapling/core/_types.py,sha256=nD2ZY_fitLohx3MfDmqoKJ9ZShrnRhQ8-d1SU1zEGAY,552
-scrapling/core/custom_types.py,sha256=-gMNOiByewoAUqFVrDp822V51rcWNlWVUOB6yGUL648,8403
+scrapling/core/custom_types.py,sha256=ztE_tshJ8i5uKqqSbsN5S6MoIUSfX6SexlhRjAnkclk,8402
 scrapling/core/mixins.py,sha256=sozbpaGL1_O_x3U-ABM5aYWpnxpCLfdbcA9SG3P7weY,3532
 scrapling/core/storage_adaptors.py,sha256=Kbak0BOJX5e9I1PbUS_4sUJi2Wxw8Bv5XsaLHAu1l2Q,6218
-scrapling/core/translator.py,sha256=oU-dQCkNQOccZPrXbPW_VSgC5ll10Bb89C3ezW2lI0o,5228
+scrapling/core/translator.py,sha256=R97lKGq1SDbx8S8Hg_w_5d4ePgukTHj_hRIKFzWiRuc,5229
 scrapling/core/utils.py,sha256=fXdANUgRBbVbOerJ94fRY9vi7n5zsbm8t3G4qQ-F3ak,3792
 scrapling/engines/__init__.py,sha256=zwMqcSdNGh-IX0d4zXazrgAeHrkqIN_v5Ia7RU1g8W0,267
-scrapling/engines/camo.py,sha256=WJNDR3OY5LLqNHRMD4YbwuqUdnEZ8U-Et_1YUn6vDiw,7773
+scrapling/engines/camo.py,sha256=dXkdfFmf3M09RXAvaZ8CE5khsblC3Wd7_6jWfu8XO6I,7618
 scrapling/engines/constants.py,sha256=jSDA6lgbvEIB8z2m2SFzCKkvFEZnp28Mondy2__FpkM,3721
-scrapling/engines/pw.py,sha256=6iNdnNF9M45FJkazeCvFRicyTFD2EkxSISJJP__uOug,12345
-scrapling/engines/static.py,sha256=Wsp6_-soZUQJT6kHoKPkLOdHU9J50chLdYxDmQjO4FQ,7101
+scrapling/engines/pw.py,sha256=gMWJAZYpJbFK-GiyRrpVrMjyMqSSetE6hf8kmf0zR2o,12729
+scrapling/engines/static.py,sha256=wzBsoOHPpN5JV1izQSSSarPBNWB-wo0BDWNFuin6ID8,7109
 scrapling/engines/toolbelt/__init__.py,sha256=BbxfC0depVOV3i3BnBnyfjHtLcZrDbhz6c5rTRczZUc,383
-scrapling/engines/toolbelt/custom.py,sha256=6Ip-9t2G8TaXLGLARQAEcbnFqvAN7AmgN1ah0glRiMs,9953
-scrapling/engines/toolbelt/fingerprints.py,sha256=kkVtZKSt2ukc0CV0g6QUvSWR0Yx5p8Mv8xiqACAsMBo,2917
+scrapling/engines/toolbelt/custom.py,sha256=ELr3_FwUqNI27E98kz-50OA5a6hQQtoIYrZoLKsvUpM,12551
+scrapling/engines/toolbelt/fingerprints.py,sha256=T9HQejHzAnHsD5EIXvrYVC5siiG5q2gOOXVIIANmzMc,2917
 scrapling/engines/toolbelt/navigation.py,sha256=Tde5_6Wv7lOeWXMzs9D6TRaxAbJ3b-zIX6-4HggZbCQ,4017
 scrapling/engines/toolbelt/bypasses/navigator_plugins.js,sha256=tbnnk3nCXB6QEQnOhDlu3n-s7lnUTAkrUsjP6FDQIQg,2104
 scrapling/engines/toolbelt/bypasses/notification_permission.js,sha256=poPM3o5WYgEX-EdiUfDCllpWfc3Umvw4jr2u6O6elus,237
@@ -28,14 +28,15 @@ scrapling/engines/toolbelt/bypasses/webdriver_fully.js,sha256=hdJw4clRAJQqIdq5gI
 scrapling/engines/toolbelt/bypasses/window_chrome.js,sha256=D7hqzNGGDorh8JVlvm2YIv7Bk2CoVkG55MDIdyqhT1w,6808
 tests/__init__.py,sha256=YHFB5ftzgLQVh6gbPfbYcY4yOS9DOBp5dBa6I-qtm8U,32
 tests/fetchers/__init__.py,sha256=6H4NgARhyTcGGd3dNCKQJ8kUFdrAEMSScQL7Ga_vU3c,43
-tests/fetchers/test_camoufox.py,sha256=XPTCDZ9sj_GpCzXyvzKF_uZWhEYX6J_jh_BLeMEl8yY,2874
+tests/fetchers/test_camoufox.py,sha256=53piGA5uuPvOx5BeUEA0bbizYihwHGxehnj5uqCr6Q0,3115
 tests/fetchers/test_httpx.py,sha256=UivOItR3-l-bXp9E6TP5Tvn2OrCdgiVkWsti-f9xdpU,3507
-tests/fetchers/test_playwright.py,sha256=YOWn89urd9NwoCHfTFj8fY4xYrRY2BeszTt5Q-TxUcs,3479
+tests/fetchers/test_playwright.py,sha256=7qwbIU2SwjiQEbaGPA_MBo6kAXM4IBmfvy5kUvKT11M,3701
+tests/fetchers/test_utils.py,sha256=FPPJkBrqgYxdGeWwapH8Vj8zyfYVLiTE1qSLu8eBWik,5728
 tests/parser/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/parser/test_automatch.py,sha256=BeeYJi3cYCghbiZmi57z4bqcGPaoUA8GAm7MALBBkkk,2486
-tests/parser/test_general.py,sha256=NfTuGLgAm-LH0dVV0pvbRcYSNI-wSu05rdnuRzmB0m4,11664
-scrapling-0.2.4.dist-info/LICENSE,sha256=XHgu8DRuT7_g3Hb9Q18YGg8eShp6axPBacbnQxT_WWQ,1499
-scrapling-0.2.4.dist-info/METADATA,sha256=uOp98w2qzOGqE4ofFFG_TgWgZGrscQHWhmP49pfIV3s,64785
-scrapling-0.2.4.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
-scrapling-0.2.4.dist-info/top_level.txt,sha256=ub7FkOEXeYmmYTUxd4pCrwXfBfAMIpZ1sCGmXCc14tI,16
-scrapling-0.2.4.dist-info/RECORD,,
+tests/parser/test_general.py,sha256=qaiVzpvqESfdXYFat6QrpnMkevPYgCzIcTZK5FwdC0s,11783
+scrapling-0.2.6.dist-info/LICENSE,sha256=XHgu8DRuT7_g3Hb9Q18YGg8eShp6axPBacbnQxT_WWQ,1499
+scrapling-0.2.6.dist-info/METADATA,sha256=cFOu2nlkXDsjyjkIt9kDu1nKKvS14xYH2LT4_VNH5j0,65362
+scrapling-0.2.6.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+scrapling-0.2.6.dist-info/top_level.txt,sha256=ub7FkOEXeYmmYTUxd4pCrwXfBfAMIpZ1sCGmXCc14tI,16
+scrapling-0.2.6.dist-info/RECORD,,

{scrapling-0.2.4.dist-info → scrapling-0.2.6.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.5.0)
+Generator: setuptools (75.6.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

tests/fetchers/test_camoufox.py

@@ -36,6 +36,7 @@ class TestStealthyFetcher(unittest.TestCase):
     def test_waiting_selector(self):
         """Test if waiting for a selector make page does not finish loading or not"""
         self.assertEqual(self.fetcher.fetch(self.html_url, wait_selector='h1').status, 200)
+        self.assertEqual(self.fetcher.fetch(self.html_url, wait_selector='h1', wait_selector_state='visible').status, 200)
 
     def test_cookies_loading(self):
         """Test if cookies are set after the request"""
@@ -56,6 +57,7 @@ class TestStealthyFetcher(unittest.TestCase):
         self.assertEqual(self.fetcher.fetch(self.html_url, block_webrtc=True, allow_webgl=True).status, 200)
         self.assertEqual(self.fetcher.fetch(self.html_url, block_webrtc=False, allow_webgl=True).status, 200)
         self.assertEqual(self.fetcher.fetch(self.html_url, block_webrtc=True, allow_webgl=False).status, 200)
+        self.assertEqual(self.fetcher.fetch(self.html_url, extra_headers={'ayo': ''}, os_randomize=True).status, 200)
 
     def test_infinite_timeout(self):
         """Test if infinite timeout breaks the code or not"""

tests/fetchers/test_playwright.py

@@ -35,6 +35,7 @@ class TestPlayWrightFetcher(unittest.TestCase):
     def test_waiting_selector(self):
         """Test if waiting for a selector make page does not finish loading or not"""
         self.assertEqual(self.fetcher.fetch(self.html_url, wait_selector='h1').status, 200)
+        self.assertEqual(self.fetcher.fetch(self.html_url, wait_selector='h1', wait_selector_state='visible').status, 200)
 
     def test_cookies_loading(self):
         """Test if cookies are set after the request"""
@@ -56,6 +57,7 @@ class TestPlayWrightFetcher(unittest.TestCase):
         self.assertEqual(self.fetcher.fetch(self.html_url, disable_webgl=False, hide_canvas=True).status, 200)
         self.assertEqual(self.fetcher.fetch(self.html_url, stealth=True).status, 200)
         self.assertEqual(self.fetcher.fetch(self.html_url, useragent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0').status, 200)
+        self.assertEqual(self.fetcher.fetch(self.html_url, extra_headers={'ayo': ''}).status, 200)
 
     def test_cdp_url(self):
         """Test if it's going to try to connect to cdp url or not"""

tests/fetchers/test_utils.py

@@ -0,0 +1,129 @@
+import unittest
+
+from scrapling.engines.toolbelt.custom import ResponseEncoding, StatusText
+
+
+class TestPlayWrightFetcher(unittest.TestCase):
+    def setUp(self):
+        self.content_type_map = {
+            # A map generated by ChatGPT for most possible `content_type` values and the expected outcome
+            'text/html; charset=UTF-8': 'UTF-8',
+            'text/html; charset=ISO-8859-1': 'ISO-8859-1',
+            'text/html': 'ISO-8859-1',
+            'application/json; charset=UTF-8': 'UTF-8',
+            'application/json': 'utf-8',
+            'text/json': 'utf-8',
+            'application/javascript; charset=UTF-8': 'UTF-8',
+            'application/javascript': 'utf-8',
+            'text/plain; charset=UTF-8': 'UTF-8',
+            'text/plain; charset=ISO-8859-1': 'ISO-8859-1',
+            'text/plain': 'ISO-8859-1',
+            'application/xhtml+xml; charset=UTF-8': 'UTF-8',
+            'application/xhtml+xml': 'utf-8',
+            'text/html; charset=windows-1252': 'windows-1252',
+            'application/json; charset=windows-1252': 'windows-1252',
+            'text/plain; charset=windows-1252': 'windows-1252',
+            'text/html; charset="UTF-8"': 'UTF-8',
+            'text/html; charset="ISO-8859-1"': 'ISO-8859-1',
+            'text/html; charset="windows-1252"': 'windows-1252',
+            'application/json; charset="UTF-8"': 'UTF-8',
+            'application/json; charset="ISO-8859-1"': 'ISO-8859-1',
+            'application/json; charset="windows-1252"': 'windows-1252',
+            'text/json; charset="UTF-8"': 'UTF-8',
+            'application/javascript; charset="UTF-8"': 'UTF-8',
+            'application/javascript; charset="ISO-8859-1"': 'ISO-8859-1',
+            'text/plain; charset="UTF-8"': 'UTF-8',
+            'text/plain; charset="ISO-8859-1"': 'ISO-8859-1',
+            'text/plain; charset="windows-1252"': 'windows-1252',
+            'application/xhtml+xml; charset="UTF-8"': 'UTF-8',
+            'application/xhtml+xml; charset="ISO-8859-1"': 'ISO-8859-1',
+            'application/xhtml+xml; charset="windows-1252"': 'windows-1252',
+            'text/html; charset="US-ASCII"': 'US-ASCII',
+            'application/json; charset="US-ASCII"': 'US-ASCII',
+            'text/plain; charset="US-ASCII"': 'US-ASCII',
+            'text/html; charset="Shift_JIS"': 'Shift_JIS',
+            'application/json; charset="Shift_JIS"': 'Shift_JIS',
+            'text/plain; charset="Shift_JIS"': 'Shift_JIS',
+            'application/xml; charset="UTF-8"': 'UTF-8',
+            'application/xml; charset="ISO-8859-1"': 'ISO-8859-1',
+            'application/xml': 'utf-8',
+            'text/xml; charset="UTF-8"': 'UTF-8',
+            'text/xml; charset="ISO-8859-1"': 'ISO-8859-1',
+            'text/xml': 'utf-8'
+        }
+        self.status_map = {
+            100: "Continue",
+            101: "Switching Protocols",
+            102: "Processing",
+            103: "Early Hints",
+            200: "OK",
+            201: "Created",
+            202: "Accepted",
+            203: "Non-Authoritative Information",
+            204: "No Content",
+            205: "Reset Content",
+            206: "Partial Content",
+            207: "Multi-Status",
+            208: "Already Reported",
+            226: "IM Used",
+            300: "Multiple Choices",
+            301: "Moved Permanently",
+            302: "Found",
+            303: "See Other",
+            304: "Not Modified",
+            305: "Use Proxy",
+            307: "Temporary Redirect",
+            308: "Permanent Redirect",
+            400: "Bad Request",
+            401: "Unauthorized",
+            402: "Payment Required",
+            403: "Forbidden",
+            404: "Not Found",
+            405: "Method Not Allowed",
+            406: "Not Acceptable",
+            407: "Proxy Authentication Required",
+            408: "Request Timeout",
+            409: "Conflict",
+            410: "Gone",
+            411: "Length Required",
+            412: "Precondition Failed",
+            413: "Payload Too Large",
+            414: "URI Too Long",
+            415: "Unsupported Media Type",
+            416: "Range Not Satisfiable",
+            417: "Expectation Failed",
+            418: "I'm a teapot",
+            421: "Misdirected Request",
+            422: "Unprocessable Entity",
+            423: "Locked",
+            424: "Failed Dependency",
+            425: "Too Early",
+            426: "Upgrade Required",
+            428: "Precondition Required",
+            429: "Too Many Requests",
+            431: "Request Header Fields Too Large",
+            451: "Unavailable For Legal Reasons",
+            500: "Internal Server Error",
+            501: "Not Implemented",
+            502: "Bad Gateway",
+            503: "Service Unavailable",
+            504: "Gateway Timeout",
+            505: "HTTP Version Not Supported",
+            506: "Variant Also Negotiates",
+            507: "Insufficient Storage",
+            508: "Loop Detected",
+            510: "Not Extended",
+            511: "Network Authentication Required"
+        }
+
+    def test_parsing_content_type(self):
+        """Test if parsing different types of content-type returns the expected result"""
+        for header_value, expected_encoding in self.content_type_map.items():
+            self.assertEqual(ResponseEncoding.get_value(header_value), expected_encoding)
+
+    def test_parsing_response_status(self):
+        """Test if using different http responses' status codes returns the expected result"""
+        for status_code, expected_status_text in self.status_map.items():
+            self.assertEqual(StatusText.get(status_code), expected_status_text)
+
+        self.assertEqual(StatusText.get(1000), "Unknown Status Code")

tests/parser/test_general.py

@@ -278,7 +278,7 @@ class TestParser(unittest.TestCase):
         self.assertEqual(len(elements), 5000)
         # Converting 5000 elements to a class and doing operations on them will take time
         # Based on my tests with 100 runs, 1 loop each Scrapling (given the extra work/features) takes 10.4ms on average
-        self.assertLess(end_time - start_time, 0.1)
+        self.assertLess(end_time - start_time, 0.5)  # Locally I test on 0.1 but on GitHub actions with browsers and threading sometimes closing adds fractions of seconds
 
 
 # Use `coverage run -m unittest --verbose tests/test_parser_functions.py` instead for the coverage report