scrapling 0.2.3__py3-none-any.whl → 0.2.5__py3-none-any.whl

scrapling/__init__.py

@@ -4,7 +4,7 @@ from scrapling.parser import Adaptor, Adaptors
 from scrapling.core.custom_types import TextHandler, AttributesHandler
 
 __author__ = "Karim Shoair (karim.shoair@pm.me)"
-__version__ = "0.2.3"
+__version__ = "0.2.5"
 __copyright__ = "Copyright (c) 2024 Karim Shoair"
 
 

scrapling/core/custom_types.py

@@ -129,9 +129,8 @@ class TextHandlers(List[TextHandler]):
 
 
 class AttributesHandler(Mapping):
-    """A read-only mapping to use instead of the standard dictionary for the speed boost but
-     at the same time I use it to add more functionalities.
-    If standard dictionary is needed, just convert this class to dictionary with `dict` function
+    """A read-only mapping to use instead of the standard dictionary for the speed boost but at the same time I use it to add more functionalities.
+        If standard dictionary is needed, just convert this class to dictionary with `dict` function
     """
     __slots__ = ('_data',)
 

scrapling/core/translator.py

@@ -1,9 +1,11 @@
 """
 Most of this file is adapted version of the translator of parsel library with some modifications simply for 1 important reason...
-To add pseudo-elements ``::text`` and ``::attr(ATTR_NAME)`` so we match Parsel/Scrapy selectors format
-which will be important in future releases but most importantly...
-    so you don't have to learn a new selectors/api method like what bs4 done with soupsieve :)
-> if you want to learn about this, head to https://cssselect.readthedocs.io/en/latest/#cssselect.FunctionalPseudoElement
+
+To add pseudo-elements ``::text`` and ``::attr(ATTR_NAME)`` so we match Parsel/Scrapy selectors format which will be important in future releases but most importantly...
+
+So you don't have to learn a new selectors/api method like what bs4 done with soupsieve :)
+
+    if you want to learn about this, head to https://cssselect.readthedocs.io/en/latest/#cssselect.FunctionalPseudoElement
 """
 
 import re

scrapling/engines/camo.py

@@ -4,6 +4,7 @@ from scrapling.core._types import Union, Callable, Optional, Dict, List, Literal
 from scrapling.engines.toolbelt import (
     Response,
     do_nothing,
+    StatusText,
     get_os_name,
     intercept_route,
     check_type_validity,
@@ -103,20 +104,22 @@ class CamoufoxEngine:
 
             if self.wait_selector and type(self.wait_selector) is str:
                 waiter = page.locator(self.wait_selector)
-                waiter.wait_for(state=self.wait_selector_state)
+                waiter.first.wait_for(state=self.wait_selector_state)
 
-            content_type = res.headers.get('content-type', '')
-            # Parse charset from content-type
-            encoding = 'utf-8'  # default encoding
-            if 'charset=' in content_type.lower():
-                encoding = content_type.lower().split('charset=')[-1].split(';')[0].strip()
+            # This will be parsed inside `Response`
+            encoding = res.headers.get('content-type', '') or 'utf-8'  # default encoding
+
+            status_text = res.status_text
+            # PlayWright API sometimes give empty status text for some reason!
+            if not status_text:
+                status_text = StatusText.get(res.status)
 
             response = Response(
                 url=res.url,
                 text=page.content(),
-                body=res.body(),
+                body=page.content().encode('utf-8'),
                 status=res.status,
-                reason=res.status_text,
+                reason=status_text,
                 encoding=encoding,
                 cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},
                 headers=res.all_headers(),

scrapling/engines/pw.py

@@ -6,6 +6,7 @@ from scrapling.engines.constants import DEFAULT_STEALTH_FLAGS, NSTBROWSER_DEFAUL
 from scrapling.engines.toolbelt import (
     Response,
     do_nothing,
+    StatusText,
     js_bypass_path,
     intercept_route,
     generate_headers,
@@ -213,20 +214,22 @@ class PlaywrightEngine:
 
             if self.wait_selector and type(self.wait_selector) is str:
                 waiter = page.locator(self.wait_selector)
-                waiter.wait_for(state=self.wait_selector_state)
+                waiter.first.wait_for(state=self.wait_selector_state)
 
-            content_type = res.headers.get('content-type', '')
-            # Parse charset from content-type
-            encoding = 'utf-8'  # default encoding
-            if 'charset=' in content_type.lower():
-                encoding = content_type.lower().split('charset=')[-1].split(';')[0].strip()
+            # This will be parsed inside `Response`
+            encoding = res.headers.get('content-type', '') or 'utf-8'  # default encoding
+
+            status_text = res.status_text
+            # PlayWright API sometimes give empty status text for some reason!
+            if not status_text:
+                status_text = StatusText.get(res.status)
 
             response = Response(
                 url=res.url,
                 text=page.content(),
-                body=res.body(),
+                body=page.content().encode('utf-8'),
                 status=res.status,
-                reason=res.status_text,
+                reason=status_text,
                 encoding=encoding,
                 cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},
                 headers=res.all_headers(),

scrapling/engines/static.py

@@ -23,7 +23,7 @@ class StaticEngine:
     @staticmethod
     def _headers_job(headers: Optional[Dict], url: str, stealth: bool) -> Dict:
         """Adds useragent to headers if it doesn't exist, generates real headers and append it to current headers, and
-        finally generates a referer header that looks like if this request came from Google's search of the current URL's domain.
+            finally generates a referer header that looks like if this request came from Google's search of the current URL's domain.
 
         :param headers: Current headers in the request if the user passed any
         :param url: The Target URL.
@@ -65,6 +65,7 @@ class StaticEngine:
 
     def get(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
         """Make basic HTTP GET request for you but with some added flavors.
+
         :param url: Target url.
         :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
             create a referer header as if this request had came from Google's search of this URL's domain.
@@ -77,6 +78,7 @@ class StaticEngine:
 
     def post(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
         """Make basic HTTP POST request for you but with some added flavors.
+
         :param url: Target url.
         :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
             create a referer header as if this request had came from Google's search of this URL's domain.
@@ -89,6 +91,7 @@ class StaticEngine:
 
     def delete(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
         """Make basic HTTP DELETE request for you but with some added flavors.
+
         :param url: Target url.
         :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
             create a referer header as if this request had came from Google's search of this URL's domain.
@@ -101,6 +104,7 @@ class StaticEngine:
 
     def put(self, url: str, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
         """Make basic HTTP PUT request for you but with some added flavors.
+
         :param url: Target url.
         :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
             create a referer header as if this request had came from Google's search of this URL's domain.

scrapling/engines/toolbelt/__init__.py

@@ -6,6 +6,7 @@ from .fingerprints import (
 from .custom import (
     Response,
     do_nothing,
+    StatusText,
     BaseFetcher,
     get_variable_name,
     check_type_validity,

scrapling/engines/toolbelt/custom.py

@@ -3,10 +3,78 @@ Functions related to custom types or type checking
 """
 import inspect
 import logging
+from email.message import Message
 
-from scrapling.core.utils import setup_basic_logging
+from scrapling.core.custom_types import MappingProxyType
 from scrapling.parser import Adaptor, SQLiteStorageSystem
-from scrapling.core._types import Any, List, Type, Union, Optional, Dict, Callable
+from scrapling.core.utils import setup_basic_logging, cache
+from scrapling.core._types import Any, List, Type, Union, Optional, Dict, Callable, Tuple
+
+
+class ResponseEncoding:
+    __DEFAULT_ENCODING = "utf-8"
+    __ISO_8859_1_CONTENT_TYPES = {"text/plain", "text/html", "text/css", "text/javascript"}
+
+    @classmethod
+    @cache(maxsize=None)
+    def __parse_content_type(cls, header_value: str) -> Tuple[str, Dict[str, str]]:
+        """Parse content type and parameters from a content-type header value.
+
+            Uses `email.message.Message` for robust header parsing according to RFC 2045.
+
+        :param header_value: Raw content-type header string
+        :return: Tuple of (content_type, parameters_dict)
+        """
+        # Create a Message object and set the Content-Type header then get the content type and parameters
+        msg = Message()
+        msg['content-type'] = header_value
+
+        content_type = msg.get_content_type()
+        params = dict(msg.get_params(failobj=[]))
+
+        # Remove the content-type from params if present somehow
+        params.pop('content-type', None)
+
+        return content_type, params
+
+    @classmethod
+    @cache(maxsize=None)
+    def get_value(cls, content_type: Optional[str]) -> str:
+        """Determine the appropriate character encoding from a content-type header.
+
+        The encoding is determined by these rules in order:
+            1. If no content-type is provided, use UTF-8
+            2. If charset parameter is present, use that encoding
+            3. If content-type is `text/*`, use ISO-8859-1 per HTTP/1.1 spec
+            4. If content-type is application/json, use UTF-8 per RFC 4627
+            5. Default to UTF-8 if nothing else matches
+
+        :param content_type: Content-Type header value or None
+        :return: String naming the character encoding
+        """
+        if not content_type:
+            return cls.__DEFAULT_ENCODING
+
+        try:
+            content_type, params = cls.__parse_content_type(content_type)
+
+            # First check for explicit charset parameter
+            if "charset" in params:
+                encoding = params["charset"].strip("'\"")
+                "test".encode(encoding)  # Validate encoding
+                return encoding
+
+            # Apply content-type specific rules
+            if content_type in cls.__ISO_8859_1_CONTENT_TYPES:
+                return "ISO-8859-1"
+
+            if content_type == "application/json":
+                return cls.__DEFAULT_ENCODING
+
+            return cls.__DEFAULT_ENCODING
+
+        except (ValueError, LookupError, UnicodeEncodeError):
+            return cls.__DEFAULT_ENCODING
 
 
 class Response(Adaptor):
@@ -19,6 +87,7 @@ class Response(Adaptor):
         self.cookies = cookies
         self.headers = headers
         self.request_headers = request_headers
+        encoding = ResponseEncoding.get_value(encoding)
         super().__init__(text=text, body=body, url=automatch_domain or url, encoding=encoding, **adaptor_arguments)
         # For back-ward compatibility
         self.adaptor = self
@@ -67,6 +136,83 @@ class BaseFetcher:
                 self.adaptor_arguments.update({'automatch_domain': automatch_domain})
 
 
+class StatusText:
+    """A class that gets the status text of response status code.
+
+        Reference: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status
+    """
+    _phrases = MappingProxyType({
+        100: "Continue",
+        101: "Switching Protocols",
+        102: "Processing",
+        103: "Early Hints",
+        200: "OK",
+        201: "Created",
+        202: "Accepted",
+        203: "Non-Authoritative Information",
+        204: "No Content",
+        205: "Reset Content",
+        206: "Partial Content",
+        207: "Multi-Status",
+        208: "Already Reported",
+        226: "IM Used",
+        300: "Multiple Choices",
+        301: "Moved Permanently",
+        302: "Found",
+        303: "See Other",
+        304: "Not Modified",
+        305: "Use Proxy",
+        307: "Temporary Redirect",
+        308: "Permanent Redirect",
+        400: "Bad Request",
+        401: "Unauthorized",
+        402: "Payment Required",
+        403: "Forbidden",
+        404: "Not Found",
+        405: "Method Not Allowed",
+        406: "Not Acceptable",
+        407: "Proxy Authentication Required",
+        408: "Request Timeout",
+        409: "Conflict",
+        410: "Gone",
+        411: "Length Required",
+        412: "Precondition Failed",
+        413: "Payload Too Large",
+        414: "URI Too Long",
+        415: "Unsupported Media Type",
+        416: "Range Not Satisfiable",
+        417: "Expectation Failed",
+        418: "I'm a teapot",
+        421: "Misdirected Request",
+        422: "Unprocessable Entity",
+        423: "Locked",
+        424: "Failed Dependency",
+        425: "Too Early",
+        426: "Upgrade Required",
+        428: "Precondition Required",
+        429: "Too Many Requests",
+        431: "Request Header Fields Too Large",
+        451: "Unavailable For Legal Reasons",
+        500: "Internal Server Error",
+        501: "Not Implemented",
+        502: "Bad Gateway",
+        503: "Service Unavailable",
+        504: "Gateway Timeout",
+        505: "HTTP Version Not Supported",
+        506: "Variant Also Negotiates",
+        507: "Insufficient Storage",
+        508: "Loop Detected",
+        510: "Not Extended",
+        511: "Network Authentication Required"
+    })
+
+    @classmethod
+    @cache(maxsize=128)
+    def get(cls, status_code: int) -> str:
+        """Get the phrase for a given HTTP status code."""
+        return cls._phrases.get(status_code, "Unknown Status Code")
+
+
 def check_if_engine_usable(engine: Callable) -> Union[Callable, None]:
     """This function check if the passed engine can be used by a Fetcher-type class or not.
 

scrapling/fetchers.py

@@ -11,6 +11,7 @@ class Fetcher(BaseFetcher):
     """
     def get(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
         """Make basic HTTP GET request for you but with some added flavors.
+
         :param url: Target url.
         :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
         :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
@@ -24,6 +25,7 @@ class Fetcher(BaseFetcher):
 
     def post(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
         """Make basic HTTP POST request for you but with some added flavors.
+
         :param url: Target url.
         :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
         :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
@@ -37,12 +39,14 @@ class Fetcher(BaseFetcher):
 
     def put(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
         """Make basic HTTP PUT request for you but with some added flavors.
+
         :param url: Target url
         :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
         :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
         :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
-        create a referer header as if this request came from Google's search of this URL's domain.
+            create a referer header as if this request came from Google's search of this URL's domain.
         :param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
+
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
         response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).put(url, stealthy_headers, **kwargs)
@@ -50,6 +54,7 @@ class Fetcher(BaseFetcher):
 
     def delete(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response:
         """Make basic HTTP DELETE request for you but with some added flavors.
+
         :param url: Target url
         :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
         :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
@@ -77,6 +82,7 @@ class StealthyFetcher(BaseFetcher):
     ) -> Response:
         """
         Opens up a browser and do your request based on your chosen options below.
+
         :param url: Target url.
         :param headless: Run the browser in headless/hidden (default), 'virtual' screen mode, or headful/visible mode.
         :param block_images: Prevent the loading of images through Firefox preferences.
@@ -127,14 +133,15 @@ class PlayWrightFetcher(BaseFetcher):
      Using this Fetcher class, you can do requests with:
         - Vanilla Playwright without any modifications other than the ones you chose.
         - Stealthy Playwright with the stealth mode I wrote for it. It's still a work in progress but it bypasses many online tests like bot.sannysoft.com
-        Some of the things stealth mode does include:
-            1) Patches the CDP runtime fingerprint.
-            2) Mimics some of the real browsers' properties by injecting several JS files and using custom options.
-            3) Using custom flags on launch to hide Playwright even more and make it faster.
-            4) Generates real browser's headers of the same type and same user OS then append it to the request.
+            Some of the things stealth mode does include:
+                1) Patches the CDP runtime fingerprint.
+                2) Mimics some of the real browsers' properties by injecting several JS files and using custom options.
+                3) Using custom flags on launch to hide Playwright even more and make it faster.
+                4) Generates real browser's headers of the same type and same user OS then append it to the request.
         - Real browsers by passing the CDP URL of your browser to be controlled by the Fetcher and most of the options can be enabled on it.
         - NSTBrowser's docker browserless option by passing the CDP URL and enabling `nstbrowser_mode` option.
-        > Note that these are the main options with PlayWright but it can be mixed together.
+
+    > Note that these are the main options with PlayWright but it can be mixed together.
     """
     def fetch(
             self, url: str, headless: Union[bool, str] = True, disable_resources: bool = None,
@@ -147,6 +154,7 @@ class PlayWrightFetcher(BaseFetcher):
             nstbrowser_mode: bool = False, nstbrowser_config: Optional[Dict] = None,
     ) -> Response:
         """Opens up a browser and do your request based on your chosen options below.
+
         :param url: Target url.
         :param headless: Run the browser in headless/hidden (default), or headful/visible mode.
         :param disable_resources: Drop requests of unnecessary resources for speed boost. It depends but it made requests ~25% faster in my tests for some websites.

{scrapling-0.2.3.dist-info → scrapling-0.2.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scrapling
-Version: 0.2.3
+Version: 0.2.5
 Summary: Scrapling is a powerful, flexible, and high-performance web scraping library for Python. It 
 Home-page: https://github.com/D4Vinci/Scrapling
 Author: Karim Shoair
@@ -32,16 +32,16 @@ Classifier: Typing :: Typed
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: requests >=2.3
-Requires-Dist: lxml >=4.5
-Requires-Dist: cssselect >=1.2
+Requires-Dist: requests>=2.3
+Requires-Dist: lxml>=4.5
+Requires-Dist: cssselect>=1.2
 Requires-Dist: w3lib
-Requires-Dist: orjson >=3
+Requires-Dist: orjson>=3
 Requires-Dist: tldextract
 Requires-Dist: httpx[brotli,zstd]
 Requires-Dist: playwright
 Requires-Dist: rebrowser-playwright
-Requires-Dist: camoufox >=0.3.10
+Requires-Dist: camoufox>=0.3.10
 Requires-Dist: browserforge
 
 # 🕷️ Scrapling: Undetectable, Lightning-Fast, and Adaptive Web Scraping for Python

{scrapling-0.2.3.dist-info → scrapling-0.2.5.dist-info}/RECORD

@@ -1,22 +1,22 @@
-scrapling/__init__.py,sha256=tNB1LdlhamZYjlxqbEwo1BpsVE48Bt8acfjl8DNKpEE,435
+scrapling/__init__.py,sha256=eSTwhUqPltd41XPPwSXElWmeRalaQEncnjBahW3KUcQ,435
 scrapling/defaults.py,sha256=blYDLiuI5DgDSLRWnUgpp21WtFOsv1BsCRCmPeg8Xc4,287
-scrapling/fetchers.py,sha256=_6mL7XSTZE1fHXBqbxE2bBHnlQP1lH-4MCiQHQd5hQs,16017
+scrapling/fetchers.py,sha256=T3aRXvvpXDbql_2BqJ90KLtgfF4HLUOImdoxWLCcY-A,16045
 scrapling/parser.py,sha256=d2n00uF5i7W5lf0afLNRdk17ZFcNyiF9EzXLRQGA0NM,54111
 scrapling/py.typed,sha256=frcCV1k9oG9oKj3dpUqdJg1PxRT2RSN_XKdLCPjaYaY,2
 scrapling/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 scrapling/core/_types.py,sha256=nD2ZY_fitLohx3MfDmqoKJ9ZShrnRhQ8-d1SU1zEGAY,552
-scrapling/core/custom_types.py,sha256=-gMNOiByewoAUqFVrDp822V51rcWNlWVUOB6yGUL648,8403
+scrapling/core/custom_types.py,sha256=ztE_tshJ8i5uKqqSbsN5S6MoIUSfX6SexlhRjAnkclk,8402
 scrapling/core/mixins.py,sha256=sozbpaGL1_O_x3U-ABM5aYWpnxpCLfdbcA9SG3P7weY,3532
 scrapling/core/storage_adaptors.py,sha256=Kbak0BOJX5e9I1PbUS_4sUJi2Wxw8Bv5XsaLHAu1l2Q,6218
-scrapling/core/translator.py,sha256=oU-dQCkNQOccZPrXbPW_VSgC5ll10Bb89C3ezW2lI0o,5228
+scrapling/core/translator.py,sha256=R97lKGq1SDbx8S8Hg_w_5d4ePgukTHj_hRIKFzWiRuc,5229
 scrapling/core/utils.py,sha256=fXdANUgRBbVbOerJ94fRY9vi7n5zsbm8t3G4qQ-F3ak,3792
 scrapling/engines/__init__.py,sha256=zwMqcSdNGh-IX0d4zXazrgAeHrkqIN_v5Ia7RU1g8W0,267
-scrapling/engines/camo.py,sha256=41vp2Nh51kKuOSZ1PijsIpROpQZgFfUPybVbEX8pEXk,7530
+scrapling/engines/camo.py,sha256=dXkdfFmf3M09RXAvaZ8CE5khsblC3Wd7_6jWfu8XO6I,7618
 scrapling/engines/constants.py,sha256=jSDA6lgbvEIB8z2m2SFzCKkvFEZnp28Mondy2__FpkM,3721
-scrapling/engines/pw.py,sha256=l5MrSW_WNBKAxAlyxbt09ka_lEGo61XKuaOgWpYmvHk,12102
-scrapling/engines/static.py,sha256=Wsp6_-soZUQJT6kHoKPkLOdHU9J50chLdYxDmQjO4FQ,7101
-scrapling/engines/toolbelt/__init__.py,sha256=BnBp34aDeohYgqdysEAAWnGZgA02YlExkc5FJLetMSo,367
-scrapling/engines/toolbelt/custom.py,sha256=8lvGHWIZoOotSTF97KgPb3CbJquel2QFx8rP8Hf2sQ4,7469
+scrapling/engines/pw.py,sha256=Llr2HjEr5kgyffEJH5N4XdWrSWvcil9WAwE1RJnOhFo,12190
+scrapling/engines/static.py,sha256=wzBsoOHPpN5JV1izQSSSarPBNWB-wo0BDWNFuin6ID8,7109
+scrapling/engines/toolbelt/__init__.py,sha256=BbxfC0depVOV3i3BnBnyfjHtLcZrDbhz6c5rTRczZUc,383
+scrapling/engines/toolbelt/custom.py,sha256=ELr3_FwUqNI27E98kz-50OA5a6hQQtoIYrZoLKsvUpM,12551
 scrapling/engines/toolbelt/fingerprints.py,sha256=kkVtZKSt2ukc0CV0g6QUvSWR0Yx5p8Mv8xiqACAsMBo,2917
 scrapling/engines/toolbelt/navigation.py,sha256=Tde5_6Wv7lOeWXMzs9D6TRaxAbJ3b-zIX6-4HggZbCQ,4017
 scrapling/engines/toolbelt/bypasses/navigator_plugins.js,sha256=tbnnk3nCXB6QEQnOhDlu3n-s7lnUTAkrUsjP6FDQIQg,2104
@@ -28,14 +28,15 @@ scrapling/engines/toolbelt/bypasses/webdriver_fully.js,sha256=hdJw4clRAJQqIdq5gI
 scrapling/engines/toolbelt/bypasses/window_chrome.js,sha256=D7hqzNGGDorh8JVlvm2YIv7Bk2CoVkG55MDIdyqhT1w,6808
 tests/__init__.py,sha256=YHFB5ftzgLQVh6gbPfbYcY4yOS9DOBp5dBa6I-qtm8U,32
 tests/fetchers/__init__.py,sha256=6H4NgARhyTcGGd3dNCKQJ8kUFdrAEMSScQL7Ga_vU3c,43
-tests/fetchers/test_camoufox.py,sha256=XPTCDZ9sj_GpCzXyvzKF_uZWhEYX6J_jh_BLeMEl8yY,2874
+tests/fetchers/test_camoufox.py,sha256=53piGA5uuPvOx5BeUEA0bbizYihwHGxehnj5uqCr6Q0,3115
 tests/fetchers/test_httpx.py,sha256=UivOItR3-l-bXp9E6TP5Tvn2OrCdgiVkWsti-f9xdpU,3507
-tests/fetchers/test_playwright.py,sha256=YOWn89urd9NwoCHfTFj8fY4xYrRY2BeszTt5Q-TxUcs,3479
+tests/fetchers/test_playwright.py,sha256=7qwbIU2SwjiQEbaGPA_MBo6kAXM4IBmfvy5kUvKT11M,3701
+tests/fetchers/test_utils.py,sha256=FPPJkBrqgYxdGeWwapH8Vj8zyfYVLiTE1qSLu8eBWik,5728
 tests/parser/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/parser/test_automatch.py,sha256=BeeYJi3cYCghbiZmi57z4bqcGPaoUA8GAm7MALBBkkk,2486
-tests/parser/test_general.py,sha256=NfTuGLgAm-LH0dVV0pvbRcYSNI-wSu05rdnuRzmB0m4,11664
-scrapling-0.2.3.dist-info/LICENSE,sha256=XHgu8DRuT7_g3Hb9Q18YGg8eShp6axPBacbnQxT_WWQ,1499
-scrapling-0.2.3.dist-info/METADATA,sha256=YGUnQmOdwDVs7rHLUCG9hpLg567s_5I0Cu8TIKKDc_Y,64785
-scrapling-0.2.3.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
-scrapling-0.2.3.dist-info/top_level.txt,sha256=ub7FkOEXeYmmYTUxd4pCrwXfBfAMIpZ1sCGmXCc14tI,16
-scrapling-0.2.3.dist-info/RECORD,,
+tests/parser/test_general.py,sha256=qaiVzpvqESfdXYFat6QrpnMkevPYgCzIcTZK5FwdC0s,11783
+scrapling-0.2.5.dist-info/LICENSE,sha256=XHgu8DRuT7_g3Hb9Q18YGg8eShp6axPBacbnQxT_WWQ,1499
+scrapling-0.2.5.dist-info/METADATA,sha256=oY8RtycwNJXbgA4ePsVkpj6SKP6Y6vMmOgF9L3lc0mg,64780
+scrapling-0.2.5.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+scrapling-0.2.5.dist-info/top_level.txt,sha256=ub7FkOEXeYmmYTUxd4pCrwXfBfAMIpZ1sCGmXCc14tI,16
+scrapling-0.2.5.dist-info/RECORD,,

{scrapling-0.2.3.dist-info → scrapling-0.2.5.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.5.0)
+Generator: setuptools (75.6.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

tests/fetchers/test_camoufox.py

@@ -36,6 +36,7 @@ class TestStealthyFetcher(unittest.TestCase):
     def test_waiting_selector(self):
         """Test if waiting for a selector make page does not finish loading or not"""
         self.assertEqual(self.fetcher.fetch(self.html_url, wait_selector='h1').status, 200)
+        self.assertEqual(self.fetcher.fetch(self.html_url, wait_selector='h1', wait_selector_state='visible').status, 200)
 
     def test_cookies_loading(self):
         """Test if cookies are set after the request"""
@@ -56,6 +57,7 @@ class TestStealthyFetcher(unittest.TestCase):
         self.assertEqual(self.fetcher.fetch(self.html_url, block_webrtc=True, allow_webgl=True).status, 200)
         self.assertEqual(self.fetcher.fetch(self.html_url, block_webrtc=False, allow_webgl=True).status, 200)
         self.assertEqual(self.fetcher.fetch(self.html_url, block_webrtc=True, allow_webgl=False).status, 200)
+        self.assertEqual(self.fetcher.fetch(self.html_url, extra_headers={'ayo': ''}, os_randomize=True).status, 200)
 
     def test_infinite_timeout(self):
         """Test if infinite timeout breaks the code or not"""

tests/fetchers/test_playwright.py

@@ -35,6 +35,7 @@ class TestPlayWrightFetcher(unittest.TestCase):
     def test_waiting_selector(self):
         """Test if waiting for a selector make page does not finish loading or not"""
         self.assertEqual(self.fetcher.fetch(self.html_url, wait_selector='h1').status, 200)
+        self.assertEqual(self.fetcher.fetch(self.html_url, wait_selector='h1', wait_selector_state='visible').status, 200)
 
     def test_cookies_loading(self):
         """Test if cookies are set after the request"""
@@ -56,6 +57,7 @@ class TestPlayWrightFetcher(unittest.TestCase):
         self.assertEqual(self.fetcher.fetch(self.html_url, disable_webgl=False, hide_canvas=True).status, 200)
         self.assertEqual(self.fetcher.fetch(self.html_url, stealth=True).status, 200)
         self.assertEqual(self.fetcher.fetch(self.html_url, useragent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0').status, 200)
+        self.assertEqual(self.fetcher.fetch(self.html_url, extra_headers={'ayo': ''}).status, 200)
 
     def test_cdp_url(self):
         """Test if it's going to try to connect to cdp url or not"""

tests/fetchers/test_utils.py

@@ -0,0 +1,129 @@
+import unittest
+
+from scrapling.engines.toolbelt.custom import ResponseEncoding, StatusText
+
+
+class TestPlayWrightFetcher(unittest.TestCase):
+    def setUp(self):
+        self.content_type_map = {
+            # A map generated by ChatGPT for most possible `content_type` values and the expected outcome
+            'text/html; charset=UTF-8': 'UTF-8',
+            'text/html; charset=ISO-8859-1': 'ISO-8859-1',
+            'text/html': 'ISO-8859-1',
+            'application/json; charset=UTF-8': 'UTF-8',
+            'application/json': 'utf-8',
+            'text/json': 'utf-8',
+            'application/javascript; charset=UTF-8': 'UTF-8',
+            'application/javascript': 'utf-8',
+            'text/plain; charset=UTF-8': 'UTF-8',
+            'text/plain; charset=ISO-8859-1': 'ISO-8859-1',
+            'text/plain': 'ISO-8859-1',
+            'application/xhtml+xml; charset=UTF-8': 'UTF-8',
+            'application/xhtml+xml': 'utf-8',
+            'text/html; charset=windows-1252': 'windows-1252',
+            'application/json; charset=windows-1252': 'windows-1252',
+            'text/plain; charset=windows-1252': 'windows-1252',
+            'text/html; charset="UTF-8"': 'UTF-8',
+            'text/html; charset="ISO-8859-1"': 'ISO-8859-1',
+            'text/html; charset="windows-1252"': 'windows-1252',
+            'application/json; charset="UTF-8"': 'UTF-8',
+            'application/json; charset="ISO-8859-1"': 'ISO-8859-1',
+            'application/json; charset="windows-1252"': 'windows-1252',
+            'text/json; charset="UTF-8"': 'UTF-8',
+            'application/javascript; charset="UTF-8"': 'UTF-8',
+            'application/javascript; charset="ISO-8859-1"': 'ISO-8859-1',
+            'text/plain; charset="UTF-8"': 'UTF-8',
+            'text/plain; charset="ISO-8859-1"': 'ISO-8859-1',
+            'text/plain; charset="windows-1252"': 'windows-1252',
+            'application/xhtml+xml; charset="UTF-8"': 'UTF-8',
+            'application/xhtml+xml; charset="ISO-8859-1"': 'ISO-8859-1',
+            'application/xhtml+xml; charset="windows-1252"': 'windows-1252',
+            'text/html; charset="US-ASCII"': 'US-ASCII',
+            'application/json; charset="US-ASCII"': 'US-ASCII',
+            'text/plain; charset="US-ASCII"': 'US-ASCII',
+            'text/html; charset="Shift_JIS"': 'Shift_JIS',
+            'application/json; charset="Shift_JIS"': 'Shift_JIS',
+            'text/plain; charset="Shift_JIS"': 'Shift_JIS',
+            'application/xml; charset="UTF-8"': 'UTF-8',
+            'application/xml; charset="ISO-8859-1"': 'ISO-8859-1',
+            'application/xml': 'utf-8',
+            'text/xml; charset="UTF-8"': 'UTF-8',
+            'text/xml; charset="ISO-8859-1"': 'ISO-8859-1',
+            'text/xml': 'utf-8'
+        }
+        self.status_map = {
+            100: "Continue",
+            101: "Switching Protocols",
+            102: "Processing",
+            103: "Early Hints",
+            200: "OK",
+            201: "Created",
+            202: "Accepted",
+            203: "Non-Authoritative Information",
+            204: "No Content",
+            205: "Reset Content",
+            206: "Partial Content",
+            207: "Multi-Status",
+            208: "Already Reported",
+            226: "IM Used",
+            300: "Multiple Choices",
+            301: "Moved Permanently",
+            302: "Found",
+            303: "See Other",
+            304: "Not Modified",
+            305: "Use Proxy",
+            307: "Temporary Redirect",
+            308: "Permanent Redirect",
+            400: "Bad Request",
+            401: "Unauthorized",
+            402: "Payment Required",
+            403: "Forbidden",
+            404: "Not Found",
+            405: "Method Not Allowed",
+            406: "Not Acceptable",
+            407: "Proxy Authentication Required",
+            408: "Request Timeout",
+            409: "Conflict",
+            410: "Gone",
+            411: "Length Required",
+            412: "Precondition Failed",
+            413: "Payload Too Large",
+            414: "URI Too Long",
+            415: "Unsupported Media Type",
+            416: "Range Not Satisfiable",
+            417: "Expectation Failed",
+            418: "I'm a teapot",
+            421: "Misdirected Request",
+            422: "Unprocessable Entity",
+            423: "Locked",
+            424: "Failed Dependency",
+            425: "Too Early",
+            426: "Upgrade Required",
+            428: "Precondition Required",
+            429: "Too Many Requests",
+            431: "Request Header Fields Too Large",
+            451: "Unavailable For Legal Reasons",
+            500: "Internal Server Error",
+            501: "Not Implemented",
+            502: "Bad Gateway",
+            503: "Service Unavailable",
+            504: "Gateway Timeout",
+            505: "HTTP Version Not Supported",
+            506: "Variant Also Negotiates",
+            507: "Insufficient Storage",
+            508: "Loop Detected",
+            510: "Not Extended",
+            511: "Network Authentication Required"
+        }
+
+    def test_parsing_content_type(self):
+        """Test if parsing different types of content-type returns the expected result"""
+        for header_value, expected_encoding in self.content_type_map.items():
+            self.assertEqual(ResponseEncoding.get_value(header_value), expected_encoding)
+
+    def test_parsing_response_status(self):
+        """Test if using different http responses' status codes returns the expected result"""
+        for status_code, expected_status_text in self.status_map.items():
+            self.assertEqual(StatusText.get(status_code), expected_status_text)
+
+        self.assertEqual(StatusText.get(1000), "Unknown Status Code")

tests/parser/test_general.py

@@ -278,7 +278,7 @@ class TestParser(unittest.TestCase):
         self.assertEqual(len(elements), 5000)
         # Converting 5000 elements to a class and doing operations on them will take time
         # Based on my tests with 100 runs, 1 loop each Scrapling (given the extra work/features) takes 10.4ms on average
-        self.assertLess(end_time - start_time, 0.1)
+        self.assertLess(end_time - start_time, 0.5)  # Locally I test on 0.1 but on GitHub actions with browsers and threading sometimes closing adds fractions of seconds
 
 
 # Use `coverage run -m unittest --verbose tests/test_parser_functions.py` instead for the coverage report