scrapling 0.2.99__py3-none-any.whl → 0.3.1__py3-none-any.whl

scrapling/engines/toolbelt/custom.py

@@ -1,19 +1,29 @@
 """
 Functions related to custom types or type checking
 """
-import inspect
+
 from email.message import Message
 
-from scrapling.core._types import (Any, Callable, Dict, List, Optional, Tuple,
-                                   Type, Union)
+from scrapling.core._types import (
+    Any,
+    Dict,
+    List,
+    Optional,
+    Tuple,
+)
 from scrapling.core.custom_types import MappingProxyType
 from scrapling.core.utils import log, lru_cache
-from scrapling.parser import Adaptor, SQLiteStorageSystem
+from scrapling.parser import Selector, SQLiteStorageSystem
 
 
 class ResponseEncoding:
     __DEFAULT_ENCODING = "utf-8"
-    __ISO_8859_1_CONTENT_TYPES = {"text/plain", "text/html", "text/css", "text/javascript"}
+    __ISO_8859_1_CONTENT_TYPES = {
+        "text/plain",
+        "text/html",
+        "text/css",
+        "text/javascript",
+    }
 
     @classmethod
     @lru_cache(maxsize=128)
@@ -27,19 +37,21 @@ class ResponseEncoding:
         """
         # Create a Message object and set the Content-Type header then get the content type and parameters
         msg = Message()
-        msg['content-type'] = header_value
+        msg["content-type"] = header_value
 
         content_type = msg.get_content_type()
         params = dict(msg.get_params(failobj=[]))
 
         # Remove the content-type from params if present somehow
-        params.pop('content-type', None)
+        params.pop("content-type", None)
 
         return content_type, params
 
     @classmethod
     @lru_cache(maxsize=128)
-    def get_value(cls, content_type: Optional[str], text: Optional[str] = 'test') -> str:
+    def get_value(
+        cls, content_type: Optional[str], text: Optional[str] = "test"
+    ) -> str:
         """Determine the appropriate character encoding from a content-type header.
 
         The encoding is determined by these rules in order:
@@ -72,7 +84,9 @@ class ResponseEncoding:
                 encoding = cls.__DEFAULT_ENCODING
 
             if encoding:
-                _ = text.encode(encoding)  # Validate encoding and validate it can encode the given text
+                _ = text.encode(
+                    encoding
+                )  # Validate encoding and validate it can encode the given text
                 return encoding
 
             return cls.__DEFAULT_ENCODING
@@ -81,48 +95,74 @@ class ResponseEncoding:
             return cls.__DEFAULT_ENCODING
 
 
-class Response(Adaptor):
+class Response(Selector):
     """This class is returned by all engines as a way to unify response type between different libraries."""
 
-    def __init__(self, url: str, text: str, body: bytes, status: int, reason: str, cookies: Dict, headers: Dict, request_headers: Dict,
-                 encoding: str = 'utf-8', method: str = 'GET', history: List = None, **adaptor_arguments: Dict):
-        automatch_domain = adaptor_arguments.pop('automatch_domain', None)
+    def __init__(
+        self,
+        url: str,
+        content: str | bytes,
+        status: int,
+        reason: str,
+        cookies: Tuple[Dict[str, str], ...] | Dict[str, str],
+        headers: Dict,
+        request_headers: Dict,
+        encoding: str = "utf-8",
+        method: str = "GET",
+        history: List = None,
+        **selector_config: Dict,
+    ):
+        adaptive_domain = selector_config.pop("adaptive_domain", None)
         self.status = status
         self.reason = reason
         self.cookies = cookies
         self.headers = headers
         self.request_headers = request_headers
         self.history = history or []
-        encoding = ResponseEncoding.get_value(encoding, text)
-        super().__init__(text=text, body=body, url=automatch_domain or url, encoding=encoding, **adaptor_arguments)
-        # For back-ward compatibility
-        self.adaptor = self
+        encoding = ResponseEncoding.get_value(
+            encoding, content.decode("utf-8") if isinstance(content, bytes) else content
+        )
+        super().__init__(
+            content=content,
+            url=adaptive_domain or url,
+            encoding=encoding,
+            **selector_config,
+        )
         # For easier debugging while working from a Python shell
-        log.info(f'Fetched ({status}) <{method} {url}> (referer: {request_headers.get("referer")})')
-
-    # def __repr__(self):
-    #     return f'<{self.__class__.__name__} [{self.status} {self.reason}]>'
+        log.info(
+            f"Fetched ({status}) <{method} {url}> (referer: {request_headers.get('referer')})"
+        )
 
 
 class BaseFetcher:
     __slots__ = ()
     huge_tree: bool = True
-    auto_match: Optional[bool] = False
+    adaptive: Optional[bool] = False
     storage: Any = SQLiteStorageSystem
     keep_cdata: Optional[bool] = False
     storage_args: Optional[Dict] = None
     keep_comments: Optional[bool] = False
-    automatch_domain: Optional[str] = None
-    parser_keywords: Tuple = ('huge_tree', 'auto_match', 'storage', 'keep_cdata', 'storage_args', 'keep_comments', 'automatch_domain',)  # Left open for the user
+    adaptive_domain: Optional[str] = None
+    parser_keywords: Tuple = (
+        "huge_tree",
+        "adaptive",
+        "storage",
+        "keep_cdata",
+        "storage_args",
+        "keep_comments",
+        "adaptive_domain",
+    )  # Left open for the user
 
     def __init__(self, *args, **kwargs):
         # For backward-compatibility before 0.2.99
-        args_str = ", ".join(args) or ''
-        kwargs_str = ", ".join(f'{k}={v}' for k, v in kwargs.items()) or ''
+        args_str = ", ".join(args) or ""
+        kwargs_str = ", ".join(f"{k}={v}" for k, v in kwargs.items()) or ""
         if args_str:
-            args_str += ', '
+            args_str += ", "
 
-        log.warning(f'This logic is deprecated now, and have no effect; It will be removed with v0.3. Use `{self.__class__.__name__}.configure({args_str}{kwargs_str})` instead before fetching')
+        log.warning(
+            f"This logic is deprecated now, and have no effect; It will be removed with v0.3. Use `{self.__class__.__name__}.configure({args_str}{kwargs_str})` instead before fetching"
+        )
         pass
 
     @classmethod
@@ -131,17 +171,17 @@ class BaseFetcher:
             huge_tree=cls.huge_tree,
             keep_comments=cls.keep_comments,
             keep_cdata=cls.keep_cdata,
-            auto_match=cls.auto_match,
+            adaptive=cls.adaptive,
             storage=cls.storage,
             storage_args=cls.storage_args,
-            automatch_domain=cls.automatch_domain,
+            adaptive_domain=cls.adaptive_domain,
         )
 
     @classmethod
     def configure(cls, **kwargs):
         """Set multiple arguments for the parser at once globally
 
-        :param kwargs: The keywords can be any arguments of the following: huge_tree, keep_comments, keep_cdata, auto_match, storage, storage_args, automatch_domain
+        :param kwargs: The keywords can be any arguments of the following: huge_tree, keep_comments, keep_cdata, adaptive, storage, storage_args, adaptive_domain
         """
         for key, value in kwargs.items():
             key = key.strip().lower()
@@ -150,30 +190,38 @@ class BaseFetcher:
                     setattr(cls, key, value)
                 else:
                     # Yup, no fun allowed LOL
-                    raise AttributeError(f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?')
+                    raise AttributeError(
+                        f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?'
+                    )
             else:
-                raise ValueError(f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?')
+                raise ValueError(
+                    f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?'
+                )
 
         if not kwargs:
-            raise AttributeError(f'You must pass a keyword to configure, current keywords: {cls.parser_keywords}?')
+            raise AttributeError(
+                f"You must pass a keyword to configure, current keywords: {cls.parser_keywords}?"
+            )
 
     @classmethod
     def _generate_parser_arguments(cls) -> Dict:
-        # Adaptor class parameters
-        # I won't validate Adaptor's class parameters here again, I will leave it to be validated later
+        # Selector class parameters
+        # I won't validate Selector's class parameters here again, I will leave it to be validated later
         parser_arguments = dict(
             huge_tree=cls.huge_tree,
             keep_comments=cls.keep_comments,
             keep_cdata=cls.keep_cdata,
-            auto_match=cls.auto_match,
+            adaptive=cls.adaptive,
             storage=cls.storage,
-            storage_args=cls.storage_args
+            storage_args=cls.storage_args,
         )
-        if cls.automatch_domain:
-            if type(cls.automatch_domain) is not str:
-                log.warning('[Ignored] The argument "automatch_domain" must be of string type')
+        if cls.adaptive_domain:
+            if not isinstance(cls.adaptive_domain, str):
+                log.warning(
+                    '[Ignored] The argument "adaptive_domain" must be of string type'
+                )
             else:
-                parser_arguments.update({'automatch_domain': cls.automatch_domain})
+                parser_arguments.update({"adaptive_domain": cls.adaptive_domain})
 
         return parser_arguments
 
@@ -181,72 +229,75 @@ class BaseFetcher:
 class StatusText:
     """A class that gets the status text of response status code.
 
-        Reference: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status
+    Reference: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status
     """
-    _phrases = MappingProxyType({
-        100: "Continue",
-        101: "Switching Protocols",
-        102: "Processing",
-        103: "Early Hints",
-        200: "OK",
-        201: "Created",
-        202: "Accepted",
-        203: "Non-Authoritative Information",
-        204: "No Content",
-        205: "Reset Content",
-        206: "Partial Content",
-        207: "Multi-Status",
-        208: "Already Reported",
-        226: "IM Used",
-        300: "Multiple Choices",
-        301: "Moved Permanently",
-        302: "Found",
-        303: "See Other",
-        304: "Not Modified",
-        305: "Use Proxy",
-        307: "Temporary Redirect",
-        308: "Permanent Redirect",
-        400: "Bad Request",
-        401: "Unauthorized",
-        402: "Payment Required",
-        403: "Forbidden",
-        404: "Not Found",
-        405: "Method Not Allowed",
-        406: "Not Acceptable",
-        407: "Proxy Authentication Required",
-        408: "Request Timeout",
-        409: "Conflict",
-        410: "Gone",
-        411: "Length Required",
-        412: "Precondition Failed",
-        413: "Payload Too Large",
-        414: "URI Too Long",
-        415: "Unsupported Media Type",
-        416: "Range Not Satisfiable",
-        417: "Expectation Failed",
-        418: "I'm a teapot",
-        421: "Misdirected Request",
-        422: "Unprocessable Entity",
-        423: "Locked",
-        424: "Failed Dependency",
-        425: "Too Early",
-        426: "Upgrade Required",
-        428: "Precondition Required",
-        429: "Too Many Requests",
-        431: "Request Header Fields Too Large",
-        451: "Unavailable For Legal Reasons",
-        500: "Internal Server Error",
-        501: "Not Implemented",
-        502: "Bad Gateway",
-        503: "Service Unavailable",
-        504: "Gateway Timeout",
-        505: "HTTP Version Not Supported",
-        506: "Variant Also Negotiates",
-        507: "Insufficient Storage",
-        508: "Loop Detected",
-        510: "Not Extended",
-        511: "Network Authentication Required"
-    })
+
+    _phrases = MappingProxyType(
+        {
+            100: "Continue",
+            101: "Switching Protocols",
+            102: "Processing",
+            103: "Early Hints",
+            200: "OK",
+            201: "Created",
+            202: "Accepted",
+            203: "Non-Authoritative Information",
+            204: "No Content",
+            205: "Reset Content",
+            206: "Partial Content",
+            207: "Multi-Status",
+            208: "Already Reported",
+            226: "IM Used",
+            300: "Multiple Choices",
+            301: "Moved Permanently",
+            302: "Found",
+            303: "See Other",
+            304: "Not Modified",
+            305: "Use Proxy",
+            307: "Temporary Redirect",
+            308: "Permanent Redirect",
+            400: "Bad Request",
+            401: "Unauthorized",
+            402: "Payment Required",
+            403: "Forbidden",
+            404: "Not Found",
+            405: "Method Not Allowed",
+            406: "Not Acceptable",
+            407: "Proxy Authentication Required",
+            408: "Request Timeout",
+            409: "Conflict",
+            410: "Gone",
+            411: "Length Required",
+            412: "Precondition Failed",
+            413: "Payload Too Large",
+            414: "URI Too Long",
+            415: "Unsupported Media Type",
+            416: "Range Not Satisfiable",
+            417: "Expectation Failed",
+            418: "I'm a teapot",
+            421: "Misdirected Request",
+            422: "Unprocessable Entity",
+            423: "Locked",
+            424: "Failed Dependency",
+            425: "Too Early",
+            426: "Upgrade Required",
+            428: "Precondition Required",
+            429: "Too Many Requests",
+            431: "Request Header Fields Too Large",
+            451: "Unavailable For Legal Reasons",
+            500: "Internal Server Error",
+            501: "Not Implemented",
+            502: "Bad Gateway",
+            503: "Service Unavailable",
+            504: "Gateway Timeout",
+            505: "HTTP Version Not Supported",
+            506: "Variant Also Negotiates",
+            507: "Insufficient Storage",
+            508: "Loop Detected",
+            510: "Not Extended",
+            511: "Network Authentication Required",
+        }
+    )
 
     @classmethod
     @lru_cache(maxsize=128)
@@ -255,32 +306,6 @@ class StatusText:
         return cls._phrases.get(status_code, "Unknown Status Code")
 
 
-def check_if_engine_usable(engine: Callable) -> Union[Callable, None]:
-    """This function check if the passed engine can be used by a Fetcher-type class or not.
-
-    :param engine: The engine class itself
-    :return: The engine class again if all checks out, otherwise raises error
-    :raise TypeError: If engine class don't have fetch method, If engine class have fetch attribute not method, or If engine class have fetch function but it doesn't take arguments
-    """
-    # if isinstance(engine, type):
-    #     raise TypeError("Expected an engine instance, not a class definition of the engine")
-
-    if hasattr(engine, 'fetch'):
-        fetch_function = getattr(engine, "fetch")
-        if callable(fetch_function):
-            if len(inspect.signature(fetch_function).parameters) > 0:
-                return engine
-            else:
-                # raise TypeError("Engine class instance must have a callable method 'fetch' with the first argument used for the url.")
-                raise TypeError("Engine class must have a callable method 'fetch' with the first argument used for the url.")
-        else:
-            # raise TypeError("Invalid engine instance! Engine class must have a callable method 'fetch'")
-            raise TypeError("Invalid engine class! Engine class must have a callable method 'fetch'")
-    else:
-        # raise TypeError("Invalid engine instance! Engine class must have the method 'fetch'")
-        raise TypeError("Invalid engine class! Engine class must have the method 'fetch'")
-
-
 def get_variable_name(var: Any) -> Optional[str]:
     """Get the name of a variable using global and local scopes.
     :param var: The variable to find the name for
@@ -291,45 +316,3 @@ def get_variable_name(var: Any) -> Optional[str]:
             if value is var:
                 return name
     return None
-
-
-def check_type_validity(variable: Any, valid_types: Union[List[Type], None], default_value: Any = None, critical: bool = False, param_name: Optional[str] = None) -> Any:
-    """Check if a variable matches the specified type constraints.
-    :param variable: The variable to check
-    :param valid_types: List of valid types for the variable
-    :param default_value: Value to return if type check fails
-    :param critical: If True, raises TypeError instead of logging error
-    :param param_name: Optional parameter name for error messages
-    :return: The original variable if valid, default_value if invalid
-    :raise TypeError: If critical=True and type check fails
-    """
-    # Use provided param_name or try to get it automatically
-    var_name = param_name or get_variable_name(variable) or "Unknown"
-
-    # Convert valid_types to a list if None
-    valid_types = valid_types or []
-
-    # Handle None value
-    if variable is None:
-        if type(None) in valid_types:
-            return variable
-        error_msg = f'Argument "{var_name}" cannot be None'
-        if critical:
-            raise TypeError(error_msg)
-        log.error(f'[Ignored] {error_msg}')
-        return default_value
-
-    # If no valid_types specified and variable has a value, return it
-    if not valid_types:
-        return variable
-
-    # Check if variable type matches any of the valid types
-    if not any(isinstance(variable, t) for t in valid_types):
-        type_names = [t.__name__ for t in valid_types]
-        error_msg = f'Argument "{var_name}" must be of type {" or ".join(type_names)}'
-        if critical:
-            raise TypeError(error_msg)
-        log.error(f'[Ignored] {error_msg}')
-        return default_value
-
-    return variable

scrapling/engines/toolbelt/fingerprints.py

@@ -2,19 +2,20 @@
 Functions related to generating headers and fingerprints generally
 """
 
-import platform
+from platform import system as platform_system
 
-from browserforge.fingerprints import Fingerprint, FingerprintGenerator
-from browserforge.headers import Browser, HeaderGenerator
 from tldextract import extract
+from browserforge.headers import Browser, HeaderGenerator
 
-from scrapling.core._types import Dict, Union
+from scrapling.core._types import Dict, Optional
 from scrapling.core.utils import lru_cache
 
+__OS_NAME__ = platform_system()
+
 
 @lru_cache(10, typed=True)
 def generate_convincing_referer(url: str) -> str:
-    """Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website
+    """Takes the domain from the URL without the subdomain/suffix and make it look like you were searching Google for this website
 
     >>> generate_convincing_referer('https://www.somewebsite.com/blah')
     'https://www.google.com/search?q=somewebsite'
@@ -23,59 +24,44 @@ def generate_convincing_referer(url: str) -> str:
     :return: Google's search URL of the domain name
     """
     website_name = extract(url).domain
-    return f'https://www.google.com/search?q={website_name}'
+    return f"https://www.google.com/search?q={website_name}"
 
 
 @lru_cache(1, typed=True)
-def get_os_name() -> Union[str, None]:
+def get_os_name() -> Optional[str]:
     """Get the current OS name in the same format needed for browserforge
 
     :return: Current OS name or `None` otherwise
     """
-    #
-    os_name = platform.system()
     return {
-        'Linux': 'linux',
-        'Darwin': 'macos',
-        'Windows': 'windows',
-        # For the future? because why not
-        'iOS': 'ios',
-    }.get(os_name)
-
-
-def generate_suitable_fingerprint() -> Fingerprint:
-    """Generates a browserforge's fingerprint that matches current OS, desktop device, and Chrome with version 128 at least.
-
-    This function was originally created to test Browserforge's injector.
-    :return: `Fingerprint` object
-    """
-    return FingerprintGenerator(
-        browser=[Browser(name='chrome', min_version=128)],
-        os=get_os_name(),  # None is ignored
-        device='desktop'
-    ).generate()
+        "Linux": "linux",
+        "Darwin": "macos",
+        "Windows": "windows",
+        # For the future? because why not?
+        "iOS": "ios",
+    }.get(__OS_NAME__)
 
 
 def generate_headers(browser_mode: bool = False) -> Dict:
     """Generate real browser-like headers using browserforge's generator
 
-    :param browser_mode: If enabled, the headers created are used for playwright so it have to match everything
+    :param browser_mode: If enabled, the headers created are used for playwright, so it has to match everything
     :return: A dictionary of the generated headers
     """
-    if browser_mode:
-        # In this mode we don't care about anything other than matching the OS and the browser type with the browser we are using
-        # So we don't raise any inconsistency red flags while websites fingerprinting us
-        os_name = get_os_name()
-        return HeaderGenerator(
-            browser=[Browser(name='chrome', min_version=130)],
-            os=os_name,  # None is ignored
-            device='desktop'
-        ).generate()
-    else:
-        # Here it's used for normal requests that aren't done through browsers so we can take it lightly
-        browsers = [
-            Browser(name='chrome', min_version=120),
-            Browser(name='firefox', min_version=120),
-            Browser(name='edge', min_version=120),
-        ]
-        return HeaderGenerator(browser=browsers, device='desktop').generate()
+    # In the browser mode, we don't care about anything other than matching the OS and the browser type with the browser we are using,
+    # So we don't raise any inconsistency red flags while websites fingerprinting us
+    os_name = get_os_name()
+    browsers = [Browser(name="chrome", min_version=130)]
+    if not browser_mode:
+        os_name = ("windows", "macos", "linux")
+        browsers.extend(
+            [
+                Browser(name="firefox", min_version=130),
+                Browser(name="edge", min_version=130),
+            ]
+        )
+
+    return HeaderGenerator(browser=browsers, os=os_name, device="desktop").generate()
+
+
+__default_useragent__ = generate_headers(browser_mode=False).get("User-Agent")