scrapling 0.2.97__py3-none-any.whl → 0.2.99__py3-none-any.whl

scrapling/__init__.py

@@ -1,12 +1,41 @@
-# Declare top-level shortcuts
-from scrapling.core.custom_types import AttributesHandler, TextHandler
-from scrapling.fetchers import (AsyncFetcher, CustomFetcher, Fetcher,
-                                PlayWrightFetcher, StealthyFetcher)
-from scrapling.parser import Adaptor, Adaptors
 
 __author__ = "Karim Shoair (karim.shoair@pm.me)"
-__version__ = "0.2.97"
+__version__ = "0.2.99"
 __copyright__ = "Copyright (c) 2024 Karim Shoair"
 
 
+# A lightweight approach to create lazy loader for each import for backward compatibility
+# This will reduces initial memory footprint significantly (only loads what's used)
+def __getattr__(name):
+    if name == 'Fetcher':
+        from scrapling.fetchers import Fetcher as cls
+        return cls
+    elif name == 'Adaptor':
+        from scrapling.parser import Adaptor as cls
+        return cls
+    elif name == 'Adaptors':
+        from scrapling.parser import Adaptors as cls
+        return cls
+    elif name == 'AttributesHandler':
+        from scrapling.core.custom_types import AttributesHandler as cls
+        return cls
+    elif name == 'TextHandler':
+        from scrapling.core.custom_types import TextHandler as cls
+        return cls
+    elif name == 'AsyncFetcher':
+        from scrapling.fetchers import AsyncFetcher as cls
+        return cls
+    elif name == 'StealthyFetcher':
+        from scrapling.fetchers import StealthyFetcher as cls
+        return cls
+    elif name == 'PlayWrightFetcher':
+        from scrapling.fetchers import PlayWrightFetcher as cls
+        return cls
+    elif name == 'CustomFetcher':
+        from scrapling.fetchers import CustomFetcher as cls
+        return cls
+    else:
+        raise AttributeError(f"module 'scrapling' has no attribute '{name}'")
+
+
 __all__ = ['Adaptor', 'Fetcher', 'AsyncFetcher', 'StealthyFetcher', 'PlayWrightFetcher']

scrapling/core/storage_adaptors.py

@@ -19,7 +19,7 @@ class StorageSystemMixin(ABC):
         """
         self.url = url
 
-    @lru_cache(126, typed=True)
+    @lru_cache(64, typed=True)
     def _get_base_url(self, default_value: str = 'default') -> str:
         if not self.url or type(self.url) is not str:
             return default_value
@@ -51,7 +51,7 @@ class StorageSystemMixin(ABC):
         raise NotImplementedError('Storage system must implement `save` method')
 
     @staticmethod
-    @lru_cache(256, typed=True)
+    @lru_cache(128, typed=True)
     def _get_hash(identifier: str) -> str:
         """If you want to hash identifier in your storage system, use this safer"""
         identifier = identifier.lower().strip()
@@ -63,7 +63,7 @@ class StorageSystemMixin(ABC):
         return f"{hash_value}_{len(identifier)}"  # Length to reduce collision chance
 
 
-@lru_cache(10, typed=True)
+@lru_cache(1, typed=True)
 class SQLiteStorageSystem(StorageSystemMixin):
     """The recommended system to use, it's race condition safe and thread safe.
     Mainly built so the library can run in threaded frameworks like scrapy or threaded tools

scrapling/core/translator.py

@@ -142,3 +142,6 @@ class HTMLTranslator(TranslatorMixin, OriginalHTMLTranslator):
     @lru_cache(maxsize=256)
     def css_to_xpath(self, css: str, prefix: str = "descendant-or-self::") -> str:
         return super().css_to_xpath(css, prefix)
+
+
+translator_instance = HTMLTranslator()

scrapling/core/utils.py

@@ -115,7 +115,7 @@ class _StorageTools:
 #     return _impl
 
 
-@lru_cache(256, typed=True)
+@lru_cache(128, typed=True)
 def clean_spaces(string):
     string = string.replace('\t', ' ')
     string = re.sub('[\n|\r]', '', string)

scrapling/defaults.py

@@ -1,10 +1,25 @@
-from .fetchers import AsyncFetcher as _AsyncFetcher
-from .fetchers import Fetcher as _Fetcher
-from .fetchers import PlayWrightFetcher as _PlayWrightFetcher
-from .fetchers import StealthyFetcher as _StealthyFetcher
+# Left this file for backward-compatibility before 0.2.99
+from scrapling.core.utils import log
 
-# If you are going to use Fetchers with the default settings, import them from this file instead for a cleaner looking code
-Fetcher = _Fetcher()
-AsyncFetcher = _AsyncFetcher()
-StealthyFetcher = _StealthyFetcher()
-PlayWrightFetcher = _PlayWrightFetcher()
+
+# A lightweight approach to create lazy loader for each import for backward compatibility
+# This will reduces initial memory footprint significantly (only loads what's used)
+def __getattr__(name):
+    if name == 'Fetcher':
+        from scrapling.fetchers import Fetcher as cls
+        log.warning('This import is deprecated now and it will be removed with v0.3. Use `from scrapling.fetchers import Fetcher` instead')
+        return cls
+    elif name == 'AsyncFetcher':
+        from scrapling.fetchers import AsyncFetcher as cls
+        log.warning('This import is deprecated now and it will be removed with v0.3. Use `from scrapling.fetchers import AsyncFetcher` instead')
+        return cls
+    elif name == 'StealthyFetcher':
+        from scrapling.fetchers import StealthyFetcher as cls
+        log.warning('This import is deprecated now and it will be removed with v0.3. Use `from scrapling.fetchers import StealthyFetcher` instead')
+        return cls
+    elif name == 'PlayWrightFetcher':
+        from scrapling.fetchers import PlayWrightFetcher as cls
+        log.warning('This import is deprecated now and it will be removed with v0.3. Use `from scrapling.fetchers import PlayWrightFetcher` instead')
+        return cls
+    else:
+        raise AttributeError(f"module 'scrapling' has no attribute '{name}'")

scrapling/engines/camo.py

@@ -16,12 +16,13 @@ from scrapling.engines.toolbelt import (Response, StatusText,
 class CamoufoxEngine:
     def __init__(
             self, headless: Union[bool, Literal['virtual']] = True, block_images: bool = False, disable_resources: bool = False,
-            block_webrtc: bool = False, allow_webgl: bool = True, network_idle: bool = False, humanize: Union[bool, float] = True,
+            block_webrtc: bool = False, allow_webgl: bool = True, network_idle: bool = False, humanize: Union[bool, float] = True, wait: Optional[int] = 0,
             timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, addons: Optional[List[str]] = None,
             wait_selector_state: SelectorWaitStates = 'attached', google_search: bool = True, extra_headers: Optional[Dict[str, str]] = None,
             proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: bool = False, disable_ads: bool = False,
             geoip: bool = False,
             adaptor_arguments: Dict = None,
+            additional_arguments: Dict = None
     ):
         """An engine that utilizes Camoufox library, check the `StealthyFetcher` class for more documentation.
 
@@ -38,6 +39,7 @@ class CamoufoxEngine:
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
         :param disable_ads: Disabled by default, this installs `uBlock Origin` addon on the browser if enabled.
         :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS.
+        :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning `Response` object.
         :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000
         :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
         :param wait_selector: Wait for a specific css selector to be in a specific state.
@@ -48,6 +50,7 @@ class CamoufoxEngine:
         :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
         :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
         :param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
+        :param additional_arguments: Additional arguments to be passed to Camoufox as additional settings and it takes higher priority than Scrapling's settings.
         """
         self.headless = headless
         self.block_images = bool(block_images)
@@ -60,10 +63,12 @@ class CamoufoxEngine:
         self.disable_ads = bool(disable_ads)
         self.geoip = bool(geoip)
         self.extra_headers = extra_headers or {}
+        self.additional_arguments = additional_arguments or {}
         self.proxy = construct_proxy_dict(proxy)
         self.addons = addons or []
         self.humanize = humanize
         self.timeout = check_type_validity(timeout, [int, float], 30000)
+        self.wait = check_type_validity(wait, [int, float], 0)
 
         # Page action callable validation
         self.page_action = None
@@ -92,6 +97,7 @@ class CamoufoxEngine:
             "block_webrtc": self.block_webrtc,
             "block_images": self.block_images,  # Careful! it makes some websites doesn't finish loading at all like stackoverflow even in headful
             "os": None if self.os_randomize else get_os_name(),
+            **self.additional_arguments
         }
 
     def _process_response_history(self, first_response):
@@ -126,6 +132,38 @@ class CamoufoxEngine:
 
         return history
 
+    async def _async_process_response_history(self, first_response):
+        """Process response history to build a list of Response objects"""
+        history = []
+        current_request = first_response.request.redirected_from
+
+        try:
+            while current_request:
+                try:
+                    current_response = await current_request.response()
+                    history.insert(0, Response(
+                        url=current_request.url,
+                        # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
+                        text='',
+                        body=b'',
+                        status=current_response.status if current_response else 301,
+                        reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301),
+                        encoding=current_response.headers.get('content-type', '') or 'utf-8',
+                        cookies={},
+                        headers=await current_response.all_headers() if current_response else {},
+                        request_headers=await current_request.all_headers(),
+                        **self.adaptor_arguments
+                    ))
+                except Exception as e:
+                    log.error(f"Error processing redirect: {e}")
+                    break
+
+                current_request = current_request.redirected_from
+        except Exception as e:
+            log.error(f"Error processing response history: {e}")
+
+        return history
+
     def fetch(self, url: str) -> Response:
         """Opens up the browser and do your request based on your chosen options.
 
@@ -177,6 +215,7 @@ class CamoufoxEngine:
                 except Exception as e:
                     log.error(f"Error waiting for selector {self.wait_selector}: {e}")
 
+            page.wait_for_timeout(self.wait)
             # In case we didn't catch a document type somehow
             final_response = final_response if final_response else first_response
             if not final_response:
@@ -263,6 +302,7 @@ class CamoufoxEngine:
                 except Exception as e:
                     log.error(f"Error waiting for selector {self.wait_selector}: {e}")
 
+            await page.wait_for_timeout(self.wait)
             # In case we didn't catch a document type somehow
             final_response = final_response if final_response else first_response
             if not final_response:
@@ -273,7 +313,7 @@ class CamoufoxEngine:
             # PlayWright API sometimes give empty status text for some reason!
             status_text = final_response.status_text or StatusText.get(final_response.status)
 
-            history = self._process_response_history(first_response)
+            history = await self._async_process_response_history(first_response)
             try:
                 page_content = await page.content()
             except Exception as e:

scrapling/engines/pw.py

@@ -21,6 +21,7 @@ class PlaywrightEngine:
             useragent: Optional[str] = None,
             network_idle: bool = False,
             timeout: Optional[float] = 30000,
+            wait: Optional[int] = 0,
             page_action: Callable = None,
             wait_selector: Optional[str] = None,
             locale: Optional[str] = 'en-US',
@@ -46,6 +47,7 @@ class PlaywrightEngine:
         :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
         :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000
+        :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning `Response` object.
         :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
         :param wait_selector: Wait for a specific css selector to be in a specific state.
         :param locale: Set the locale for the browser if wanted. The default value is `en-US`.
@@ -76,6 +78,7 @@ class PlaywrightEngine:
         self.cdp_url = cdp_url
         self.useragent = useragent
         self.timeout = check_type_validity(timeout, [int, float], 30000)
+        self.wait = check_type_validity(wait, [int, float], 0)
         if page_action is not None:
             if callable(page_action):
                 self.page_action = page_action
@@ -126,7 +129,7 @@ class PlaywrightEngine:
 
         return cdp_url
 
-    @lru_cache(126, typed=True)
+    @lru_cache(32, typed=True)
     def __set_flags(self):
         """Returns the flags that will be used while launching the browser if stealth mode is enabled"""
         flags = DEFAULT_STEALTH_FLAGS
@@ -169,7 +172,7 @@ class PlaywrightEngine:
 
         return context_kwargs
 
-    @lru_cache(10)
+    @lru_cache(1)
     def __stealth_scripts(self):
         # Basic bypasses nothing fancy as I'm still working on it
         # But with adding these bypasses to the above config, it bypasses many online tests like
@@ -220,6 +223,38 @@ class PlaywrightEngine:
 
         return history
 
+    async def _async_process_response_history(self, first_response):
+        """Process response history to build a list of Response objects"""
+        history = []
+        current_request = first_response.request.redirected_from
+
+        try:
+            while current_request:
+                try:
+                    current_response = await current_request.response()
+                    history.insert(0, Response(
+                        url=current_request.url,
+                        # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
+                        text='',
+                        body=b'',
+                        status=current_response.status if current_response else 301,
+                        reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301),
+                        encoding=current_response.headers.get('content-type', '') or 'utf-8',
+                        cookies={},
+                        headers=await current_response.all_headers() if current_response else {},
+                        request_headers=await current_request.all_headers(),
+                        **self.adaptor_arguments
+                    ))
+                except Exception as e:
+                    log.error(f"Error processing redirect: {e}")
+                    break
+
+                current_request = current_request.redirected_from
+        except Exception as e:
+            log.error(f"Error processing response history: {e}")
+
+        return history
+
     def fetch(self, url: str) -> Response:
         """Opens up the browser and do your request based on your chosen options.
 
@@ -289,6 +324,7 @@ class PlaywrightEngine:
                 except Exception as e:
                     log.error(f"Error waiting for selector {self.wait_selector}: {e}")
 
+            page.wait_for_timeout(self.wait)
             # In case we didn't catch a document type somehow
             final_response = final_response if final_response else first_response
             if not final_response:
@@ -392,6 +428,7 @@ class PlaywrightEngine:
                 except Exception as e:
                     log.error(f"Error waiting for selector {self.wait_selector}: {e}")
 
+            await page.wait_for_timeout(self.wait)
             # In case we didn't catch a document type somehow
             final_response = final_response if final_response else first_response
             if not final_response:
@@ -402,7 +439,7 @@ class PlaywrightEngine:
             # PlayWright API sometimes give empty status text for some reason!
             status_text = final_response.status_text or StatusText.get(final_response.status)
 
-            history = self._process_response_history(first_response)
+            history = await self._async_process_response_history(first_response)
             try:
                 page_content = await page.content()
             except Exception as e:

scrapling/engines/static.py

@@ -7,7 +7,7 @@ from scrapling.core.utils import log, lru_cache
 from .toolbelt import Response, generate_convincing_referer, generate_headers
 
 
-@lru_cache(5, typed=True)  # Singleton easily
+@lru_cache(2, typed=True)  # Singleton easily
 class StaticEngine:
     def __init__(
             self, url: str, proxy: Optional[str] = None, stealthy_headers: bool = True, follow_redirects: bool = True,

scrapling/engines/toolbelt/custom.py

@@ -16,7 +16,7 @@ class ResponseEncoding:
     __ISO_8859_1_CONTENT_TYPES = {"text/plain", "text/html", "text/css", "text/javascript"}
 
     @classmethod
-    @lru_cache(maxsize=256)
+    @lru_cache(maxsize=128)
     def __parse_content_type(cls, header_value: str) -> Tuple[str, Dict[str, str]]:
         """Parse content type and parameters from a content-type header value.
 
@@ -38,7 +38,7 @@ class ResponseEncoding:
         return content_type, params
 
     @classmethod
-    @lru_cache(maxsize=256)
+    @lru_cache(maxsize=128)
     def get_value(cls, content_type: Optional[str], text: Optional[str] = 'test') -> str:
         """Determine the appropriate character encoding from a content-type header.
 
@@ -105,41 +105,77 @@ class Response(Adaptor):
 
 
 class BaseFetcher:
-    def __init__(
-            self, huge_tree: bool = True, keep_comments: Optional[bool] = False, auto_match: Optional[bool] = True,
-            storage: Any = SQLiteStorageSystem, storage_args: Optional[Dict] = None,
-            automatch_domain: Optional[str] = None, keep_cdata: Optional[bool] = False,
-    ):
-        """Arguments below are the same from the Adaptor class so you can pass them directly, the rest of Adaptor's arguments
-        are detected and passed automatically from the Fetcher based on the response for accessibility.
-
-        :param huge_tree: Enabled by default, should always be enabled when parsing large HTML documents. This controls
-            libxml2 feature that forbids parsing certain large documents to protect from possible memory exhaustion.
-        :param keep_comments: While parsing the HTML body, drop comments or not. Disabled by default for obvious reasons
-        :param keep_cdata: While parsing the HTML body, drop cdata or not. Disabled by default for cleaner HTML.
-        :param auto_match: Globally turn-off the auto-match feature in all functions, this argument takes higher
-            priority over all auto-match related arguments/functions in the class.
-        :param storage: The storage class to be passed for auto-matching functionalities, see ``Docs`` for more info.
-        :param storage_args: A dictionary of ``argument->value`` pairs to be passed for the storage class.
-            If empty, default values will be used.
-        :param automatch_domain: For cases where you want to automatch selectors across different websites as if they were on the same website, use this argument to unify them.
-            Otherwise, the domain of the request is used by default.
+    __slots__ = ()
+    huge_tree: bool = True
+    auto_match: Optional[bool] = False
+    storage: Any = SQLiteStorageSystem
+    keep_cdata: Optional[bool] = False
+    storage_args: Optional[Dict] = None
+    keep_comments: Optional[bool] = False
+    automatch_domain: Optional[str] = None
+    parser_keywords: Tuple = ('huge_tree', 'auto_match', 'storage', 'keep_cdata', 'storage_args', 'keep_comments', 'automatch_domain',)  # Left open for the user
+
+    def __init__(self, *args, **kwargs):
+        # For backward-compatibility before 0.2.99
+        args_str = ", ".join(args) or ''
+        kwargs_str = ", ".join(f'{k}={v}' for k, v in kwargs.items()) or ''
+        if args_str:
+            args_str += ', '
+
+        log.warning(f'This logic is deprecated now, and have no effect; It will be removed with v0.3. Use `{self.__class__.__name__}.configure({args_str}{kwargs_str})` instead before fetching')
+        pass
+
+    @classmethod
+    def display_config(cls):
+        return dict(
+            huge_tree=cls.huge_tree,
+            keep_comments=cls.keep_comments,
+            keep_cdata=cls.keep_cdata,
+            auto_match=cls.auto_match,
+            storage=cls.storage,
+            storage_args=cls.storage_args,
+            automatch_domain=cls.automatch_domain,
+        )
+
+    @classmethod
+    def configure(cls, **kwargs):
+        """Set multiple arguments for the parser at once globally
+
+        :param kwargs: The keywords can be any arguments of the following: huge_tree, keep_comments, keep_cdata, auto_match, storage, storage_args, automatch_domain
         """
+        for key, value in kwargs.items():
+            key = key.strip().lower()
+            if hasattr(cls, key):
+                if key in cls.parser_keywords:
+                    setattr(cls, key, value)
+                else:
+                    # Yup, no fun allowed LOL
+                    raise AttributeError(f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?')
+            else:
+                raise ValueError(f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?')
+
+        if not kwargs:
+            raise AttributeError(f'You must pass a keyword to configure, current keywords: {cls.parser_keywords}?')
+
+    @classmethod
+    def _generate_parser_arguments(cls) -> Dict:
         # Adaptor class parameters
         # I won't validate Adaptor's class parameters here again, I will leave it to be validated later
-        self.adaptor_arguments = dict(
-            huge_tree=huge_tree,
-            keep_comments=keep_comments,
-            keep_cdata=keep_cdata,
-            auto_match=auto_match,
-            storage=storage,
-            storage_args=storage_args
+        parser_arguments = dict(
+            huge_tree=cls.huge_tree,
+            keep_comments=cls.keep_comments,
+            keep_cdata=cls.keep_cdata,
+            auto_match=cls.auto_match,
+            storage=cls.storage,
+            storage_args=cls.storage_args
         )
-        if automatch_domain:
-            if type(automatch_domain) is not str:
+        if cls.automatch_domain:
+            if type(cls.automatch_domain) is not str:
                 log.warning('[Ignored] The argument "automatch_domain" must be of string type')
             else:
-                self.adaptor_arguments.update({'automatch_domain': automatch_domain})
+                parser_arguments.update({'automatch_domain': cls.automatch_domain})
+
+        return parser_arguments
 
 
 class StatusText:

scrapling/engines/toolbelt/fingerprints.py

@@ -12,7 +12,7 @@ from scrapling.core._types import Dict, Union
 from scrapling.core.utils import lru_cache
 
 
-@lru_cache(128, typed=True)
+@lru_cache(10, typed=True)
 def generate_convincing_referer(url: str) -> str:
     """Takes the domain from the URL without the subdomain/suffix and make it look like you were searching google for this website
 
@@ -26,7 +26,7 @@ def generate_convincing_referer(url: str) -> str:
     return f'https://www.google.com/search?q={website_name}'
 
 
-@lru_cache(128, typed=True)
+@lru_cache(1, typed=True)
 def get_os_name() -> Union[str, None]:
     """Get the current OS name in the same format needed for browserforge
 

scrapling/engines/toolbelt/navigation.py

@@ -110,7 +110,7 @@ def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:
         raise ValueError(f"Invalid CDP URL: {str(e)}")
 
 
-@lru_cache(126, typed=True)
+@lru_cache(10, typed=True)
 def js_bypass_path(filename: str) -> str:
     """Takes the base filename of JS file inside the `bypasses` folder then return the full path of it