scrapling 0.2.98__py3-none-any.whl → 0.2.99__py3-none-any.whl

scrapling/__init__.py

@@ -1,6 +1,6 @@
 
 __author__ = "Karim Shoair (karim.shoair@pm.me)"
-__version__ = "0.2.98"
+__version__ = "0.2.99"
 __copyright__ = "Copyright (c) 2024 Karim Shoair"
 
 

scrapling/defaults.py

@@ -1,19 +1,25 @@
-# If you are going to use Fetchers with the default settings, import them from this file instead for a cleaner looking code
+# Left this file for backward-compatibility before 0.2.99
+from scrapling.core.utils import log
+
 
 # A lightweight approach to create lazy loader for each import for backward compatibility
 # This will reduces initial memory footprint significantly (only loads what's used)
 def __getattr__(name):
     if name == 'Fetcher':
         from scrapling.fetchers import Fetcher as cls
-        return cls()
+        log.warning('This import is deprecated now and it will be removed with v0.3. Use `from scrapling.fetchers import Fetcher` instead')
+        return cls
     elif name == 'AsyncFetcher':
         from scrapling.fetchers import AsyncFetcher as cls
-        return cls()
+        log.warning('This import is deprecated now and it will be removed with v0.3. Use `from scrapling.fetchers import AsyncFetcher` instead')
+        return cls
     elif name == 'StealthyFetcher':
         from scrapling.fetchers import StealthyFetcher as cls
-        return cls()
+        log.warning('This import is deprecated now and it will be removed with v0.3. Use `from scrapling.fetchers import StealthyFetcher` instead')
+        return cls
     elif name == 'PlayWrightFetcher':
         from scrapling.fetchers import PlayWrightFetcher as cls
-        return cls()
+        log.warning('This import is deprecated now and it will be removed with v0.3. Use `from scrapling.fetchers import PlayWrightFetcher` instead')
+        return cls
     else:
         raise AttributeError(f"module 'scrapling' has no attribute '{name}'")

scrapling/engines/camo.py

@@ -16,12 +16,13 @@ from scrapling.engines.toolbelt import (Response, StatusText,
 class CamoufoxEngine:
     def __init__(
             self, headless: Union[bool, Literal['virtual']] = True, block_images: bool = False, disable_resources: bool = False,
-            block_webrtc: bool = False, allow_webgl: bool = True, network_idle: bool = False, humanize: Union[bool, float] = True,
+            block_webrtc: bool = False, allow_webgl: bool = True, network_idle: bool = False, humanize: Union[bool, float] = True, wait: Optional[int] = 0,
             timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, addons: Optional[List[str]] = None,
             wait_selector_state: SelectorWaitStates = 'attached', google_search: bool = True, extra_headers: Optional[Dict[str, str]] = None,
             proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: bool = False, disable_ads: bool = False,
             geoip: bool = False,
             adaptor_arguments: Dict = None,
+            additional_arguments: Dict = None
     ):
         """An engine that utilizes Camoufox library, check the `StealthyFetcher` class for more documentation.
 
@@ -38,6 +39,7 @@ class CamoufoxEngine:
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
         :param disable_ads: Disabled by default, this installs `uBlock Origin` addon on the browser if enabled.
         :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS.
+        :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning `Response` object.
         :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000
         :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
         :param wait_selector: Wait for a specific css selector to be in a specific state.
@@ -48,6 +50,7 @@ class CamoufoxEngine:
         :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
         :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
         :param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
+        :param additional_arguments: Additional arguments to be passed to Camoufox as additional settings and it takes higher priority than Scrapling's settings.
         """
         self.headless = headless
         self.block_images = bool(block_images)
@@ -60,10 +63,12 @@ class CamoufoxEngine:
         self.disable_ads = bool(disable_ads)
         self.geoip = bool(geoip)
         self.extra_headers = extra_headers or {}
+        self.additional_arguments = additional_arguments or {}
         self.proxy = construct_proxy_dict(proxy)
         self.addons = addons or []
         self.humanize = humanize
         self.timeout = check_type_validity(timeout, [int, float], 30000)
+        self.wait = check_type_validity(wait, [int, float], 0)
 
         # Page action callable validation
         self.page_action = None
@@ -92,6 +97,7 @@ class CamoufoxEngine:
             "block_webrtc": self.block_webrtc,
             "block_images": self.block_images,  # Careful! it makes some websites doesn't finish loading at all like stackoverflow even in headful
             "os": None if self.os_randomize else get_os_name(),
+            **self.additional_arguments
         }
 
     def _process_response_history(self, first_response):
@@ -126,6 +132,38 @@ class CamoufoxEngine:
 
         return history
 
+    async def _async_process_response_history(self, first_response):
+        """Process response history to build a list of Response objects"""
+        history = []
+        current_request = first_response.request.redirected_from
+
+        try:
+            while current_request:
+                try:
+                    current_response = await current_request.response()
+                    history.insert(0, Response(
+                        url=current_request.url,
+                        # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
+                        text='',
+                        body=b'',
+                        status=current_response.status if current_response else 301,
+                        reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301),
+                        encoding=current_response.headers.get('content-type', '') or 'utf-8',
+                        cookies={},
+                        headers=await current_response.all_headers() if current_response else {},
+                        request_headers=await current_request.all_headers(),
+                        **self.adaptor_arguments
+                    ))
+                except Exception as e:
+                    log.error(f"Error processing redirect: {e}")
+                    break
+
+                current_request = current_request.redirected_from
+        except Exception as e:
+            log.error(f"Error processing response history: {e}")
+
+        return history
+
     def fetch(self, url: str) -> Response:
         """Opens up the browser and do your request based on your chosen options.
 
@@ -177,6 +215,7 @@ class CamoufoxEngine:
                 except Exception as e:
                     log.error(f"Error waiting for selector {self.wait_selector}: {e}")
 
+            page.wait_for_timeout(self.wait)
             # In case we didn't catch a document type somehow
             final_response = final_response if final_response else first_response
             if not final_response:
@@ -263,6 +302,7 @@ class CamoufoxEngine:
                 except Exception as e:
                     log.error(f"Error waiting for selector {self.wait_selector}: {e}")
 
+            await page.wait_for_timeout(self.wait)
             # In case we didn't catch a document type somehow
             final_response = final_response if final_response else first_response
             if not final_response:
@@ -273,7 +313,7 @@ class CamoufoxEngine:
             # PlayWright API sometimes give empty status text for some reason!
             status_text = final_response.status_text or StatusText.get(final_response.status)
 
-            history = self._process_response_history(first_response)
+            history = await self._async_process_response_history(first_response)
             try:
                 page_content = await page.content()
             except Exception as e:

scrapling/engines/pw.py

@@ -21,6 +21,7 @@ class PlaywrightEngine:
             useragent: Optional[str] = None,
             network_idle: bool = False,
             timeout: Optional[float] = 30000,
+            wait: Optional[int] = 0,
             page_action: Callable = None,
             wait_selector: Optional[str] = None,
             locale: Optional[str] = 'en-US',
@@ -46,6 +47,7 @@ class PlaywrightEngine:
         :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
         :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000
+        :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning `Response` object.
         :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
         :param wait_selector: Wait for a specific css selector to be in a specific state.
         :param locale: Set the locale for the browser if wanted. The default value is `en-US`.
@@ -76,6 +78,7 @@ class PlaywrightEngine:
         self.cdp_url = cdp_url
         self.useragent = useragent
         self.timeout = check_type_validity(timeout, [int, float], 30000)
+        self.wait = check_type_validity(wait, [int, float], 0)
         if page_action is not None:
             if callable(page_action):
                 self.page_action = page_action
@@ -220,6 +223,38 @@ class PlaywrightEngine:
 
         return history
 
+    async def _async_process_response_history(self, first_response):
+        """Process response history to build a list of Response objects"""
+        history = []
+        current_request = first_response.request.redirected_from
+
+        try:
+            while current_request:
+                try:
+                    current_response = await current_request.response()
+                    history.insert(0, Response(
+                        url=current_request.url,
+                        # using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
+                        text='',
+                        body=b'',
+                        status=current_response.status if current_response else 301,
+                        reason=(current_response.status_text or StatusText.get(current_response.status)) if current_response else StatusText.get(301),
+                        encoding=current_response.headers.get('content-type', '') or 'utf-8',
+                        cookies={},
+                        headers=await current_response.all_headers() if current_response else {},
+                        request_headers=await current_request.all_headers(),
+                        **self.adaptor_arguments
+                    ))
+                except Exception as e:
+                    log.error(f"Error processing redirect: {e}")
+                    break
+
+                current_request = current_request.redirected_from
+        except Exception as e:
+            log.error(f"Error processing response history: {e}")
+
+        return history
+
     def fetch(self, url: str) -> Response:
         """Opens up the browser and do your request based on your chosen options.
 
@@ -289,6 +324,7 @@ class PlaywrightEngine:
                 except Exception as e:
                     log.error(f"Error waiting for selector {self.wait_selector}: {e}")
 
+            page.wait_for_timeout(self.wait)
             # In case we didn't catch a document type somehow
             final_response = final_response if final_response else first_response
             if not final_response:
@@ -392,6 +428,7 @@ class PlaywrightEngine:
                 except Exception as e:
                     log.error(f"Error waiting for selector {self.wait_selector}: {e}")
 
+            await page.wait_for_timeout(self.wait)
             # In case we didn't catch a document type somehow
             final_response = final_response if final_response else first_response
             if not final_response:
@@ -402,7 +439,7 @@ class PlaywrightEngine:
             # PlayWright API sometimes give empty status text for some reason!
             status_text = final_response.status_text or StatusText.get(final_response.status)
 
-            history = self._process_response_history(first_response)
+            history = await self._async_process_response_history(first_response)
             try:
                 page_content = await page.content()
             except Exception as e:

scrapling/engines/toolbelt/custom.py

@@ -105,41 +105,77 @@ class Response(Adaptor):
 
 
 class BaseFetcher:
-    def __init__(
-            self, huge_tree: bool = True, keep_comments: Optional[bool] = False, auto_match: Optional[bool] = True,
-            storage: Any = SQLiteStorageSystem, storage_args: Optional[Dict] = None,
-            automatch_domain: Optional[str] = None, keep_cdata: Optional[bool] = False,
-    ):
-        """Arguments below are the same from the Adaptor class so you can pass them directly, the rest of Adaptor's arguments
-        are detected and passed automatically from the Fetcher based on the response for accessibility.
-
-        :param huge_tree: Enabled by default, should always be enabled when parsing large HTML documents. This controls
-            libxml2 feature that forbids parsing certain large documents to protect from possible memory exhaustion.
-        :param keep_comments: While parsing the HTML body, drop comments or not. Disabled by default for obvious reasons
-        :param keep_cdata: While parsing the HTML body, drop cdata or not. Disabled by default for cleaner HTML.
-        :param auto_match: Globally turn-off the auto-match feature in all functions, this argument takes higher
-            priority over all auto-match related arguments/functions in the class.
-        :param storage: The storage class to be passed for auto-matching functionalities, see ``Docs`` for more info.
-        :param storage_args: A dictionary of ``argument->value`` pairs to be passed for the storage class.
-            If empty, default values will be used.
-        :param automatch_domain: For cases where you want to automatch selectors across different websites as if they were on the same website, use this argument to unify them.
-            Otherwise, the domain of the request is used by default.
+    __slots__ = ()
+    huge_tree: bool = True
+    auto_match: Optional[bool] = False
+    storage: Any = SQLiteStorageSystem
+    keep_cdata: Optional[bool] = False
+    storage_args: Optional[Dict] = None
+    keep_comments: Optional[bool] = False
+    automatch_domain: Optional[str] = None
+    parser_keywords: Tuple = ('huge_tree', 'auto_match', 'storage', 'keep_cdata', 'storage_args', 'keep_comments', 'automatch_domain',)  # Left open for the user
+
+    def __init__(self, *args, **kwargs):
+        # For backward-compatibility before 0.2.99
+        args_str = ", ".join(args) or ''
+        kwargs_str = ", ".join(f'{k}={v}' for k, v in kwargs.items()) or ''
+        if args_str:
+            args_str += ', '
+
+        log.warning(f'This logic is deprecated now, and have no effect; It will be removed with v0.3. Use `{self.__class__.__name__}.configure({args_str}{kwargs_str})` instead before fetching')
+        pass
+
+    @classmethod
+    def display_config(cls):
+        return dict(
+            huge_tree=cls.huge_tree,
+            keep_comments=cls.keep_comments,
+            keep_cdata=cls.keep_cdata,
+            auto_match=cls.auto_match,
+            storage=cls.storage,
+            storage_args=cls.storage_args,
+            automatch_domain=cls.automatch_domain,
+        )
+
+    @classmethod
+    def configure(cls, **kwargs):
+        """Set multiple arguments for the parser at once globally
+
+        :param kwargs: The keywords can be any arguments of the following: huge_tree, keep_comments, keep_cdata, auto_match, storage, storage_args, automatch_domain
         """
+        for key, value in kwargs.items():
+            key = key.strip().lower()
+            if hasattr(cls, key):
+                if key in cls.parser_keywords:
+                    setattr(cls, key, value)
+                else:
+                    # Yup, no fun allowed LOL
+                    raise AttributeError(f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?')
+            else:
+                raise ValueError(f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?')
+
+        if not kwargs:
+            raise AttributeError(f'You must pass a keyword to configure, current keywords: {cls.parser_keywords}?')
+
+    @classmethod
+    def _generate_parser_arguments(cls) -> Dict:
         # Adaptor class parameters
         # I won't validate Adaptor's class parameters here again, I will leave it to be validated later
-        self.adaptor_arguments = dict(
-            huge_tree=huge_tree,
-            keep_comments=keep_comments,
-            keep_cdata=keep_cdata,
-            auto_match=auto_match,
-            storage=storage,
-            storage_args=storage_args
+        parser_arguments = dict(
+            huge_tree=cls.huge_tree,
+            keep_comments=cls.keep_comments,
+            keep_cdata=cls.keep_cdata,
+            auto_match=cls.auto_match,
+            storage=cls.storage,
+            storage_args=cls.storage_args
         )
-        if automatch_domain:
-            if type(automatch_domain) is not str:
+        if cls.automatch_domain:
+            if type(cls.automatch_domain) is not str:
                 log.warning('[Ignored] The argument "automatch_domain" must be of string type')
             else:
-                self.adaptor_arguments.update({'automatch_domain': automatch_domain})
+                parser_arguments.update({'automatch_domain': cls.automatch_domain})
+
+        return parser_arguments
 
 
 class StatusText: