scrapling 0.2.7__py3-none-any.whl → 0.2.9__py3-none-any.whl

scrapling/__init__.py

@@ -1,11 +1,12 @@
 # Declare top-level shortcuts
-from scrapling.fetchers import Fetcher, StealthyFetcher, PlayWrightFetcher, CustomFetcher
+from scrapling.core.custom_types import AttributesHandler, TextHandler
+from scrapling.fetchers import (AsyncFetcher, CustomFetcher, Fetcher,
+                                PlayWrightFetcher, StealthyFetcher)
 from scrapling.parser import Adaptor, Adaptors
-from scrapling.core.custom_types import TextHandler, AttributesHandler
 
 __author__ = "Karim Shoair (karim.shoair@pm.me)"
-__version__ = "0.2.7"
+__version__ = "0.2.9"
 __copyright__ = "Copyright (c) 2024 Karim Shoair"
 
 
-__all__ = ['Adaptor', 'Fetcher', 'StealthyFetcher', 'PlayWrightFetcher']
+__all__ = ['Adaptor', 'Fetcher', 'AsyncFetcher', 'StealthyFetcher', 'PlayWrightFetcher']

scrapling/core/_types.py

@@ -2,9 +2,8 @@
 Type definitions for type checking purposes.
 """
 
-from typing import (
-    Dict, Optional, Union, Callable, Any, List, Tuple, Pattern, Generator, Iterable, Type, TYPE_CHECKING, Literal
-)
+from typing import (TYPE_CHECKING, Any, Callable, Dict, Generator, Iterable,
+                    List, Literal, Optional, Pattern, Tuple, Type, Union)
 
 try:
     from typing import Protocol

scrapling/core/custom_types.py

@@ -1,24 +1,83 @@
 import re
-from types import MappingProxyType
 from collections.abc import Mapping
+from types import MappingProxyType
 
-from scrapling.core.utils import _is_iterable, flatten
-from scrapling.core._types import Dict, List, Union, Pattern, SupportsIndex
-
-from orjson import loads, dumps
+from orjson import dumps, loads
 from w3lib.html import replace_entities as _replace_entities
 
+from scrapling.core._types import Dict, List, Pattern, SupportsIndex, Union
+from scrapling.core.utils import _is_iterable, flatten
+
 
 class TextHandler(str):
     """Extends standard Python string by adding more functionality"""
     __slots__ = ()
 
     def __new__(cls, string):
-        # Because str is immutable and we can't override __init__
-        if type(string) is str:
+        if isinstance(string, str):
             return super().__new__(cls, string)
-        else:
-            return super().__new__(cls, '')
+        return super().__new__(cls, '')
+
+    # Make methods from original `str` class return `TextHandler` instead of returning `str` again
+    # Of course, this stupid workaround is only so we can keep the auto-completion working without issues in your IDE
+    # and I made sonnet write it for me :)
+    def strip(self, chars=None):
+        return TextHandler(super().strip(chars))
+
+    def lstrip(self, chars=None):
+        return TextHandler(super().lstrip(chars))
+
+    def rstrip(self, chars=None):
+        return TextHandler(super().rstrip(chars))
+
+    def capitalize(self):
+        return TextHandler(super().capitalize())
+
+    def casefold(self):
+        return TextHandler(super().casefold())
+
+    def center(self, width, fillchar=' '):
+        return TextHandler(super().center(width, fillchar))
+
+    def expandtabs(self, tabsize=8):
+        return TextHandler(super().expandtabs(tabsize))
+
+    def format(self, *args, **kwargs):
+        return TextHandler(super().format(*args, **kwargs))
+
+    def format_map(self, mapping):
+        return TextHandler(super().format_map(mapping))
+
+    def join(self, iterable):
+        return TextHandler(super().join(iterable))
+
+    def ljust(self, width, fillchar=' '):
+        return TextHandler(super().ljust(width, fillchar))
+
+    def rjust(self, width, fillchar=' '):
+        return TextHandler(super().rjust(width, fillchar))
+
+    def swapcase(self):
+        return TextHandler(super().swapcase())
+
+    def title(self):
+        return TextHandler(super().title())
+
+    def translate(self, table):
+        return TextHandler(super().translate(table))
+
+    def zfill(self, width):
+        return TextHandler(super().zfill(width))
+
+    def replace(self, old, new, count=-1):
+        return TextHandler(super().replace(old, new, count))
+
+    def upper(self):
+        return TextHandler(super().upper())
+
+    def lower(self):
+        return TextHandler(super().lower())
+    ##############
 
     def sort(self, reverse: bool = False) -> str:
         """Return a sorted version of the string"""
@@ -30,11 +89,21 @@ class TextHandler(str):
         data = re.sub(' +', ' ', data)
         return self.__class__(data.strip())
 
+    # For easy copy-paste from Scrapy/parsel code when needed :)
+    def get(self, default=None):
+        return self
+
+    def get_all(self):
+        return self
+
+    extract = get_all
+    extract_first = get
+
     def json(self) -> Dict:
         """Return json response if the response is jsonable otherwise throw error"""
-        # Using __str__ function as a workaround for orjson issue with subclasses of str
+        # Using str function as a workaround for orjson issue with subclasses of str
         # Check this out: https://github.com/ijl/orjson/issues/445
-        return loads(self.__str__())
+        return loads(str(self))
 
     def re(
             self, regex: Union[str, Pattern[str]], replace_entities: bool = True, clean_match: bool = False,
@@ -127,6 +196,19 @@ class TextHandlers(List[TextHandler]):
                 return result
         return default
 
+    # For easy copy-paste from Scrapy/parsel code when needed :)
+    def get(self, default=None):
+        """Returns the first item of the current list
+        :param default: the default value to return if the current list is empty
+        """
+        return self[0] if len(self) > 0 else default
+
+    def extract(self):
+        return self
+
+    extract_first = get
+    get_all = extract
+
 
 class AttributesHandler(Mapping):
     """A read-only mapping to use instead of the standard dictionary for the speed boost but at the same time I use it to add more functionalities.

scrapling/core/storage_adaptors.py

@@ -1,16 +1,15 @@
-import orjson
 import sqlite3
-import logging
 import threading
-from hashlib import sha256
 from abc import ABC, abstractmethod
+from hashlib import sha256
 
-from scrapling.core._types import Dict, Optional, Union
-from scrapling.core.utils import _StorageTools, cache
-
+import orjson
 from lxml import html
 from tldextract import extract as tld
 
+from scrapling.core._types import Dict, Optional, Union
+from scrapling.core.utils import _StorageTools, log, lru_cache
+
 
 class StorageSystemMixin(ABC):
     # If you want to make your own storage system, you have to inherit from this
@@ -20,7 +19,7 @@ class StorageSystemMixin(ABC):
         """
         self.url = url
 
-    @cache(None, typed=True)
+    @lru_cache(None, typed=True)
     def _get_base_url(self, default_value: str = 'default') -> str:
         if not self.url or type(self.url) is not str:
             return default_value
@@ -52,7 +51,7 @@ class StorageSystemMixin(ABC):
         raise NotImplementedError('Storage system must implement `save` method')
 
     @staticmethod
-    @cache(None, typed=True)
+    @lru_cache(None, typed=True)
     def _get_hash(identifier: str) -> str:
         """If you want to hash identifier in your storage system, use this safer"""
         identifier = identifier.lower().strip()
@@ -64,7 +63,7 @@ class StorageSystemMixin(ABC):
         return f"{hash_value}_{len(identifier)}"  # Length to reduce collision chance
 
 
-@cache(None, typed=True)
+@lru_cache(None, typed=True)
 class SQLiteStorageSystem(StorageSystemMixin):
     """The recommended system to use, it's race condition safe and thread safe.
     Mainly built so the library can run in threaded frameworks like scrapy or threaded tools
@@ -86,7 +85,7 @@ class SQLiteStorageSystem(StorageSystemMixin):
         self.connection.execute("PRAGMA journal_mode=WAL")
         self.cursor = self.connection.cursor()
         self._setup_database()
-        logging.debug(
+        log.debug(
             f'Storage system loaded with arguments (storage_file="{storage_file}", url="{url}")'
         )
 

scrapling/core/translator.py

@@ -10,15 +10,14 @@ So you don't have to learn a new selectors/api method like what bs4 done with so
 
 import re
 
-from w3lib.html import HTML5_WHITESPACE
-from scrapling.core.utils import cache
-from scrapling.core._types import Any, Optional, Protocol, Self
-
-from cssselect.xpath import ExpressionError
-from cssselect.xpath import XPathExpr as OriginalXPathExpr
 from cssselect import HTMLTranslator as OriginalHTMLTranslator
 from cssselect.parser import Element, FunctionalPseudoElement, PseudoElement
+from cssselect.xpath import ExpressionError
+from cssselect.xpath import XPathExpr as OriginalXPathExpr
+from w3lib.html import HTML5_WHITESPACE
 
+from scrapling.core._types import Any, Optional, Protocol, Self
+from scrapling.core.utils import lru_cache
 
 regex = f"[{HTML5_WHITESPACE}]+"
 replace_html5_whitespaces = re.compile(regex).sub
@@ -140,6 +139,6 @@ class TranslatorMixin:
 
 
 class HTMLTranslator(TranslatorMixin, OriginalHTMLTranslator):
-    @cache(maxsize=256)
+    @lru_cache(maxsize=256)
     def css_to_xpath(self, css: str, prefix: str = "descendant-or-self::") -> str:
         return super().css_to_xpath(css, prefix)

scrapling/core/utils.py

@@ -1,23 +1,45 @@
-import re
 import logging
+import re
 from itertools import chain
-# Using cache on top of a class is brilliant way to achieve Singleton design pattern without much code
-from functools import lru_cache as cache  # functools.cache is available on Python 3.9+ only so let's keep lru_cache
-
-from scrapling.core._types import Dict, Iterable, Any, Union
 
 import orjson
 from lxml import html
 
+from scrapling.core._types import Any, Dict, Iterable, Union
+
+# Using cache on top of a class is brilliant way to achieve Singleton design pattern without much code
+# functools.cache is available on Python 3.9+ only so let's keep lru_cache
+from functools import lru_cache  # isort:skip
+
 html_forbidden = {html.HtmlComment, }
-logging.basicConfig(
-        level=logging.ERROR,
-        format='%(asctime)s - %(levelname)s - %(message)s',
-        handlers=[
-            logging.StreamHandler()
-        ]
+
+
+@lru_cache(1, typed=True)
+def setup_logger():
+    """Create and configure a logger with a standard format.
+
+    :returns: logging.Logger: Configured logger instance
+    """
+    logger = logging.getLogger('scrapling')
+    logger.setLevel(logging.INFO)
+
+    formatter = logging.Formatter(
+        fmt="[%(asctime)s] %(levelname)s: %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S"
     )
 
+    console_handler = logging.StreamHandler()
+    console_handler.setFormatter(formatter)
+
+    # Add handler to logger (if not already added)
+    if not logger.handlers:
+        logger.addHandler(console_handler)
+
+    return logger
+
+
+log = setup_logger()
+
 
 def is_jsonable(content: Union[bytes, str]) -> bool:
     if type(content) is bytes:
@@ -30,23 +52,6 @@ def is_jsonable(content: Union[bytes, str]) -> bool:
         return False
 
 
-@cache(None, typed=True)
-def setup_basic_logging(level: str = 'debug'):
-    levels = {
-        'debug': logging.DEBUG,
-        'info': logging.INFO,
-        'warning': logging.WARNING,
-        'error': logging.ERROR,
-        'critical': logging.CRITICAL
-    }
-    formatter = logging.Formatter("[%(asctime)s] %(levelname)s: %(message)s", "%Y-%m-%d %H:%M:%S")
-    lvl = levels[level.lower()]
-    handler = logging.StreamHandler()
-    handler.setFormatter(formatter)
-    # Configure the root logger
-    logging.basicConfig(level=lvl, handlers=[handler])
-
-
 def flatten(lst: Iterable):
     return list(chain.from_iterable(lst))
 
@@ -94,7 +99,7 @@ class _StorageTools:
         parent = element.getparent()
         return tuple(
             (element.tag,) if parent is None else (
-                    cls._get_element_path(parent) + (element.tag,)
+                cls._get_element_path(parent) + (element.tag,)
             )
         )
 
@@ -110,7 +115,7 @@ class _StorageTools:
 #     return _impl
 
 
-@cache(None, typed=True)
+@lru_cache(None, typed=True)
 def clean_spaces(string):
     string = string.replace('\t', ' ')
     string = re.sub('[\n|\r]', '', string)

scrapling/defaults.py

@@ -1,6 +1,7 @@
-from .fetchers import Fetcher, StealthyFetcher, PlayWrightFetcher
+from .fetchers import AsyncFetcher, Fetcher, PlayWrightFetcher, StealthyFetcher
 
 # If you are going to use Fetchers with the default settings, import them from this file instead for a cleaner looking code
 Fetcher = Fetcher()
+AsyncFetcher = AsyncFetcher()
 StealthyFetcher = StealthyFetcher()
 PlayWrightFetcher = PlayWrightFetcher()

scrapling/engines/__init__.py

@@ -1,7 +1,7 @@
 from .camo import CamoufoxEngine
-from .static import StaticEngine
-from .pw import PlaywrightEngine
 from .constants import DEFAULT_DISABLED_RESOURCES, DEFAULT_STEALTH_FLAGS
+from .pw import PlaywrightEngine
+from .static import StaticEngine
 from .toolbelt import check_if_engine_usable
 
 __all__ = ['CamoufoxEngine', 'PlaywrightEngine']

scrapling/engines/camo.py

@@ -1,28 +1,26 @@
-import logging
-from scrapling.core._types import Union, Callable, Optional, Dict, List, Literal
-
-from scrapling.engines.toolbelt import (
-    Response,
-    do_nothing,
-    StatusText,
-    get_os_name,
-    intercept_route,
-    check_type_validity,
-    construct_proxy_dict,
-    generate_convincing_referer,
-)
-
 from camoufox import DefaultAddons
+from camoufox.async_api import AsyncCamoufox
 from camoufox.sync_api import Camoufox
 
+from scrapling.core._types import (Callable, Dict, List, Literal, Optional,
+                                   Union)
+from scrapling.core.utils import log
+from scrapling.engines.toolbelt import (Response, StatusText,
+                                        async_intercept_route,
+                                        check_type_validity,
+                                        construct_proxy_dict,
+                                        generate_convincing_referer,
+                                        get_os_name, intercept_route)
+
 
 class CamoufoxEngine:
     def __init__(
             self, headless: Optional[Union[bool, Literal['virtual']]] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False,
-            block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = False, network_idle: Optional[bool] = False, humanize: Optional[Union[bool, float]] = True,
-            timeout: Optional[float] = 30000, page_action: Callable = do_nothing, wait_selector: Optional[str] = None, addons: Optional[List[str]] = None,
+            block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = True, network_idle: Optional[bool] = False, humanize: Optional[Union[bool, float]] = True,
+            timeout: Optional[float] = 30000, page_action: Callable = None, wait_selector: Optional[str] = None, addons: Optional[List[str]] = None,
             wait_selector_state: str = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None,
             proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: Optional[bool] = None, disable_ads: Optional[bool] = True,
+            geoip: Optional[bool] = False,
             adaptor_arguments: Dict = None,
     ):
         """An engine that utilizes Camoufox library, check the `StealthyFetcher` class for more documentation.
@@ -36,13 +34,15 @@ class CamoufoxEngine:
         :param block_webrtc: Blocks WebRTC entirely.
         :param addons: List of Firefox addons to use. Must be paths to extracted addons.
         :param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.
-        :param allow_webgl: Whether to allow WebGL. To prevent leaks, only use this for special cases.
+        :param allow_webgl: Enabled by default. Disabling it WebGL not recommended as many WAFs now checks if WebGL is enabled.
         :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
         :param disable_ads: Enabled by default, this installs `uBlock Origin` addon on the browser if enabled.
         :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS.
         :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000
         :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
         :param wait_selector: Wait for a specific css selector to be in a specific state.
+        :param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, & spoof the WebRTC IP address.
+            It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region.
         :param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
         :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
         :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
@@ -58,16 +58,20 @@ class CamoufoxEngine:
         self.google_search = bool(google_search)
         self.os_randomize = bool(os_randomize)
         self.disable_ads = bool(disable_ads)
+        self.geoip = bool(geoip)
         self.extra_headers = extra_headers or {}
         self.proxy = construct_proxy_dict(proxy)
         self.addons = addons or []
         self.humanize = humanize
         self.timeout = check_type_validity(timeout, [int, float], 30000)
-        if callable(page_action):
-            self.page_action = page_action
+        if page_action is not None:
+            if callable(page_action):
+                self.page_action = page_action
+            else:
+                self.page_action = None
+                log.error('[Ignored] Argument "page_action" must be callable')
         else:
-            self.page_action = do_nothing
-            logging.error('[Ignored] Argument "page_action" must be callable')
+            self.page_action = None
 
         self.wait_selector = wait_selector
         self.wait_selector_state = wait_selector_state
@@ -81,6 +85,7 @@ class CamoufoxEngine:
         """
         addons = [] if self.disable_ads else [DefaultAddons.UBO]
         with Camoufox(
+                geoip=self.geoip,
                 proxy=self.proxy,
                 addons=self.addons,
                 exclude_addons=addons,
@@ -106,7 +111,8 @@ class CamoufoxEngine:
             if self.network_idle:
                 page.wait_for_load_state('networkidle')
 
-            page = self.page_action(page)
+            if self.page_action is not None:
+                page = self.page_action(page)
 
             if self.wait_selector and type(self.wait_selector) is str:
                 waiter = page.locator(self.wait_selector)
@@ -119,11 +125,8 @@ class CamoufoxEngine:
 
             # This will be parsed inside `Response`
             encoding = res.headers.get('content-type', '') or 'utf-8'  # default encoding
-
-            status_text = res.status_text
             # PlayWright API sometimes give empty status text for some reason!
-            if not status_text:
-                status_text = StatusText.get(res.status)
+            status_text = res.status_text or StatusText.get(res.status)
 
             response = Response(
                 url=res.url,
@@ -140,3 +143,70 @@ class CamoufoxEngine:
             page.close()
 
         return response
+
+    async def async_fetch(self, url: str) -> Response:
+        """Opens up the browser and do your request based on your chosen options.
+
+        :param url: Target url.
+        :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
+        """
+        addons = [] if self.disable_ads else [DefaultAddons.UBO]
+        async with AsyncCamoufox(
+                geoip=self.geoip,
+                proxy=self.proxy,
+                addons=self.addons,
+                exclude_addons=addons,
+                headless=self.headless,
+                humanize=self.humanize,
+                i_know_what_im_doing=True,  # To turn warnings off with the user configurations
+                allow_webgl=self.allow_webgl,
+                block_webrtc=self.block_webrtc,
+                block_images=self.block_images,  # Careful! it makes some websites doesn't finish loading at all like stackoverflow even in headful
+                os=None if self.os_randomize else get_os_name(),
+        ) as browser:
+            page = await browser.new_page()
+            page.set_default_navigation_timeout(self.timeout)
+            page.set_default_timeout(self.timeout)
+            if self.disable_resources:
+                await page.route("**/*", async_intercept_route)
+
+            if self.extra_headers:
+                await page.set_extra_http_headers(self.extra_headers)
+
+            res = await page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
+            await page.wait_for_load_state(state="domcontentloaded")
+            if self.network_idle:
+                await page.wait_for_load_state('networkidle')
+
+            if self.page_action is not None:
+                page = await self.page_action(page)
+
+            if self.wait_selector and type(self.wait_selector) is str:
+                waiter = page.locator(self.wait_selector)
+                await waiter.first.wait_for(state=self.wait_selector_state)
+                # Wait again after waiting for the selector, helpful with protections like Cloudflare
+                await page.wait_for_load_state(state="load")
+                await page.wait_for_load_state(state="domcontentloaded")
+                if self.network_idle:
+                    await page.wait_for_load_state('networkidle')
+
+            # This will be parsed inside `Response`
+            encoding = res.headers.get('content-type', '') or 'utf-8'  # default encoding
+            # PlayWright API sometimes give empty status text for some reason!
+            status_text = res.status_text or StatusText.get(res.status)
+
+            response = Response(
+                url=res.url,
+                text=await page.content(),
+                body=(await page.content()).encode('utf-8'),
+                status=res.status,
+                reason=status_text,
+                encoding=encoding,
+                cookies={cookie['name']: cookie['value'] for cookie in await page.context.cookies()},
+                headers=await res.all_headers(),
+                request_headers=await res.request.all_headers(),
+                **self.adaptor_arguments
+            )
+            await page.close()
+
+        return response

scrapling/engines/constants.py

@@ -1,5 +1,5 @@
 # Disable loading these resources for speed
-DEFAULT_DISABLED_RESOURCES = [
+DEFAULT_DISABLED_RESOURCES = {
     'font',
     'image',
     'media',
@@ -10,9 +10,9 @@ DEFAULT_DISABLED_RESOURCES = [
     'websocket',
     'csp_report',
     'stylesheet',
-]
+}
 
-DEFAULT_STEALTH_FLAGS = [
+DEFAULT_STEALTH_FLAGS = (
     # Explanation: https://peter.sh/experiments/chromium-command-line-switches/
     # Generally this will make the browser faster and less detectable
     '--no-pings',
@@ -87,7 +87,7 @@ DEFAULT_STEALTH_FLAGS = [
     '--enable-features=NetworkService,NetworkServiceInProcess,TrustTokens,TrustTokensAlwaysAllowIssuance',
     '--blink-settings=primaryHoverType=2,availableHoverTypes=2,primaryPointerType=4,availablePointerTypes=4',
     '--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process,TranslateUI,BlinkGenPropertyTrees',
-]
+)
 
 # Defaulting to the docker mode, token doesn't matter in it as it's passed for the container
 NSTBROWSER_DEFAULT_QUERY = {