scrapling 0.1.2__py3-none-any.whl → 0.2.1__py3-none-any.whl

scrapling/__init__.py

@@ -1,10 +1,11 @@
 # Declare top-level shortcuts
+from scrapling.fetchers import Fetcher, StealthyFetcher, PlayWrightFetcher, CustomFetcher
 from scrapling.parser import Adaptor, Adaptors
-from scrapling.custom_types import TextHandler, AttributesHandler
+from scrapling.core.custom_types import TextHandler, AttributesHandler
 
 __author__ = "Karim Shoair (karim.shoair@pm.me)"
-__version__ = "0.1.2"
+__version__ = "0.2.1"
 __copyright__ = "Copyright (c) 2024 Karim Shoair"
 
 
-__all__ = ['Adaptor', 'Adaptors', 'TextHandler', 'AttributesHandler']
+__all__ = ['Adaptor', 'Fetcher', 'StealthyFetcher', 'PlayWrightFetcher']

scrapling/core/_types.py

@@ -0,0 +1,25 @@
+"""
+Type definitions for type checking purposes.
+"""
+
+from typing import (
+    Dict, Optional, Union, Callable, Any, List, Tuple, Pattern, Generator, Iterable, Type, TYPE_CHECKING, Literal
+)
+
+try:
+    from typing import Protocol
+except ImportError:
+    # Added in Python 3.8
+    Protocol = object
+
+try:
+    from typing import SupportsIndex
+except ImportError:
+    # 'SupportsIndex' got added in Python 3.8
+    SupportsIndex = None
+
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+else:
+    Self = object

scrapling/{custom_types.py → core/custom_types.py}

@@ -1,9 +1,9 @@
 import re
 from types import MappingProxyType
 from collections.abc import Mapping
-from typing import Dict, List, Union, Pattern
 
-from scrapling.utils import _is_iterable, flatten
+from scrapling.core.utils import _is_iterable, flatten
+from scrapling.core._types import Dict, List, Union, Pattern, SupportsIndex
 
 from orjson import loads, dumps
 from w3lib.html import replace_entities as _replace_entities
@@ -69,7 +69,7 @@ class TextHandler(str):
         return [TextHandler(_replace_entities(s)) for s in results]
 
     def re_first(self, regex: Union[str, Pattern[str]], default=None, replace_entities: bool = True,
-                 clean_match: bool = False, case_sensitive: bool = False,):
+                 clean_match: bool = False, case_sensitive: bool = False) -> Union[str, None]:
         """Apply the given regex to text and return the first match if found, otherwise return the default value.
 
         :param regex: Can be either a compiled regular expression or a string.
@@ -83,6 +83,51 @@ class TextHandler(str):
         return result[0] if result else default
 
 
+class TextHandlers(List[TextHandler]):
+    """
+    The :class:`TextHandlers` class is a subclass of the builtin ``List`` class, which provides a few additional methods.
+    """
+    __slots__ = ()
+
+    def __getitem__(self, pos: Union[SupportsIndex, slice]) -> Union[TextHandler, "TextHandlers[TextHandler]"]:
+        lst = super().__getitem__(pos)
+        if isinstance(pos, slice):
+            return self.__class__(lst)
+        else:
+            return lst
+
+    def re(self, regex: Union[str, Pattern[str]], replace_entities: bool = True, clean_match: bool = False,
+            case_sensitive: bool = False) -> 'List[str]':
+        """Call the ``.re()`` method for each element in this list and return
+        their results flattened as TextHandlers.
+
+        :param regex: Can be either a compiled regular expression or a string.
+        :param replace_entities: if enabled character entity references are replaced by their corresponding character
+        :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching
+        :param case_sensitive: if enabled, function will set the regex to ignore letters case while compiling it
+        """
+        results = [
+            n.re(regex, replace_entities, clean_match, case_sensitive) for n in self
+        ]
+        return flatten(results)
+
+    def re_first(self, regex: Union[str, Pattern[str]], default=None, replace_entities: bool = True,
+                 clean_match: bool = False, case_sensitive: bool = False) -> Union[str, None]:
+        """Call the ``.re_first()`` method for each element in this list and return
+        the first result or the default value otherwise.
+
+        :param regex: Can be either a compiled regular expression or a string.
+        :param default: The default value to be returned if there is no match
+        :param replace_entities: if enabled character entity references are replaced by their corresponding character
+        :param clean_match: if enabled, this will ignore all whitespaces and consecutive spaces while matching
+        :param case_sensitive: if enabled, function will set the regex to ignore letters case while compiling it
+        """
+        for n in self:
+            for result in n.re(regex, replace_entities, clean_match, case_sensitive):
+                return result
+        return default
+
+
 class AttributesHandler(Mapping):
     """A read-only mapping to use instead of the standard dictionary for the speed boost but
      at the same time I use it to add more functionalities.

scrapling/{mixins.py → core/mixins.py}

@@ -4,7 +4,7 @@ class SelectorsGeneration:
     Trying to generate selectors like Firefox or maybe cleaner ones!? Ehm
     Inspiration: https://searchfox.org/mozilla-central/source/devtools/shared/inspector/css-logic.js#591"""
 
-    def __general_selection(self, selection: str = 'css') -> str:
+    def __general_selection(self, selection: str = 'css', full_path=False) -> str:
         """Generate a selector for the current element.
         :return: A string of the generated selector.
         """
@@ -20,10 +20,11 @@ class SelectorsGeneration:
                         else f"[@id='{target.attrib['id']}']"
                     )
                     selectorPath.append(part)
-                    return (
-                        " > ".join(reversed(selectorPath)) if css
-                        else '//*' + "/".join(reversed(selectorPath))
-                    )
+                    if not full_path:
+                        return (
+                            " > ".join(reversed(selectorPath)) if css
+                            else '//*' + "/".join(reversed(selectorPath))
+                        )
                 else:
                     part = f'{target.tag}'
                     # We won't use classes anymore because I some websites share exact classes between elements
@@ -60,15 +61,29 @@ class SelectorsGeneration:
         )
 
     @property
-    def css_selector(self) -> str:
+    def generate_css_selector(self) -> str:
         """Generate a CSS selector for the current element
         :return: A string of the generated selector.
         """
         return self.__general_selection()
 
     @property
-    def xpath_selector(self) -> str:
+    def generate_full_css_selector(self) -> str:
+        """Generate a complete CSS selector for the current element
+        :return: A string of the generated selector.
+        """
+        return self.__general_selection(full_path=True)
+
+    @property
+    def generate_xpath_selector(self) -> str:
         """Generate a XPath selector for the current element
         :return: A string of the generated selector.
         """
         return self.__general_selection('xpath')
+
+    @property
+    def generate_full_xpath_selector(self) -> str:
+        """Generate a complete XPath selector for the current element
+        :return: A string of the generated selector.
+        """
+        return self.__general_selection('xpath', full_path=True)

scrapling/{storage_adaptors.py → core/storage_adaptors.py}

@@ -4,9 +4,9 @@ import logging
 import threading
 from hashlib import sha256
 from abc import ABC, abstractmethod
-from typing import Dict, Optional, Union
 
-from scrapling.utils import _StorageTools, cache
+from scrapling.core._types import Dict, Optional, Union
+from scrapling.core.utils import _StorageTools, cache
 
 from lxml import html
 from tldextract import extract as tld

scrapling/{translator.py → core/translator.py}

@@ -9,24 +9,14 @@ which will be important in future releases but most importantly...
 import re
 
 from w3lib.html import HTML5_WHITESPACE
-from typing import TYPE_CHECKING, Any, Optional
-try:
-    from typing import Protocol
-except ImportError:
-    # Added in Python 3.8
-    Protocol = object
-
-from scrapling.utils import cache
+from scrapling.core.utils import cache
+from scrapling.core._types import Any, Optional, Protocol, Self
 
 from cssselect.xpath import ExpressionError
 from cssselect.xpath import XPathExpr as OriginalXPathExpr
 from cssselect import HTMLTranslator as OriginalHTMLTranslator
 from cssselect.parser import Element, FunctionalPseudoElement, PseudoElement
 
-if TYPE_CHECKING:
-    # typing.Self requires Python 3.11
-    from typing_extensions import Self
-
 
 regex = f"[{HTML5_WHITESPACE}]+"
 replace_html5_whitespaces = re.compile(regex).sub

scrapling/{utils.py → core/utils.py}

@@ -1,14 +1,14 @@
 import re
-import os
 import logging
 from itertools import chain
-from logging import handlers
 # Using cache on top of a class is brilliant way to achieve Singleton design pattern without much code
 from functools import lru_cache as cache  # functools.cache is available on Python 3.9+ only so let's keep lru_cache
 
-from typing import Dict, Iterable, Any
+from scrapling.core._types import Dict, Iterable, Any, Union
 
+import orjson
 from lxml import html
+
 html_forbidden = {html.HtmlComment, }
 logging.basicConfig(
         level=logging.ERROR,
@@ -19,6 +19,17 @@ logging.basicConfig(
     )
 
 
+def is_jsonable(content: Union[bytes, str]) -> bool:
+    if type(content) is bytes:
+        content = content.decode()
+
+    try:
+        _ = orjson.loads(content)
+        return True
+    except orjson.JSONDecodeError:
+        return False
+
+
 @cache(None, typed=True)
 def setup_basic_logging(level: str = 'debug'):
     levels = {
@@ -45,64 +56,6 @@ def _is_iterable(s: Any):
     return isinstance(s, (list, tuple,))
 
 
-@cache(None, typed=True)
-class _Logger(object):
-    # I will leave this class here for now in case I decide I want to come back to use it :)
-    __slots__ = ('console_logger', 'logger_file_path',)
-    levels = {
-        'debug': logging.DEBUG,
-        'info': logging.INFO,
-        'warning': logging.WARNING,
-        'error': logging.ERROR,
-        'critical': logging.CRITICAL
-    }
-
-    def __init__(self, filename: str = 'debug.log', level: str = 'debug', when: str = 'midnight', backcount: int = 1):
-        os.makedirs(os.path.join(os.path.dirname(__file__), 'logs'), exist_ok=True)
-        format_str = logging.Formatter("[%(asctime)s] %(levelname)s: %(message)s", "%Y-%m-%d %H:%M:%S")
-
-        # on-screen output
-        lvl = self.levels[level.lower()]
-        self.console_logger = logging.getLogger('Scrapling')
-        self.console_logger.setLevel(lvl)
-        console_handler = logging.StreamHandler()
-        console_handler.setLevel(lvl)
-        console_handler.setFormatter(format_str)
-        self.console_logger.addHandler(console_handler)
-
-        if lvl == logging.DEBUG:
-            filename = os.path.join(os.path.dirname(__file__), 'logs', filename)
-            self.logger_file_path = filename
-            # Automatically generates the logging file at specified intervals
-            file_handler = handlers.TimedRotatingFileHandler(
-                # If more than (backcount+1) existed, oldest logs will be deleted
-                filename=filename, when=when, backupCount=backcount, encoding='utf-8'
-            )
-            file_handler.setLevel(lvl)
-            file_handler.setFormatter(format_str)
-            # This for the logger when it appends the date to the new log
-            file_handler.namer = lambda name: name.replace(".log", "") + ".log"
-            self.console_logger.addHandler(file_handler)
-            self.debug(f'Debug log path: {self.logger_file_path}')
-        else:
-            self.logger_file_path = None
-
-    def debug(self, message: str) -> None:
-        self.console_logger.debug(message)
-
-    def info(self, message: str) -> None:
-        self.console_logger.info(message)
-
-    def warning(self, message: str) -> None:
-        self.console_logger.warning(message)
-
-    def error(self, message: str) -> None:
-        self.console_logger.error(message)
-
-    def critical(self, message: str) -> None:
-        self.console_logger.critical(message)
-
-
 class _StorageTools:
     @staticmethod
     def __clean_attributes(element: html.HtmlElement, forbidden: tuple = ()) -> Dict:

scrapling/engines/__init__.py

@@ -0,0 +1,7 @@
+from .camo import CamoufoxEngine
+from .static import StaticEngine
+from .pw import PlaywrightEngine
+from .constants import DEFAULT_DISABLED_RESOURCES, DEFAULT_STEALTH_FLAGS
+from .toolbelt import check_if_engine_usable
+
+__all__ = ['CamoufoxEngine', 'PlaywrightEngine']

scrapling/engines/camo.py

@@ -0,0 +1,128 @@
+import logging
+from scrapling.core._types import Union, Callable, Optional, Dict, List, Literal
+
+from scrapling.engines.toolbelt import (
+    Response,
+    do_nothing,
+    get_os_name,
+    intercept_route,
+    check_type_validity,
+    construct_proxy_dict,
+    generate_convincing_referer,
+)
+
+from camoufox.sync_api import Camoufox
+
+
+class CamoufoxEngine:
+    def __init__(
+            self, headless: Optional[Union[bool, Literal['virtual']]] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False,
+            block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = False, network_idle: Optional[bool] = False, humanize: Optional[Union[bool, float]] = True,
+            timeout: Optional[float] = 30000, page_action: Callable = do_nothing, wait_selector: Optional[str] = None, addons: Optional[List[str]] = None,
+            wait_selector_state: str = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None,
+            proxy: Optional[Union[str, Dict[str, str]]] = None, os_randomize: Optional[bool] = None, adaptor_arguments: Dict = None
+    ):
+        """An engine that utilizes Camoufox library, check the `StealthyFetcher` class for more documentation.
+
+        :param headless: Run the browser in headless/hidden (default), virtual screen mode, or headful/visible mode.
+        :param block_images: Prevent the loading of images through Firefox preferences.
+            This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
+        :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends but it made requests ~25% faster in my tests for some websites.
+            Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
+            This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.
+        :param block_webrtc: Blocks WebRTC entirely.
+        :param addons: List of Firefox addons to use. Must be paths to extracted addons.
+        :param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.
+        :param allow_webgl: Whether to allow WebGL. To prevent leaks, only use this for special cases.
+        :param network_idle: Wait for the page until there are no network connections for at least 500 ms.
+        :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS.
+        :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000
+        :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
+        :param wait_selector: Wait for a specific css selector to be in a specific state.
+        :param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
+        :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
+        :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
+        :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
+        :param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
+        """
+        self.headless = headless
+        self.block_images = bool(block_images)
+        self.disable_resources = bool(disable_resources)
+        self.block_webrtc = bool(block_webrtc)
+        self.allow_webgl = bool(allow_webgl)
+        self.network_idle = bool(network_idle)
+        self.google_search = bool(google_search)
+        self.os_randomize = bool(os_randomize)
+        self.extra_headers = extra_headers or {}
+        self.proxy = construct_proxy_dict(proxy)
+        self.addons = addons or []
+        self.humanize = humanize
+        self.timeout = check_type_validity(timeout, [int, float], 30000)
+        if callable(page_action):
+            self.page_action = page_action
+        else:
+            self.page_action = do_nothing
+            logging.error('[Ignored] Argument "page_action" must be callable')
+
+        self.wait_selector = wait_selector
+        self.wait_selector_state = wait_selector_state
+        self.adaptor_arguments = adaptor_arguments if adaptor_arguments else {}
+
+    def fetch(self, url: str) -> Response:
+        """Opens up the browser and do your request based on your chosen options.
+
+        :param url: Target url.
+        :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
+        """
+        with Camoufox(
+                proxy=self.proxy,
+                addons=self.addons,
+                headless=self.headless,
+                humanize=self.humanize,
+                i_know_what_im_doing=True,  # To turn warnings off with the user configurations
+                allow_webgl=self.allow_webgl,
+                block_webrtc=self.block_webrtc,
+                block_images=self.block_images,  # Careful! it makes some websites doesn't finish loading at all like stackoverflow even in headful
+                os=None if self.os_randomize else get_os_name(),
+        ) as browser:
+            page = browser.new_page()
+            page.set_default_navigation_timeout(self.timeout)
+            page.set_default_timeout(self.timeout)
+            if self.disable_resources:
+                page.route("**/*", intercept_route)
+
+            if self.extra_headers:
+                page.set_extra_http_headers(self.extra_headers)
+
+            res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
+            page.wait_for_load_state(state="domcontentloaded")
+            if self.network_idle:
+                page.wait_for_load_state('networkidle')
+
+            page = self.page_action(page)
+
+            if self.wait_selector and type(self.wait_selector) is str:
+                waiter = page.locator(self.wait_selector)
+                waiter.wait_for(state=self.wait_selector_state)
+
+            content_type = res.headers.get('content-type', '')
+            # Parse charset from content-type
+            encoding = 'utf-8'  # default encoding
+            if 'charset=' in content_type.lower():
+                encoding = content_type.lower().split('charset=')[-1].split(';')[0].strip()
+
+            response = Response(
+                url=res.url,
+                text=page.content(),
+                content=res.body(),
+                status=res.status,
+                reason=res.status_text,
+                encoding=encoding,
+                cookies={cookie['name']: cookie['value'] for cookie in page.context.cookies()},
+                headers=res.all_headers(),
+                request_headers=res.request.all_headers(),
+                adaptor_arguments=self.adaptor_arguments
+            )
+            page.close()
+
+        return response

scrapling/engines/constants.py

@@ -0,0 +1,108 @@
+# Disable loading these resources for speed
+DEFAULT_DISABLED_RESOURCES = [
+    'font',
+    'image',
+    'media',
+    'beacon',
+    'object',
+    'imageset',
+    'texttrack',
+    'websocket',
+    'csp_report',
+    'stylesheet',
+]
+
+DEFAULT_STEALTH_FLAGS = [
+    # Explanation: https://peter.sh/experiments/chromium-command-line-switches/
+    # Generally this will make the browser faster and less detectable
+    '--no-pings',
+    '--incognito',
+    '--test-type',
+    '--lang=en-US',
+    '--mute-audio',
+    '--no-first-run',
+    '--disable-sync',
+    '--hide-scrollbars',
+    '--disable-logging',
+    '--start-maximized',  # For headless check bypass
+    '--enable-async-dns',
+    '--disable-breakpad',
+    '--disable-infobars',
+    '--accept-lang=en-US',
+    '--use-mock-keychain',
+    '--disable-translate',
+    '--disable-extensions',
+    '--disable-voice-input',
+    '--window-position=0,0',
+    '--disable-wake-on-wifi',
+    '--ignore-gpu-blocklist',
+    '--enable-tcp-fast-open',
+    '--enable-web-bluetooth',
+    '--disable-hang-monitor',
+    '--password-store=basic',
+    '--disable-cloud-import',
+    '--disable-default-apps',
+    '--disable-print-preview',
+    '--disable-dev-shm-usage',
+    '--disable-popup-blocking',
+    '--metrics-recording-only',
+    '--disable-crash-reporter',
+    '--disable-partial-raster',
+    '--disable-gesture-typing',
+    '--disable-checker-imaging',
+    '--disable-prompt-on-repost',
+    '--force-color-profile=srgb',
+    '--font-render-hinting=none',
+    '--no-default-browser-check',
+    '--aggressive-cache-discard',
+    '--disable-component-update',
+    '--disable-cookie-encryption',
+    '--disable-domain-reliability',
+    '--disable-threaded-animation',
+    '--disable-threaded-scrolling',
+    # '--disable-reading-from-canvas',  # For Firefox
+    '--enable-simple-cache-backend',
+    '--disable-background-networking',
+    '--disable-session-crashed-bubble',
+    '--enable-surface-synchronization',
+    '--disable-image-animation-resync',
+    '--disable-renderer-backgrounding',
+    '--disable-ipc-flooding-protection',
+    '--prerender-from-omnibox=disabled',
+    '--safebrowsing-disable-auto-update',
+    '--disable-offer-upload-credit-cards',
+    '--disable-features=site-per-process',
+    '--disable-background-timer-throttling',
+    '--disable-new-content-rendering-timeout',
+    '--run-all-compositor-stages-before-draw',
+    '--disable-client-side-phishing-detection',
+    '--disable-backgrounding-occluded-windows',
+    '--disable-layer-tree-host-memory-pressure',
+    '--autoplay-policy=no-user-gesture-required',
+    '--disable-offer-store-unmasked-wallet-cards',
+    '--disable-blink-features=AutomationControlled',
+    '--webrtc-ip-handling-policy=disable_non_proxied_udp',
+    '--disable-component-extensions-with-background-pages',
+    '--force-webrtc-ip-handling-policy=disable_non_proxied_udp',
+    '--enable-features=NetworkService,NetworkServiceInProcess,TrustTokens,TrustTokensAlwaysAllowIssuance',
+    '--blink-settings=primaryHoverType=2,availableHoverTypes=2,primaryPointerType=4,availablePointerTypes=4',
+    '--disable-features=AudioServiceOutOfProcess,IsolateOrigins,site-per-process,TranslateUI,BlinkGenPropertyTrees',
+]
+
+# Defaulting to the docker mode, token doesn't matter in it as it's passed for the container
+NSTBROWSER_DEFAULT_QUERY = {
+    "once": True,
+    "headless": True,
+    "autoClose": True,
+    "fingerprint": {
+        "flags": {
+            "timezone": "BasedOnIp",
+            "screen": "Custom"
+        },
+        "platform": 'linux',  # support: windows, mac, linux
+        "kernel": 'chromium',  # only support: chromium
+        "kernelMilestone": '128',
+        "hardwareConcurrency": 8,
+        "deviceMemory": 8,
+    },
+}