scrapling 0.2.5__py3-none-any.whl → 0.2.6__py3-none-any.whl

scrapling/__init__.py

@@ -4,7 +4,7 @@ from scrapling.parser import Adaptor, Adaptors
 from scrapling.core.custom_types import TextHandler, AttributesHandler
 
 __author__ = "Karim Shoair (karim.shoair@pm.me)"
-__version__ = "0.2.5"
+__version__ = "0.2.6"
 __copyright__ = "Copyright (c) 2024 Karim Shoair"
 
 

scrapling/engines/pw.py

@@ -27,11 +27,12 @@ class PlaywrightEngine:
             page_action: Callable = do_nothing,
             wait_selector: Optional[str] = None,
             wait_selector_state: Optional[str] = 'attached',
-            stealth: bool = False,
-            hide_canvas: bool = True,
-            disable_webgl: bool = False,
+            stealth: Optional[bool] = False,
+            real_chrome: Optional[bool] = False,
+            hide_canvas: Optional[bool] = False,
+            disable_webgl: Optional[bool] = False,
             cdp_url: Optional[str] = None,
-            nstbrowser_mode: bool = False,
+            nstbrowser_mode: Optional[bool] = False,
             nstbrowser_config: Optional[Dict] = None,
             google_search: Optional[bool] = True,
             extra_headers: Optional[Dict[str, str]] = None,
@@ -51,6 +52,7 @@ class PlaywrightEngine:
         :param wait_selector: Wait for a specific css selector to be in a specific state.
         :param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
         :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
+        :param real_chrome: If you have chrome browser installed on your device, enable this and the Fetcher will launch an instance of your browser and use it.
         :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
         :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
         :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.
@@ -67,6 +69,7 @@ class PlaywrightEngine:
         self.stealth = bool(stealth)
         self.hide_canvas = bool(hide_canvas)
         self.disable_webgl = bool(disable_webgl)
+        self.real_chrome = bool(real_chrome)
         self.google_search = bool(google_search)
         self.extra_headers = extra_headers or {}
         self.proxy = construct_proxy_dict(proxy)
@@ -119,7 +122,8 @@ class PlaywrightEngine:
         :param url: Target url.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
-        if not self.stealth:
+        if not self.stealth or self.real_chrome:
+            # Because rebrowser_playwright doesn't play well with real browsers
             from playwright.sync_api import sync_playwright
         else:
             from rebrowser_playwright.sync_api import sync_playwright
@@ -130,8 +134,8 @@ class PlaywrightEngine:
                 extra_headers = {}
                 useragent = self.useragent
             else:
-                extra_headers = generate_headers(browser_mode=True)
-                useragent = extra_headers.get('User-Agent')
+                extra_headers = {}
+                useragent = generate_headers(browser_mode=True).get('User-Agent')
 
             # Prepare the flags before diving
             flags = DEFAULT_STEALTH_FLAGS
@@ -146,9 +150,11 @@ class PlaywrightEngine:
                 browser = p.chromium.connect_over_cdp(endpoint_url=cdp_url)
             else:
                 if self.stealth:
-                    browser = p.chromium.launch(headless=self.headless, args=flags, ignore_default_args=['--enable-automation'], chromium_sandbox=True)
+                    browser = p.chromium.launch(
+                        headless=self.headless, args=flags, ignore_default_args=['--enable-automation'], chromium_sandbox=True, channel='chrome' if self.real_chrome else 'chromium'
+                    )
                 else:
-                    browser = p.chromium.launch(headless=self.headless, ignore_default_args=['--enable-automation'])
+                    browser = p.chromium.launch(headless=self.headless, ignore_default_args=['--enable-automation'], channel='chrome' if self.real_chrome else 'chromium')
 
             # Creating the context
             if self.stealth:

scrapling/engines/toolbelt/fingerprints.py

@@ -67,7 +67,7 @@ def generate_headers(browser_mode: bool = False) -> Dict:
         # So we don't raise any inconsistency red flags while websites fingerprinting us
         os_name = get_os_name()
         return HeaderGenerator(
-            browser=[Browser(name='chrome', min_version=128)],
+            browser=[Browser(name='chrome', min_version=130)],
             os=os_name,  # None is ignored
             device='desktop'
         ).generate()

scrapling/fetchers.py

@@ -138,7 +138,7 @@ class PlayWrightFetcher(BaseFetcher):
                 2) Mimics some of the real browsers' properties by injecting several JS files and using custom options.
                 3) Using custom flags on launch to hide Playwright even more and make it faster.
                 4) Generates real browser's headers of the same type and same user OS then append it to the request.
-        - Real browsers by passing the CDP URL of your browser to be controlled by the Fetcher and most of the options can be enabled on it.
+        - Real browsers by passing the `real_chrome` argument or the CDP URL of your browser to be controlled by the Fetcher and most of the options can be enabled on it.
         - NSTBrowser's docker browserless option by passing the CDP URL and enabling `nstbrowser_mode` option.
 
     > Note that these are the main options with PlayWright but it can be mixed together.
@@ -146,12 +146,12 @@ class PlayWrightFetcher(BaseFetcher):
     def fetch(
             self, url: str, headless: Union[bool, str] = True, disable_resources: bool = None,
             useragent: Optional[str] = None, network_idle: Optional[bool] = False, timeout: Optional[float] = 30000,
-            page_action: Callable = do_nothing, wait_selector: Optional[str] = None, wait_selector_state: Optional[str] = 'attached',
-            hide_canvas: bool = True, disable_webgl: bool = False, extra_headers: Optional[Dict[str, str]] = None, google_search: Optional[bool] = True,
+            page_action: Optional[Callable] = do_nothing, wait_selector: Optional[str] = None, wait_selector_state: Optional[str] = 'attached',
+            hide_canvas: Optional[bool] = False, disable_webgl: Optional[bool] = False, extra_headers: Optional[Dict[str, str]] = None, google_search: Optional[bool] = True,
             proxy: Optional[Union[str, Dict[str, str]]] = None,
-            stealth: bool = False,
+            stealth: Optional[bool] = False, real_chrome: Optional[bool] = False,
             cdp_url: Optional[str] = None,
-            nstbrowser_mode: bool = False, nstbrowser_config: Optional[Dict] = None,
+            nstbrowser_mode: Optional[bool] = False, nstbrowser_config: Optional[Dict] = None,
     ) -> Response:
         """Opens up a browser and do your request based on your chosen options below.
 
@@ -167,6 +167,7 @@ class PlayWrightFetcher(BaseFetcher):
         :param wait_selector: Wait for a specific css selector to be in a specific state.
         :param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
         :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
+        :param real_chrome: If you have chrome browser installed on your device, enable this and the Fetcher will launch an instance of your browser and use it.
         :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
         :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
         :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
@@ -184,6 +185,7 @@ class PlayWrightFetcher(BaseFetcher):
             cdp_url=cdp_url,
             headless=headless,
             useragent=useragent,
+            real_chrome=real_chrome,
             page_action=page_action,
             hide_canvas=hide_canvas,
             network_idle=network_idle,

{scrapling-0.2.5.dist-info → scrapling-0.2.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scrapling
-Version: 0.2.5
+Version: 0.2.6
 Summary: Scrapling is a powerful, flexible, and high-performance web scraping library for Python. It 
 Home-page: https://github.com/D4Vinci/Scrapling
 Author: Karim Shoair
@@ -39,7 +39,7 @@ Requires-Dist: w3lib
 Requires-Dist: orjson>=3
 Requires-Dist: tldextract
 Requires-Dist: httpx[brotli,zstd]
-Requires-Dist: playwright
+Requires-Dist: playwright==1.48
 Requires-Dist: rebrowser-playwright
 Requires-Dist: camoufox>=0.3.10
 Requires-Dist: browserforge
@@ -336,9 +336,11 @@ Using this Fetcher class, you can make requests with:
      * Mimics some of the real browsers' properties by injecting several JS files and using custom options.
      * Using custom flags on launch to hide Playwright even more and make it faster.
      * Generates real browser's headers of the same type and same user OS then append it to the request's headers.
-  3) Real browsers by passing the CDP URL of your browser to be controlled by the Fetcher and most of the options can be enabled on it.
+  3) Real browsers by passing the `real_chrome` argument or the CDP URL of your browser to be controlled by the Fetcher and most of the options can be enabled on it.
   4) [NSTBrowser](https://app.nstbrowser.io/r/1vO5e5)'s [docker browserless](https://hub.docker.com/r/nstbrowser/browserless) option by passing the CDP URL and enabling `nstbrowser_mode` option.
 
+> Hence using the `real_chrome` argument requires that you have chrome browser installed on your device
+
 Add that to a lot of controlling/hiding options as you will see in the arguments list below.
 
 <details><summary><strong>Expand this for the complete list of arguments</strong></summary>
@@ -360,6 +362,7 @@ Add that to a lot of controlling/hiding options as you will see in the arguments
 |     hide_canvas     | Add random noise to canvas operations to prevent fingerprinting.                                                                                                                                                                                                                                                                                                                                                |    ✔️    |
 |    disable_webgl    | Disables WebGL and WebGL 2.0 support entirely.                                                                                                                                                                                                                                                                                                                                                                  |    ✔️    |
 |       stealth       | Enables stealth mode, always check the documentation to see what stealth mode does currently.                                                                                                                                                                                                                                                                                                                   |    ✔️    |
+|     real_chrome     | If you have chrome browser installed on your device, enable this and the Fetcher will launch an instance of your browser and use it.                                                                                                                                                                                                                                                                            |    ✔️    |
 |       cdp_url       | Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.                                                                                                                                                                                                                                                                                           |    ✔️    |
 |   nstbrowser_mode   | Enables NSTBrowser mode, **it have to be used with `cdp_url` argument or it will get completely ignored.**                                                                                                                                                                                                                                                                                                      |    ✔️    |
 |  nstbrowser_config  | The config you want to send with requests to the NSTBrowser. _If left empty, Scrapling defaults to an optimized NSTBrowser's docker browserless config._                                                                                                                                                                                                                                                        |    ✔️    |

{scrapling-0.2.5.dist-info → scrapling-0.2.6.dist-info}/RECORD

@@ -1,6 +1,6 @@
-scrapling/__init__.py,sha256=eSTwhUqPltd41XPPwSXElWmeRalaQEncnjBahW3KUcQ,435
+scrapling/__init__.py,sha256=NnIpEZcBGs5Pu2TjqPCacC7N6LN37SbnniBU1AhgdXs,435
 scrapling/defaults.py,sha256=blYDLiuI5DgDSLRWnUgpp21WtFOsv1BsCRCmPeg8Xc4,287
-scrapling/fetchers.py,sha256=T3aRXvvpXDbql_2BqJ90KLtgfF4HLUOImdoxWLCcY-A,16045
+scrapling/fetchers.py,sha256=-gc-Yo1MjF_4cdJ-5rxZqNC0owxFXTFoEBj08BFEYPs,16361
 scrapling/parser.py,sha256=d2n00uF5i7W5lf0afLNRdk17ZFcNyiF9EzXLRQGA0NM,54111
 scrapling/py.typed,sha256=frcCV1k9oG9oKj3dpUqdJg1PxRT2RSN_XKdLCPjaYaY,2
 scrapling/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -13,11 +13,11 @@ scrapling/core/utils.py,sha256=fXdANUgRBbVbOerJ94fRY9vi7n5zsbm8t3G4qQ-F3ak,3792
 scrapling/engines/__init__.py,sha256=zwMqcSdNGh-IX0d4zXazrgAeHrkqIN_v5Ia7RU1g8W0,267
 scrapling/engines/camo.py,sha256=dXkdfFmf3M09RXAvaZ8CE5khsblC3Wd7_6jWfu8XO6I,7618
 scrapling/engines/constants.py,sha256=jSDA6lgbvEIB8z2m2SFzCKkvFEZnp28Mondy2__FpkM,3721
-scrapling/engines/pw.py,sha256=Llr2HjEr5kgyffEJH5N4XdWrSWvcil9WAwE1RJnOhFo,12190
+scrapling/engines/pw.py,sha256=gMWJAZYpJbFK-GiyRrpVrMjyMqSSetE6hf8kmf0zR2o,12729
 scrapling/engines/static.py,sha256=wzBsoOHPpN5JV1izQSSSarPBNWB-wo0BDWNFuin6ID8,7109
 scrapling/engines/toolbelt/__init__.py,sha256=BbxfC0depVOV3i3BnBnyfjHtLcZrDbhz6c5rTRczZUc,383
 scrapling/engines/toolbelt/custom.py,sha256=ELr3_FwUqNI27E98kz-50OA5a6hQQtoIYrZoLKsvUpM,12551
-scrapling/engines/toolbelt/fingerprints.py,sha256=kkVtZKSt2ukc0CV0g6QUvSWR0Yx5p8Mv8xiqACAsMBo,2917
+scrapling/engines/toolbelt/fingerprints.py,sha256=T9HQejHzAnHsD5EIXvrYVC5siiG5q2gOOXVIIANmzMc,2917
 scrapling/engines/toolbelt/navigation.py,sha256=Tde5_6Wv7lOeWXMzs9D6TRaxAbJ3b-zIX6-4HggZbCQ,4017
 scrapling/engines/toolbelt/bypasses/navigator_plugins.js,sha256=tbnnk3nCXB6QEQnOhDlu3n-s7lnUTAkrUsjP6FDQIQg,2104
 scrapling/engines/toolbelt/bypasses/notification_permission.js,sha256=poPM3o5WYgEX-EdiUfDCllpWfc3Umvw4jr2u6O6elus,237
@@ -35,8 +35,8 @@ tests/fetchers/test_utils.py,sha256=FPPJkBrqgYxdGeWwapH8Vj8zyfYVLiTE1qSLu8eBWik,
 tests/parser/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/parser/test_automatch.py,sha256=BeeYJi3cYCghbiZmi57z4bqcGPaoUA8GAm7MALBBkkk,2486
 tests/parser/test_general.py,sha256=qaiVzpvqESfdXYFat6QrpnMkevPYgCzIcTZK5FwdC0s,11783
-scrapling-0.2.5.dist-info/LICENSE,sha256=XHgu8DRuT7_g3Hb9Q18YGg8eShp6axPBacbnQxT_WWQ,1499
-scrapling-0.2.5.dist-info/METADATA,sha256=oY8RtycwNJXbgA4ePsVkpj6SKP6Y6vMmOgF9L3lc0mg,64780
-scrapling-0.2.5.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-scrapling-0.2.5.dist-info/top_level.txt,sha256=ub7FkOEXeYmmYTUxd4pCrwXfBfAMIpZ1sCGmXCc14tI,16
-scrapling-0.2.5.dist-info/RECORD,,
+scrapling-0.2.6.dist-info/LICENSE,sha256=XHgu8DRuT7_g3Hb9Q18YGg8eShp6axPBacbnQxT_WWQ,1499
+scrapling-0.2.6.dist-info/METADATA,sha256=cFOu2nlkXDsjyjkIt9kDu1nKKvS14xYH2LT4_VNH5j0,65362
+scrapling-0.2.6.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+scrapling-0.2.6.dist-info/top_level.txt,sha256=ub7FkOEXeYmmYTUxd4pCrwXfBfAMIpZ1sCGmXCc14tI,16
+scrapling-0.2.6.dist-info/RECORD,,