PyPI - scrapling - Versions diffs - 0.3__tar.gz → 0.3.1__tar.gz - Mend

scrapling 0.3tar.gz → 0.3.1tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (48) hide show

{scrapling-0.3/scrapling.egg-info → scrapling-0.3.1}/PKG-INFO RENAMED Viewed

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapling
-Version: 0.3
+Version: 0.3.1
 Summary: Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!
 Home-page: https://github.com/D4Vinci/Scrapling
 Author: Karim Shoair
@@ -155,8 +155,8 @@ Built for the modern Web, Scrapling has its own rapid parsing engine and its fet
 <!-- sponsors -->
 <a href="https://evomi.com?utm_source=github&utm_medium=banner&utm_campaign=d4vinci-scrapling" target="_blank" title="Evomi is your Swiss Quality Proxy Provider, starting at $0.49/GB"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/evomi.png"></a>
-<a href="https://www.swiftproxy.net/" target="_blank" title="Unlock Reliable Proxy Services with Swiftproxy!"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/swiftproxy.png"></a>
 <a href="https://petrosky.io/d4vinci" target="_blank" title="PetroSky delivers cutting-edge VPS hosting."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/petrosky.png"></a>
+<a href="https://www.swiftproxy.net/" target="_blank" title="Unlock Reliable Proxy Services with Swiftproxy!"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/swiftproxy.png"></a>
 <a href="https://serpapi.com/?utm_source=scrapling" target="_blank" title="Scrape Google and other search engines with SerpApi"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/SerpApi.png"></a>
 <!-- /sponsors -->
@@ -273,7 +273,7 @@ from scrapling.parser import Selector
 page = Selector("<html>...</html>")
 ```
-And it works exactly the same!
+And it works exactly the same way!
 ### Async Session Management Examples
 ```python
@@ -302,6 +302,8 @@ async with AsyncStealthySession(max_pages=2) as session:
 Scrapling v0.3 includes a powerful command-line interface:
+[![asciicast](https://asciinema.org/a/736339.svg)](https://asciinema.org/a/736339)
 ```bash
 # Launch interactive Web Scraping shell
 scrapling shell

{scrapling-0.3 → scrapling-0.3.1}/README.md RENAMED Viewed

@@ -68,8 +68,8 @@ Built for the modern Web, Scrapling has its own rapid parsing engine and its fet
 <!-- sponsors -->
 <a href="https://evomi.com?utm_source=github&utm_medium=banner&utm_campaign=d4vinci-scrapling" target="_blank" title="Evomi is your Swiss Quality Proxy Provider, starting at $0.49/GB"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/evomi.png"></a>
-<a href="https://www.swiftproxy.net/" target="_blank" title="Unlock Reliable Proxy Services with Swiftproxy!"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/swiftproxy.png"></a>
 <a href="https://petrosky.io/d4vinci" target="_blank" title="PetroSky delivers cutting-edge VPS hosting."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/petrosky.png"></a>
+<a href="https://www.swiftproxy.net/" target="_blank" title="Unlock Reliable Proxy Services with Swiftproxy!"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/swiftproxy.png"></a>
 <a href="https://serpapi.com/?utm_source=scrapling" target="_blank" title="Scrape Google and other search engines with SerpApi"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/SerpApi.png"></a>
 <!-- /sponsors -->
@@ -186,7 +186,7 @@ from scrapling.parser import Selector
 page = Selector("<html>...</html>")
 ```
-And it works exactly the same!
+And it works exactly the same way!
 ### Async Session Management Examples
 ```python
@@ -215,6 +215,8 @@ async with AsyncStealthySession(max_pages=2) as session:
 Scrapling v0.3 includes a powerful command-line interface:
+[![asciicast](https://asciinema.org/a/736339.svg)](https://asciinema.org/a/736339)
 ```bash
 # Launch interactive Web Scraping shell
 scrapling shell

{scrapling-0.3 → scrapling-0.3.1}/scrapling/__init__.py RENAMED Viewed

@@ -1,5 +1,5 @@
 __author__ = "Karim Shoair (karim.shoair@pm.me)"
-__version__ = "0.3"
+__version__ = "0.3.1"
 __copyright__ = "Copyright (c) 2024 Karim Shoair"

{scrapling-0.3 → scrapling-0.3.1}/scrapling/core/shell.py RENAMED Viewed

@@ -20,7 +20,6 @@ from logging import (
     getLevelName,
 )
-from IPython.terminal.embed import InteractiveShellEmbed
 from orjson import loads as json_loads, JSONDecodeError
 from scrapling import __version__
@@ -394,8 +393,7 @@ class CurlParser:
         else:  # pragma: no cover
             log.error("Input must be a valid curl command string or a Request object.")
-        return None
+            return None
 def show_page_in_browser(page: Selector):  # pragma: no cover
@@ -544,6 +542,8 @@ Type 'exit' or press Ctrl+D to exit.
     def start(self):  # pragma: no cover
         """Start the interactive shell"""
+        from IPython.terminal.embed import InteractiveShellEmbed
         # Get our namespace with application objects
         namespace = self.get_namespace()
         ipython_shell = InteractiveShellEmbed(

{scrapling-0.3 → scrapling-0.3.1}/scrapling/engines/_browsers/_camoufox.py RENAMED Viewed

@@ -60,6 +60,7 @@ class StealthySession:
         "timeout",
         "page_action",
         "wait_selector",
+        "init_script",
         "addons",
         "wait_selector_state",
         "cookies",
@@ -95,6 +96,7 @@ class StealthySession:
         timeout: int | float = 30000,
         page_action: Optional[Callable] = None,
         wait_selector: Optional[str] = None,
+        init_script: Optional[str] = None,
         addons: Optional[List[str]] = None,
         wait_selector_state: SelectorWaitStates = "attached",
         cookies: Optional[List[Dict]] = None,
@@ -128,6 +130,7 @@ class StealthySession:
         :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
         :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
         :param wait_selector: Wait for a specific CSS selector to be in a specific state.
+        :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
         :param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address.
             It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region.
         :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
@@ -153,6 +156,7 @@ class StealthySession:
             "timeout": timeout,
             "page_action": page_action,
             "wait_selector": wait_selector,
+            "init_script": init_script,
             "addons": addons,
             "wait_selector_state": wait_selector_state,
             "cookies": cookies,
@@ -180,6 +184,7 @@ class StealthySession:
         self.timeout = config.timeout
         self.page_action = config.page_action
         self.wait_selector = config.wait_selector
+        self.init_script = config.init_script
         self.addons = config.addons
         self.wait_selector_state = config.wait_selector_state
         self.cookies = config.cookies
@@ -234,6 +239,9 @@ class StealthySession:
                 **self.launch_options
             )
         )
+        if self.init_script:  # pragma: no cover
+            self.context.add_init_script(path=self.init_script)
         if self.cookies:  # pragma: no cover
             self.context.add_cookies(self.cookies)
@@ -474,6 +482,7 @@ class AsyncStealthySession(StealthySession):
         timeout: int | float = 30000,
         page_action: Optional[Callable] = None,
         wait_selector: Optional[str] = None,
+        init_script: Optional[str] = None,
         addons: Optional[List[str]] = None,
         wait_selector_state: SelectorWaitStates = "attached",
         cookies: Optional[List[Dict]] = None,
@@ -507,6 +516,7 @@ class AsyncStealthySession(StealthySession):
         :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
         :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
         :param wait_selector: Wait for a specific CSS selector to be in a specific state.
+        :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
         :param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address.
             It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region.
         :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
@@ -531,6 +541,7 @@ class AsyncStealthySession(StealthySession):
             timeout,
             page_action,
             wait_selector,
+            init_script,
             addons,
             wait_selector_state,
             cookies,
@@ -557,6 +568,9 @@ class AsyncStealthySession(StealthySession):
                 **self.launch_options
             )
         )
+        if self.init_script:  # pragma: no cover
+            await self.context.add_init_script(path=self.init_script)
         if self.cookies:
             await self.context.add_cookies(self.cookies)

{scrapling-0.3 → scrapling-0.3.1}/scrapling/engines/_browsers/_controllers.py RENAMED Viewed

@@ -60,6 +60,7 @@ class DynamicSession:
         "disable_resources",
         "network_idle",
         "wait_selector",
+        "init_script",
         "wait_selector_state",
         "wait",
         "playwright",
@@ -94,6 +95,7 @@ class DynamicSession:
         timeout: int | float = 30000,
         disable_resources: bool = False,
         wait_selector: Optional[str] = None,
+        init_script: Optional[str] = None,
         cookies: Optional[List[Dict]] = None,
         network_idle: bool = False,
         wait_selector_state: SelectorWaitStates = "attached",
@@ -112,6 +114,7 @@ class DynamicSession:
         :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
         :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
         :param wait_selector: Wait for a specific CSS selector to be in a specific state.
+        :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
         :param locale: Set the locale for the browser if wanted. The default value is `en-US`.
         :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
         :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
@@ -143,6 +146,7 @@ class DynamicSession:
             "selector_config": selector_config,
             "disable_resources": disable_resources,
             "wait_selector": wait_selector,
+            "init_script": init_script,
             "cookies": cookies,
             "network_idle": network_idle,
             "wait_selector_state": wait_selector_state,
@@ -168,6 +172,7 @@ class DynamicSession:
         self.cdp_url = config.cdp_url
         self.network_idle = config.network_idle
         self.wait_selector = config.wait_selector
+        self.init_script = config.init_script
         self.wait_selector_state = config.wait_selector_state
         self.playwright: Optional[Playwright] = None
@@ -243,6 +248,9 @@ class DynamicSession:
                 user_data_dir="", **self.launch_options
             )
+        if self.init_script:  # pragma: no cover
+            self.context.add_init_script(path=self.init_script)
         if self.cookies:  # pragma: no cover
             self.context.add_cookies(self.cookies)
@@ -409,6 +417,7 @@ class AsyncDynamicSession(DynamicSession):
         timeout: int | float = 30000,
         disable_resources: bool = False,
         wait_selector: Optional[str] = None,
+        init_script: Optional[str] = None,
         cookies: Optional[List[Dict]] = None,
         network_idle: bool = False,
         wait_selector_state: SelectorWaitStates = "attached",
@@ -427,6 +436,7 @@ class AsyncDynamicSession(DynamicSession):
         :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
         :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
         :param wait_selector: Wait for a specific CSS selector to be in a specific state.
+        :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
         :param locale: Set the locale for the browser if wanted. The default value is `en-US`.
         :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
         :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
@@ -459,6 +469,7 @@ class AsyncDynamicSession(DynamicSession):
             timeout,
             disable_resources,
             wait_selector,
+            init_script,
             cookies,
             network_idle,
             wait_selector_state,
@@ -494,6 +505,9 @@ class AsyncDynamicSession(DynamicSession):
                 )
             )
+        if self.init_script:  # pragma: no cover
+            await self.context.add_init_script(path=self.init_script)
         if self.cookies:
             await self.context.add_cookies(self.cookies)

{scrapling-0.3 → scrapling-0.3.1}/scrapling/engines/_browsers/_validators.py RENAMED Viewed

@@ -32,6 +32,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
     extra_headers: Optional[Dict[str, str]] = None
     useragent: Optional[str] = None
     timeout: int | float = 30000
+    init_script: Optional[str] = None
     disable_resources: bool = False
     wait_selector: Optional[str] = None
     cookies: Optional[List[Dict]] = None
@@ -58,6 +59,15 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
         if not self.selector_config:
             self.selector_config = {}
+        if self.init_script is not None:
+            script_path = Path(self.init_script)
+            if not script_path.exists():
+                raise ValueError("Init script path not found")
+            elif not script_path.is_file():
+                raise ValueError("Init script is not a file")
+            elif not script_path.is_absolute():
+                raise ValueError("Init script is not a absolute path")
     @staticmethod
     def __validate_cdp(cdp_url):
         try:
@@ -90,6 +100,7 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
     solve_cloudflare: bool = False
     wait: int | float = 0
     timeout: int | float = 30000
+    init_script: Optional[str] = None
     page_action: Optional[Callable] = None
     wait_selector: Optional[str] = None
     addons: Optional[List[str]] = None
@@ -131,6 +142,15 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
                         f"Addon's path is not a folder, you need to pass a folder of the extracted addon: {addon}"
                     )
+        if self.init_script is not None:
+            script_path = Path(self.init_script)
+            if not script_path.exists():
+                raise ValueError("Init script path not found")
+            elif not script_path.is_file():
+                raise ValueError("Init script is not a file")
+            elif not script_path.is_absolute():
+                raise ValueError("Init script is not a absolute path")
         if not self.cookies:
             self.cookies = []
         if self.solve_cloudflare and self.timeout < 60_000:

{scrapling-0.3 → scrapling-0.3.1}/scrapling/fetchers.py RENAMED Viewed

@@ -62,6 +62,7 @@ class StealthyFetcher(BaseFetcher):
         timeout: int | float = 30000,
         page_action: Optional[Callable] = None,
         wait_selector: Optional[str] = None,
+        init_script: Optional[str] = None,
         addons: Optional[List[str]] = None,
         wait_selector_state: SelectorWaitStates = "attached",
         cookies: Optional[List[Dict]] = None,
@@ -97,6 +98,7 @@ class StealthyFetcher(BaseFetcher):
         :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
         :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
         :param wait_selector: Wait for a specific CSS selector to be in a specific state.
+        :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request.
         :param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address.
             It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region.
         :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
@@ -127,6 +129,7 @@ class StealthyFetcher(BaseFetcher):
             disable_ads=disable_ads,
             allow_webgl=allow_webgl,
             page_action=page_action,
+            init_script=init_script,
             network_idle=network_idle,
             block_images=block_images,
             block_webrtc=block_webrtc,
@@ -158,6 +161,7 @@ class StealthyFetcher(BaseFetcher):
         timeout: int | float = 30000,
         page_action: Optional[Callable] = None,
         wait_selector: Optional[str] = None,
+        init_script: Optional[str] = None,
         addons: Optional[List[str]] = None,
         wait_selector_state: SelectorWaitStates = "attached",
         cookies: Optional[List[Dict]] = None,
@@ -193,6 +197,7 @@ class StealthyFetcher(BaseFetcher):
         :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
         :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
         :param wait_selector: Wait for a specific CSS selector to be in a specific state.
+        :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request.
         :param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address.
             It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region.
         :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
@@ -223,6 +228,7 @@ class StealthyFetcher(BaseFetcher):
             disable_ads=disable_ads,
             allow_webgl=allow_webgl,
             page_action=page_action,
+            init_script=init_script,
             network_idle=network_idle,
             block_images=block_images,
             block_webrtc=block_webrtc,
@@ -276,6 +282,7 @@ class DynamicFetcher(BaseFetcher):
         timeout: int | float = 30000,
         disable_resources: bool = False,
         wait_selector: Optional[str] = None,
+        init_script: Optional[str] = None,
         cookies: Optional[Iterable[Dict]] = None,
         network_idle: bool = False,
         wait_selector_state: SelectorWaitStates = "attached",
@@ -295,6 +302,7 @@ class DynamicFetcher(BaseFetcher):
         :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
         :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
         :param wait_selector: Wait for a specific CSS selector to be in a specific state.
+        :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request.
         :param locale: Set the locale for the browser if wanted. The default value is `en-US`.
         :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
         :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
@@ -328,6 +336,7 @@ class DynamicFetcher(BaseFetcher):
             real_chrome=real_chrome,
             page_action=page_action,
             hide_canvas=hide_canvas,
+            init_script=init_script,
             network_idle=network_idle,
             google_search=google_search,
             extra_headers=extra_headers,
@@ -359,6 +368,7 @@ class DynamicFetcher(BaseFetcher):
         timeout: int | float = 30000,
         disable_resources: bool = False,
         wait_selector: Optional[str] = None,
+        init_script: Optional[str] = None,
         cookies: Optional[Iterable[Dict]] = None,
         network_idle: bool = False,
         wait_selector_state: SelectorWaitStates = "attached",
@@ -378,6 +388,7 @@ class DynamicFetcher(BaseFetcher):
         :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
         :param page_action: Added for automation. A function that takes the `page` object, does the automation you need, then returns `page` again.
         :param wait_selector: Wait for a specific CSS selector to be in a specific state.
+        :param init_script: An absolute path to a JavaScript file to be executed on page creation with this request.
         :param locale: Set the locale for the browser if wanted. The default value is `en-US`.
         :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
         :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.
@@ -412,6 +423,7 @@ class DynamicFetcher(BaseFetcher):
             real_chrome=real_chrome,
             page_action=page_action,
             hide_canvas=hide_canvas,
+            init_script=init_script,
             network_idle=network_idle,
             google_search=google_search,
             extra_headers=extra_headers,

{scrapling-0.3 → scrapling-0.3.1/scrapling.egg-info}/PKG-INFO RENAMED Viewed

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapling
-Version: 0.3
+Version: 0.3.1
 Summary: Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!
 Home-page: https://github.com/D4Vinci/Scrapling
 Author: Karim Shoair
@@ -155,8 +155,8 @@ Built for the modern Web, Scrapling has its own rapid parsing engine and its fet
 <!-- sponsors -->
 <a href="https://evomi.com?utm_source=github&utm_medium=banner&utm_campaign=d4vinci-scrapling" target="_blank" title="Evomi is your Swiss Quality Proxy Provider, starting at $0.49/GB"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/evomi.png"></a>
-<a href="https://www.swiftproxy.net/" target="_blank" title="Unlock Reliable Proxy Services with Swiftproxy!"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/swiftproxy.png"></a>
 <a href="https://petrosky.io/d4vinci" target="_blank" title="PetroSky delivers cutting-edge VPS hosting."><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/petrosky.png"></a>
+<a href="https://www.swiftproxy.net/" target="_blank" title="Unlock Reliable Proxy Services with Swiftproxy!"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/swiftproxy.png"></a>
 <a href="https://serpapi.com/?utm_source=scrapling" target="_blank" title="Scrape Google and other search engines with SerpApi"><img src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/SerpApi.png"></a>
 <!-- /sponsors -->
@@ -273,7 +273,7 @@ from scrapling.parser import Selector
 page = Selector("<html>...</html>")
 ```
-And it works exactly the same!
+And it works exactly the same way!
 ### Async Session Management Examples
 ```python
@@ -302,6 +302,8 @@ async with AsyncStealthySession(max_pages=2) as session:
 Scrapling v0.3 includes a powerful command-line interface:
+[![asciicast](https://asciinema.org/a/736339.svg)](https://asciinema.org/a/736339)
 ```bash
 # Launch interactive Web Scraping shell
 scrapling shell

{scrapling-0.3 → scrapling-0.3.1}/setup.cfg RENAMED Viewed

@@ -1,6 +1,6 @@
 [metadata]
 name = scrapling
-version = 0.3
+version = 0.3.1
 author = Karim Shoair
 author_email = karim.shoair@pm.me
 description = Scrapling is an undetectable, powerful, flexible, high-performance Python library that makes Web Scraping easy and effortless as it should be!