PyPI - scrapling - Versions diffs - 0.2.91__tar.gz → 0.2.93__tar.gz - Mend

scrapling 0.2.91tar.gz → 0.2.93tar.gz

Files changed (57) hide show

{scrapling-0.2.91 → scrapling-0.2.93}/MANIFEST.in RENAMED Viewed

@@ -4,7 +4,10 @@ include *.js
 include scrapling/engines/toolbelt/bypasses/*.js
 include scrapling/*.db
 include scrapling/*.db*
+include scrapling/*.db-*
 include scrapling/py.typed
+include scrapling/.scrapling_dependencies_installed
+include .scrapling_dependencies_installed
 recursive-exclude * __pycache__
 recursive-exclude * *.py[co]

{scrapling-0.2.91/scrapling.egg-info → scrapling-0.2.93}/PKG-INFO RENAMED Viewed

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: scrapling
-Version: 0.2.91
+Version: 0.2.93
 Summary: Scrapling is a powerful, flexible, and high-performance web scraping library for Python. It
 Home-page: https://github.com/D4Vinci/Scrapling
 Author: Karim Shoair
@@ -10,7 +10,7 @@ Project-URL: Documentation, https://github.com/D4Vinci/Scrapling/tree/main/docs
 Project-URL: Source, https://github.com/D4Vinci/Scrapling
 Project-URL: Tracker, https://github.com/D4Vinci/Scrapling/issues
 Classifier: Operating System :: OS Independent
-Classifier: Development Status :: 4 - Beta
+Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: BSD License
 Classifier: Natural Language :: English
@@ -31,16 +31,27 @@ Classifier: Typing :: Typed
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: requests>=2.3
-Requires-Dist: lxml>=4.5
+Requires-Dist: lxml>=5.0
 Requires-Dist: cssselect>=1.2
+Requires-Dist: click
 Requires-Dist: w3lib
 Requires-Dist: orjson>=3
 Requires-Dist: tldextract
 Requires-Dist: httpx[brotli,socks,zstd]
 Requires-Dist: playwright>=1.49.1
 Requires-Dist: rebrowser-playwright>=1.49.1
-Requires-Dist: camoufox[geoip]>=0.4.9
+Requires-Dist: camoufox[geoip]>=0.4.10
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: project-url
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 # 🕷️ Scrapling: Undetectable, Lightning-Fast, and Adaptive Web Scraping for Python
 [![Tests](https://github.com/D4Vinci/Scrapling/actions/workflows/tests.yml/badge.svg)](https://github.com/D4Vinci/Scrapling/actions/workflows/tests.yml) [![PyPI version](https://badge.fury.io/py/Scrapling.svg)](https://badge.fury.io/py/Scrapling) [![Supported Python versions](https://img.shields.io/pypi/pyversions/scrapling.svg)](https://pypi.org/project/scrapling/) [![PyPI Downloads](https://static.pepy.tech/badge/scrapling)](https://pepy.tech/project/scrapling)
@@ -77,6 +88,21 @@ Scrapling is a high-performance, intelligent web scraping library for Python tha
 [![Evomi Banner](https://my.evomi.com/images/brand/cta.png)](https://evomi.com?utm_source=github&utm_medium=banner&utm_campaign=d4vinci-scrapling)
 ---
+[Scrapeless](https://www.scrapeless.com/?utm_source=github&utm_medium=ads&utm_campaign=scraping&utm_term=D4Vinci) is your all-in-one web scraping toolkit, starting at just $0.60 per 1k URLs!
+- 🚀 Scraping API: Effortless and highly customizable data extraction with a single API call, providing structured data from any website.
+- ⚡ Scraping Browser: AI-powered and LLM-driven, it simulates human-like behavior with genuine fingerprints and headless browser support, ensuring seamless, block-free scraping.
+- 🔒 Web Unlocker: Bypass CAPTCHAs, IP blocks, and dynamic content in real time, ensuring uninterrupted access.
+- 🌐 Proxies: Use high-quality, rotating proxies to scrape top platforms like Amazon, Shopee, and more, with global coverage in 195+ countries.
+- 💼 Enterprise-Grade: Custom solutions for large-scale and complex data needs.
+- 🎁 Free Trial: Try before you buy—experience our service firsthand.
+- 💬 Pay-Per-Use: Flexible, cost-effective pricing with no long-term commitments.
+- 🔧 Easy Integration: Seamlessly integrate with your existing tools and workflows for hassle-free automation.
+[![Scrapeless Banner](https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/scrapeless.jpg)](https://www.scrapeless.com/?utm_source=github&utm_medium=ads&utm_campaign=scraping&utm_term=D4Vinci)
+---
 ## Table of content
   * [Key Features](#key-features)
     * [Fetch websites as you prefer](#fetch-websites-as-you-prefer-with-async-support)
@@ -121,27 +147,27 @@ Scrapling is a high-performance, intelligent web scraping library for Python tha
 ## Key Features
 ### Fetch websites as you prefer with async support
-- **HTTP requests**: Stealthy and fast HTTP requests with `Fetcher`
-- **Stealthy fetcher**: Annoying anti-bot protection? No problem! Scrapling can bypass almost all of them with `StealthyFetcher` with default configuration!
-- **Your preferred browser**: Use your real browser with CDP, [NSTbrowser](https://app.nstbrowser.io/r/1vO5e5)'s browserless, PlayWright with stealth mode, or even vanilla PlayWright -  All is possible with `PlayWrightFetcher`!
+- **HTTP Requests**: Fast and stealthy HTTP requests with the `Fetcher` class.
+- **Dynamic Loading & Automation**: Fetch dynamic websites with the `PlayWrightFetcher` class through your real browser, Scrapling's stealth mode, Playwright's Chrome browser, or [NSTbrowser](https://app.nstbrowser.io/r/1vO5e5)'s browserless!
+- **Anti-bot Protections Bypass**: Easily bypass protections with `StealthyFetcher` and `PlayWrightFetcher` classes.
 ### Adaptive Scraping
-- 🔄 **Smart Element Tracking**: Locate previously identified elements after website structure changes, using an intelligent similarity system and integrated storage.
-- 🎯 **Flexible Querying**: Use CSS selectors, XPath, Elements filters, text search, or regex - chain them however you want!
-- 🔍 **Find Similar Elements**: Automatically locate elements similar to the element you want on the page (Ex: other products like the product you found on the page).
+- 🔄 **Smart Element Tracking**: Relocate elements after website changes, using an intelligent similarity system and integrated storage.
+- 🎯 **Flexible Selection**: CSS selectors, XPath selectors, filters-based search, text search, regex search and more.
+- 🔍 **Find Similar Elements**: Automatically locate elements similar to the element you found!
 - 🧠 **Smart Content Scraping**: Extract data from multiple websites without specific selectors using Scrapling powerful features.
-### Performance
-- 🚀 **Lightning Fast**: Built from the ground up with performance in mind, outperforming most popular Python scraping libraries (outperforming BeautifulSoup in parsing by up to 620x in our tests).
+### High Performance
+- 🚀 **Lightning Fast**: Built from the ground up with performance in mind, outperforming most popular Python scraping libraries.
 - 🔋 **Memory Efficient**: Optimized data structures for minimal memory footprint.
-- ⚡ **Fast JSON serialization**: 10x faster JSON serialization than the standard json library with more options.
+- ⚡ **Fast JSON serialization**: 10x faster than standard library.
-### Developing Experience
-- 🛠️ **Powerful Navigation API**: Traverse the DOM tree easily in all directions and get the info you want (parent, ancestors, sibling, children, next/previous element, and more).
-- 🧬 **Rich Text Processing**: All strings have built-in methods for regex matching, cleaning, and more. All elements' attributes are read-only dictionaries that are faster than standard dictionaries with added methods.
-- 📝 **Automatic Selector Generation**: Create robust CSS/XPath selectors for any element.
-- 🔌 **API Similar to Scrapy/BeautifulSoup**: Familiar methods and similar pseudo-elements for Scrapy and BeautifulSoup users.
-- 📘 **Type hints and test coverage**: Complete type coverage and almost full test coverage for better IDE support and fewer bugs, respectively.
+### Developer Friendly
+- 🛠️ **Powerful Navigation API**: Easy DOM traversal in all directions.
+- 🧬 **Rich Text Processing**: All strings have built-in regex, cleaning methods, and more. All elements' attributes are optimized dictionaries that takes less memory than standard dictionaries with added methods.
+- 📝 **Auto Selectors Generation**: Generate robust short and full CSS/XPath selectors for any element.
+- 🔌 **Familiar API**: Similar to Scrapy/BeautifulSoup and the same pseudo-elements used in Scrapy.
+- 📘 **Type hints**: Complete type/doc-strings coverage for future-proofing and best autocompletion support.
 ## Getting Started
@@ -150,21 +176,22 @@ from scrapling import Fetcher
 fetcher = Fetcher(auto_match=False)
-# Fetch a web page and create an Adaptor instance
+# Do http GET request to a web page and create an Adaptor instance
 page = fetcher.get('https://quotes.toscrape.com/', stealthy_headers=True)
-# Get all strings in the full page
+# Get all text content from all HTML tags in the page except `script` and `style` tags
 page.get_all_text(ignore_tags=('script', 'style'))
-# Get all quotes, any of these methods will return a list of strings (TextHandlers)
+# Get all quotes elements, any of these methods will return a list of strings directly (TextHandlers)
 quotes = page.css('.quote .text::text')  # CSS selector
 quotes = page.xpath('//span[@class="text"]/text()')  # XPath
 quotes = page.css('.quote').css('.text::text')  # Chained selectors
 quotes = [element.text for element in page.css('.quote .text')]  # Slower than bulk query above
 # Get the first quote element
-quote = page.css_first('.quote')  # / page.css('.quote').first / page.css('.quote')[0]
+quote = page.css_first('.quote')  # same as page.css('.quote').first or page.css('.quote')[0]
 # Tired of selectors? Use find_all/find
+# Get all 'div' HTML tags that one of its 'class' values is 'quote'
 quotes = page.find_all('div', {'class': 'quote'})
 # Same as
 quotes = page.find_all('div', class_='quote')
@@ -172,10 +199,10 @@ quotes = page.find_all(['div'], class_='quote')
 quotes = page.find_all(class_='quote')  # and so on...
 # Working with elements
-quote.html_content  # Inner HTML
-quote.prettify()  # Prettified version of Inner HTML
-quote.attrib  # Element attributes
-quote.path  # DOM path to element (List)
+quote.html_content  # Get Inner HTML of this element
+quote.prettify()  # Prettified version of Inner HTML above
+quote.attrib  # Get that element's attributes
+quote.path  # DOM path to element (List of all ancestors from <html> tag till the element itself)
 ```
 To keep it simple, all methods can be chained on top of each other!
@@ -211,52 +238,18 @@ Scrapling can find elements with more methods and it returns full element `Adapt
 > All benchmarks' results are an average of 100 runs. See our [benchmarks.py](https://github.com/D4Vinci/Scrapling/blob/main/benchmarks.py) for methodology and to run your comparisons.
 ## Installation
-Scrapling is a breeze to get started with - Starting from version 0.2.9, we require at least Python 3.9 to work.
+Scrapling is a breeze to get started with; Starting from version 0.2.9, we require at least Python 3.9 to work.
 ```bash
 pip3 install scrapling
 ```
-- For using the `StealthyFetcher`, go to the command line and download the browser with
-<details><summary>Windows OS</summary>
+Then run this command to install browsers' dependencies needed to use Fetcher classes
 ```bash
-camoufox fetch --browserforge
-```
-</details>
-<details><summary>MacOS</summary>
-```bash
-python3 -m camoufox fetch --browserforge
-```
-</details>
-<details><summary>Linux</summary>
-```bash
-python -m camoufox fetch --browserforge
-```
-On a fresh installation of Linux, you may also need the following Firefox dependencies:
-- Debian-based distros
-    ```bash
-    sudo apt install -y libgtk-3-0 libx11-xcb1 libasound2
-    ```
-- Arch-based distros
-    ```bash
-    sudo pacman -S gtk3 libx11 libxcb cairo libasound alsa-lib
-    ```
-</details>
-<small> See the official <a href="https://camoufox.com/python/installation/#download-the-browser">Camoufox documentation</a> for more info on installation</small>
-- If you are going to use the `PlayWrightFetcher` options, then install Playwright's Chromium browser with:
-```commandline
-playwright install chromium
-```
-- If you are going to use normal requests only with the `Fetcher` class then update the fingerprints files with:
-```commandline
-python -m browserforge update
+scrapling install
 ```
+If you have any installation issues, please open an issue.
 ## Fetching Websites
-Fetchers are basically interfaces that do requests or fetch pages for you in a single request fashion and then return an `Adaptor` object for you. This feature was introduced because the only option we had before was to fetch the page as you wanted it, then pass it manually to the `Adaptor` class to create an `Adaptor` instance and start playing around with the page.
+Fetchers are interfaces built on top of other libraries with added features that do requests or fetch pages for you in a single request fashion and then return an `Adaptor` object. This feature was introduced because the only option we had before was to fetch the page as you wanted it, then pass it manually to the `Adaptor` class to create an `Adaptor` instance and start playing around with the page.
 ### Features
 You might be slightly confused by now so let me clear things up. All fetcher-type classes are imported in the same way
@@ -325,7 +318,7 @@ True
 |      humanize       | Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.                                                                                                                                                                                                                                  |    ✔️    |
 |     allow_webgl     | Enabled by default. Disabling it WebGL not recommended as many WAFs now checks if WebGL is enabled.                                                                                                                                                                                                                                                                                                             |    ✔️    |
 |        geoip        | Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, & spoof the WebRTC IP address. It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region.                                                                                                                                             |    ✔️    |
-|     disable_ads     | Enabled by default, this installs `uBlock Origin` addon on the browser if enabled.                                                                                                                                                                                                                                                                                                                              |    ✔️    |
+|     disable_ads     | Disabled by default, this installs `uBlock Origin` addon on the browser if enabled.                                                                                                                                                                                                                                                                                                                             |    ✔️    |
 |    network_idle     | Wait for the page until there are no network connections for at least 500 ms.                                                                                                                                                                                                                                                                                                                                   |    ✔️    |
 |       timeout       | The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000.                                                                                                                                                                                                                                                                                                    |    ✔️    |
 |    wait_selector    | Wait for a specific css selector to be in a specific state.                                                                                                                                                                                                                                                                                                                                                     |    ✔️    |
@@ -607,7 +600,7 @@ Inspired by BeautifulSoup's `find_all` function you can find elements by using `
   * Any string passed is considered a tag name
   * Any iterable passed like List/Tuple/Set is considered an iterable of tag names.
   * Any dictionary is considered a mapping of HTML element(s) attribute names and attribute values.
-  * Any regex patterns passed are used as filters
+  * Any regex patterns passed are used as filters to elements by their text content
   * Any functions passed are used as filters
   * Any keyword argument passed is considered as an HTML element attribute with its value.
@@ -616,7 +609,7 @@ So the way it works is after collecting all passed arguments and keywords, each
 1. All elements with the passed tag name(s).
 2. All elements that match all passed attribute(s).
-3. All elements that match all passed regex patterns.
+3. All elements that its text content match all passed regex patterns.
 4. All elements that fulfill all passed function(s).
 Note: The filtering process always starts from the first filter it finds in the filtering order above so if no tag name(s) are passed but attributes are passed, the process starts from that layer and so on. **But the order in which you pass the arguments doesn't matter.**

{scrapling-0.2.91 → scrapling-0.2.93}/README.md RENAMED Viewed

@@ -33,6 +33,21 @@ Scrapling is a high-performance, intelligent web scraping library for Python tha
 [![Evomi Banner](https://my.evomi.com/images/brand/cta.png)](https://evomi.com?utm_source=github&utm_medium=banner&utm_campaign=d4vinci-scrapling)
 ---
+[Scrapeless](https://www.scrapeless.com/?utm_source=github&utm_medium=ads&utm_campaign=scraping&utm_term=D4Vinci) is your all-in-one web scraping toolkit, starting at just $0.60 per 1k URLs!
+- 🚀 Scraping API: Effortless and highly customizable data extraction with a single API call, providing structured data from any website.
+- ⚡ Scraping Browser: AI-powered and LLM-driven, it simulates human-like behavior with genuine fingerprints and headless browser support, ensuring seamless, block-free scraping.
+- 🔒 Web Unlocker: Bypass CAPTCHAs, IP blocks, and dynamic content in real time, ensuring uninterrupted access.
+- 🌐 Proxies: Use high-quality, rotating proxies to scrape top platforms like Amazon, Shopee, and more, with global coverage in 195+ countries.
+- 💼 Enterprise-Grade: Custom solutions for large-scale and complex data needs.
+- 🎁 Free Trial: Try before you buy—experience our service firsthand.
+- 💬 Pay-Per-Use: Flexible, cost-effective pricing with no long-term commitments.
+- 🔧 Easy Integration: Seamlessly integrate with your existing tools and workflows for hassle-free automation.
+[![Scrapeless Banner](https://raw.githubusercontent.com/D4Vinci/Scrapling/main/images/scrapeless.jpg)](https://www.scrapeless.com/?utm_source=github&utm_medium=ads&utm_campaign=scraping&utm_term=D4Vinci)
+---
 ## Table of content
   * [Key Features](#key-features)
     * [Fetch websites as you prefer](#fetch-websites-as-you-prefer-with-async-support)
@@ -77,27 +92,27 @@ Scrapling is a high-performance, intelligent web scraping library for Python tha
 ## Key Features
 ### Fetch websites as you prefer with async support
-- **HTTP requests**: Stealthy and fast HTTP requests with `Fetcher`
-- **Stealthy fetcher**: Annoying anti-bot protection? No problem! Scrapling can bypass almost all of them with `StealthyFetcher` with default configuration!
-- **Your preferred browser**: Use your real browser with CDP, [NSTbrowser](https://app.nstbrowser.io/r/1vO5e5)'s browserless, PlayWright with stealth mode, or even vanilla PlayWright -  All is possible with `PlayWrightFetcher`!
+- **HTTP Requests**: Fast and stealthy HTTP requests with the `Fetcher` class.
+- **Dynamic Loading & Automation**: Fetch dynamic websites with the `PlayWrightFetcher` class through your real browser, Scrapling's stealth mode, Playwright's Chrome browser, or [NSTbrowser](https://app.nstbrowser.io/r/1vO5e5)'s browserless!
+- **Anti-bot Protections Bypass**: Easily bypass protections with `StealthyFetcher` and `PlayWrightFetcher` classes.
 ### Adaptive Scraping
-- 🔄 **Smart Element Tracking**: Locate previously identified elements after website structure changes, using an intelligent similarity system and integrated storage.
-- 🎯 **Flexible Querying**: Use CSS selectors, XPath, Elements filters, text search, or regex - chain them however you want!
-- 🔍 **Find Similar Elements**: Automatically locate elements similar to the element you want on the page (Ex: other products like the product you found on the page).
+- 🔄 **Smart Element Tracking**: Relocate elements after website changes, using an intelligent similarity system and integrated storage.
+- 🎯 **Flexible Selection**: CSS selectors, XPath selectors, filters-based search, text search, regex search and more.
+- 🔍 **Find Similar Elements**: Automatically locate elements similar to the element you found!
 - 🧠 **Smart Content Scraping**: Extract data from multiple websites without specific selectors using Scrapling powerful features.
-### Performance
-- 🚀 **Lightning Fast**: Built from the ground up with performance in mind, outperforming most popular Python scraping libraries (outperforming BeautifulSoup in parsing by up to 620x in our tests).
+### High Performance
+- 🚀 **Lightning Fast**: Built from the ground up with performance in mind, outperforming most popular Python scraping libraries.
 - 🔋 **Memory Efficient**: Optimized data structures for minimal memory footprint.
-- ⚡ **Fast JSON serialization**: 10x faster JSON serialization than the standard json library with more options.
+- ⚡ **Fast JSON serialization**: 10x faster than standard library.
-### Developing Experience
-- 🛠️ **Powerful Navigation API**: Traverse the DOM tree easily in all directions and get the info you want (parent, ancestors, sibling, children, next/previous element, and more).
-- 🧬 **Rich Text Processing**: All strings have built-in methods for regex matching, cleaning, and more. All elements' attributes are read-only dictionaries that are faster than standard dictionaries with added methods.
-- 📝 **Automatic Selector Generation**: Create robust CSS/XPath selectors for any element.
-- 🔌 **API Similar to Scrapy/BeautifulSoup**: Familiar methods and similar pseudo-elements for Scrapy and BeautifulSoup users.
-- 📘 **Type hints and test coverage**: Complete type coverage and almost full test coverage for better IDE support and fewer bugs, respectively.
+### Developer Friendly
+- 🛠️ **Powerful Navigation API**: Easy DOM traversal in all directions.
+- 🧬 **Rich Text Processing**: All strings have built-in regex, cleaning methods, and more. All elements' attributes are optimized dictionaries that takes less memory than standard dictionaries with added methods.
+- 📝 **Auto Selectors Generation**: Generate robust short and full CSS/XPath selectors for any element.
+- 🔌 **Familiar API**: Similar to Scrapy/BeautifulSoup and the same pseudo-elements used in Scrapy.
+- 📘 **Type hints**: Complete type/doc-strings coverage for future-proofing and best autocompletion support.
 ## Getting Started
@@ -106,21 +121,22 @@ from scrapling import Fetcher
 fetcher = Fetcher(auto_match=False)
-# Fetch a web page and create an Adaptor instance
+# Do http GET request to a web page and create an Adaptor instance
 page = fetcher.get('https://quotes.toscrape.com/', stealthy_headers=True)
-# Get all strings in the full page
+# Get all text content from all HTML tags in the page except `script` and `style` tags
 page.get_all_text(ignore_tags=('script', 'style'))
-# Get all quotes, any of these methods will return a list of strings (TextHandlers)
+# Get all quotes elements, any of these methods will return a list of strings directly (TextHandlers)
 quotes = page.css('.quote .text::text')  # CSS selector
 quotes = page.xpath('//span[@class="text"]/text()')  # XPath
 quotes = page.css('.quote').css('.text::text')  # Chained selectors
 quotes = [element.text for element in page.css('.quote .text')]  # Slower than bulk query above
 # Get the first quote element
-quote = page.css_first('.quote')  # / page.css('.quote').first / page.css('.quote')[0]
+quote = page.css_first('.quote')  # same as page.css('.quote').first or page.css('.quote')[0]
 # Tired of selectors? Use find_all/find
+# Get all 'div' HTML tags that one of its 'class' values is 'quote'
 quotes = page.find_all('div', {'class': 'quote'})
 # Same as
 quotes = page.find_all('div', class_='quote')
@@ -128,10 +144,10 @@ quotes = page.find_all(['div'], class_='quote')
 quotes = page.find_all(class_='quote')  # and so on...
 # Working with elements
-quote.html_content  # Inner HTML
-quote.prettify()  # Prettified version of Inner HTML
-quote.attrib  # Element attributes
-quote.path  # DOM path to element (List)
+quote.html_content  # Get Inner HTML of this element
+quote.prettify()  # Prettified version of Inner HTML above
+quote.attrib  # Get that element's attributes
+quote.path  # DOM path to element (List of all ancestors from <html> tag till the element itself)
 ```
 To keep it simple, all methods can be chained on top of each other!
@@ -167,52 +183,18 @@ Scrapling can find elements with more methods and it returns full element `Adapt
 > All benchmarks' results are an average of 100 runs. See our [benchmarks.py](https://github.com/D4Vinci/Scrapling/blob/main/benchmarks.py) for methodology and to run your comparisons.
 ## Installation
-Scrapling is a breeze to get started with - Starting from version 0.2.9, we require at least Python 3.9 to work.
+Scrapling is a breeze to get started with; Starting from version 0.2.9, we require at least Python 3.9 to work.
 ```bash
 pip3 install scrapling
 ```
-- For using the `StealthyFetcher`, go to the command line and download the browser with
-<details><summary>Windows OS</summary>
+Then run this command to install browsers' dependencies needed to use Fetcher classes
 ```bash
-camoufox fetch --browserforge
-```
-</details>
-<details><summary>MacOS</summary>
-```bash
-python3 -m camoufox fetch --browserforge
-```
-</details>
-<details><summary>Linux</summary>
-```bash
-python -m camoufox fetch --browserforge
-```
-On a fresh installation of Linux, you may also need the following Firefox dependencies:
-- Debian-based distros
-    ```bash
-    sudo apt install -y libgtk-3-0 libx11-xcb1 libasound2
-    ```
-- Arch-based distros
-    ```bash
-    sudo pacman -S gtk3 libx11 libxcb cairo libasound alsa-lib
-    ```
-</details>
-<small> See the official <a href="https://camoufox.com/python/installation/#download-the-browser">Camoufox documentation</a> for more info on installation</small>
-- If you are going to use the `PlayWrightFetcher` options, then install Playwright's Chromium browser with:
-```commandline
-playwright install chromium
-```
-- If you are going to use normal requests only with the `Fetcher` class then update the fingerprints files with:
-```commandline
-python -m browserforge update
+scrapling install
 ```
+If you have any installation issues, please open an issue.
 ## Fetching Websites
-Fetchers are basically interfaces that do requests or fetch pages for you in a single request fashion and then return an `Adaptor` object for you. This feature was introduced because the only option we had before was to fetch the page as you wanted it, then pass it manually to the `Adaptor` class to create an `Adaptor` instance and start playing around with the page.
+Fetchers are interfaces built on top of other libraries with added features that do requests or fetch pages for you in a single request fashion and then return an `Adaptor` object. This feature was introduced because the only option we had before was to fetch the page as you wanted it, then pass it manually to the `Adaptor` class to create an `Adaptor` instance and start playing around with the page.
 ### Features
 You might be slightly confused by now so let me clear things up. All fetcher-type classes are imported in the same way
@@ -281,7 +263,7 @@ True
 |      humanize       | Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.                                                                                                                                                                                                                                  |    ✔️    |
 |     allow_webgl     | Enabled by default. Disabling it WebGL not recommended as many WAFs now checks if WebGL is enabled.                                                                                                                                                                                                                                                                                                             |    ✔️    |
 |        geoip        | Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, & spoof the WebRTC IP address. It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region.                                                                                                                                             |    ✔️    |
-|     disable_ads     | Enabled by default, this installs `uBlock Origin` addon on the browser if enabled.                                                                                                                                                                                                                                                                                                                              |    ✔️    |
+|     disable_ads     | Disabled by default, this installs `uBlock Origin` addon on the browser if enabled.                                                                                                                                                                                                                                                                                                                             |    ✔️    |
 |    network_idle     | Wait for the page until there are no network connections for at least 500 ms.                                                                                                                                                                                                                                                                                                                                   |    ✔️    |
 |       timeout       | The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000.                                                                                                                                                                                                                                                                                                    |    ✔️    |
 |    wait_selector    | Wait for a specific css selector to be in a specific state.                                                                                                                                                                                                                                                                                                                                                     |    ✔️    |
@@ -563,7 +545,7 @@ Inspired by BeautifulSoup's `find_all` function you can find elements by using `
   * Any string passed is considered a tag name
   * Any iterable passed like List/Tuple/Set is considered an iterable of tag names.
   * Any dictionary is considered a mapping of HTML element(s) attribute names and attribute values.
-  * Any regex patterns passed are used as filters
+  * Any regex patterns passed are used as filters to elements by their text content
   * Any functions passed are used as filters
   * Any keyword argument passed is considered as an HTML element attribute with its value.
@@ -572,7 +554,7 @@ So the way it works is after collecting all passed arguments and keywords, each
 1. All elements with the passed tag name(s).
 2. All elements that match all passed attribute(s).
-3. All elements that match all passed regex patterns.
+3. All elements that its text content match all passed regex patterns.
 4. All elements that fulfill all passed function(s).
 Note: The filtering process always starts from the first filter it finds in the filtering order above so if no tag name(s) are passed but attributes are passed, the process starts from that layer and so on. **But the order in which you pass the arguments doesn't matter.**

{scrapling-0.2.91 → scrapling-0.2.93}/scrapling/__init__.py RENAMED Viewed

@@ -5,7 +5,7 @@ from scrapling.fetchers import (AsyncFetcher, CustomFetcher, Fetcher,
 from scrapling.parser import Adaptor, Adaptors
 __author__ = "Karim Shoair (karim.shoair@pm.me)"
-__version__ = "0.2.91"
+__version__ = "0.2.93"
 __copyright__ = "Copyright (c) 2024 Karim Shoair"

scrapling-0.2.93/scrapling/cli.py ADDED Viewed

@@ -0,0 +1,37 @@
+import os
+import subprocess
+import sys
+from pathlib import Path
+import click
+def get_package_dir():
+    return Path(os.path.dirname(__file__))
+def run_command(command, line):
+    print(f"Installing {line}...")
+    _ = subprocess.check_call(command, shell=True)
+    # I meant to not use try except here
+@click.command(help="Install all Scrapling's Fetchers dependencies")
+def install():
+    if not get_package_dir().joinpath(".scrapling_dependencies_installed").exists():
+        run_command([sys.executable, "-m", "playwright", "install", 'chromium'], 'Playwright browsers')
+        run_command([sys.executable, "-m", "playwright", "install-deps", 'chromium', 'firefox'], 'Playwright dependencies')
+        run_command([sys.executable, "-m", "camoufox", "fetch", '--browserforge'], 'Camoufox browser and databases')
+        # if no errors raised by above commands, then we add below file
+        get_package_dir().joinpath(".scrapling_dependencies_installed").touch()
+    else:
+        print('The dependencies are already installed')
+@click.group()
+def main():
+    pass
+# Adding commands
+main.add_command(install)

{scrapling-0.2.91 → scrapling-0.2.93}/scrapling/core/_types.py RENAMED Viewed

@@ -3,7 +3,8 @@ Type definitions for type checking purposes.
 """
 from typing import (TYPE_CHECKING, Any, Callable, Dict, Generator, Iterable,
-                    List, Literal, Optional, Pattern, Tuple, Type, Union)
+                    List, Literal, Optional, Pattern, Tuple, Type, TypeVar,
+                    Union)
 SelectorWaitStates = Literal["attached", "detached", "hidden", "visible"]

scrapling 0.2.91__tar.gz → 0.2.93__tar.gz

scrapling 0.2.91tar.gz → 0.2.93tar.gz