scrapy-seleniumbase-cdp 0.0.1__tar.gz

scrapy_seleniumbase_cdp-0.0.1/.github/workflows/publish.yml

@@ -0,0 +1,21 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [ published ]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Download release assets
+        run: |
+          gh release download ${{ github.event.release.tag_name }} -D dist -R ${{ github.repository }}
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

scrapy_seleniumbase_cdp-0.0.1/.github/workflows/release.yml

@@ -0,0 +1,41 @@
+name: Create GitHub release
+
+on:
+  push:
+    tags:
+      - v[0-9]+.[0-9]+.[0-9]+
+
+jobs:
+  create_release:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+
+      - name: Install build tools
+        run: |
+          python -m pip install --upgrade pip
+          pip install build
+
+      - name: Build package
+        run: python -m build
+
+      - name: Generate changelog
+        uses: orhun/git-cliff-action@v4
+        with:
+          args: -vv --latest --strip all
+
+      - name: Create release
+        run: gh release create ${{ github.ref_name }} -F git-cliff/CHANGELOG.md dist/*
+        env:
+          GH_TOKEN: ${{ secrets.RELEASE_TOKEN }}

scrapy_seleniumbase_cdp-0.0.1/.gitignore

@@ -0,0 +1,10 @@
+# jetbrains
+*.iml
+.idea
+
+# python
+.venv
+dist
+
+# misc
+.DS_Store

scrapy_seleniumbase_cdp-0.0.1/LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Original code Copyright (c) Quartz-Core
+Modifications Copyright (c) nyg
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

scrapy_seleniumbase_cdp-0.0.1/PKG-INFO

@@ -0,0 +1,161 @@
+Metadata-Version: 2.4
+Name: scrapy-seleniumbase-cdp
+Version: 0.0.1
+Summary: Scrapy downloader middleware that uses SeleniumBase's pure CDP mode to make requests.
+Project-URL: Homepage, https://github.com/nyg/scrapy-seleniumbase-cdp
+Project-URL: Issues, https://github.com/nyg/scrapy-seleniumbase-cdp/issue
+Author: nyg
+License: MIT
+License-File: LICENSE
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+
+# scrapy-selenium-cdp
+
+Scrapy downloader middleware that uses SeleniumBase's pure CDP mode to make
+requests, allowing to bypass most anti-bot protections (e.g. CloudFlare).
+
+Using Selenium's pure CDP mode also makes the middle more platform independent
+as no WebDriver is required.
+
+🚧 Work in progress 🚧
+
+## Installation
+
+```
+pip install git+https://github.com/nyg/scrapy-seleniumbase-cdp
+```
+
+## Configuration
+
+1. Provide keyword arguments for Driver in dict. For example:
+
+   ```python
+   SELENIUMBASE_DRIVER_KWARGS = {
+   "browser": "chrome",
+   "uc": True,
+   "headless": True,
+   "block_images": True,
+   }
+   ```
+
+2. Add the `SeleniumBaseMiddleware` to the downloader middlewares:
+    ```python
+    DOWNLOADER_MIDDLEWARES = {
+        'scrapy_seleniumbase.SeleniumBaseMiddleware': 800
+    }
+    ```
+
+## Usage
+
+Use the `scrapy_seleniumbase.SeleniumBaseRequest` instead of the scrapy built-in
+`Request` like below:
+
+```python
+from scrapy_seleniumbase import SeleniumBaseRequest
+
+yield SeleniumBaseRequest(url=url, callback=self.parse_result)
+```
+
+The request will be handled by seleniumbase, and the request will have an
+additional `meta` key, named `driver` containing the seleniumbase driver with
+the request processed.
+
+```python
+def parse_result(self, response):
+    print(response.request.meta['driver'].title)
+```
+
+For more information about the available driver methods and attributes, refer to
+the [selenium python documentation][1] (all vanilla selenium driver methods are
+available) and [seleniumbase documentation][2] (look for "driver" specific
+methods, located at the end of the page).
+
+The `selector` response attribute work as usual (but contains the html processed
+by the selenium driver).
+
+```python
+def parse_result(self, response):
+    print(response.selector.xpath('//title/@text'))
+```
+
+### Additional arguments
+
+The `scrapy_selenium.SeleniumBaseRequest` accept 5 additional arguments:
+
+#### `wait_time` / `wait_until`
+
+When used, webdriver will perform an [explicit wait][3] before returning the
+response to the spider.
+
+```python
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+
+yield SeleniumBaseRequest(
+    url=url,
+    callback=self.parse_result,
+    wait_time=10,
+    wait_until=EC.element_to_be_clickable((By.ID, 'someid'))
+)
+```
+
+#### `screenshot`
+
+When used, webdriver will take a screenshot of the page and the binary data of
+the .png captured will be added to the response `meta`:
+
+```python
+yield SeleniumBaseRequest(
+    url=url,
+    callback=self.parse_result,
+    screenshot=True
+)
+
+
+def parse_result(self, response):
+    with open('image.png', 'wb') as image_file:
+        image_file.write(response.meta['screenshot'])
+```
+
+#### `script`
+
+When used, webdriver will execute custom JavaScript code.
+
+```python
+yield SeleniumBaseRequest(
+    url=url,
+    callback=self.parse_result,
+    script='window.scrollTo(0, document.body.scrollHeight);',
+)
+```
+
+#### `driver_methods`
+
+When used, seleniumbase webdriver will execute methods, provided as strings in a
+list, before returning page's html.
+
+```python
+def start_requests(self):
+    for url in self.start_urls:
+        yield SeleniumRequest(
+            url=url,
+            driver_methods=['''.find_element("xpath","some_xpath").click()'''])
+
+)
+```
+
+## License
+
+This project is licensed under the MIT License. It is a fork
+of [Quartz-Core/scrapy-seleniumbase](https://github.com/Quartz-Core/scrapy-seleniumbase)
+which was originally released under the WTFPL.
+
+[1]: http://selenium-python.readthedocs.io/api.html#module-selenium.webdriver.remote.webdriver
+
+[2]: https://seleniumbase.io/help_docs/method_summary/#seleniumbase-methods-api-reference
+
+[3]: http://selenium-python.readthedocs.io/waits.html#explicit-waits

scrapy_seleniumbase_cdp-0.0.1/README.md

@@ -0,0 +1,146 @@
+# scrapy-selenium-cdp
+
+Scrapy downloader middleware that uses SeleniumBase's pure CDP mode to make
+requests, allowing to bypass most anti-bot protections (e.g. CloudFlare).
+
+Using Selenium's pure CDP mode also makes the middle more platform independent
+as no WebDriver is required.
+
+🚧 Work in progress 🚧
+
+## Installation
+
+```
+pip install git+https://github.com/nyg/scrapy-seleniumbase-cdp
+```
+
+## Configuration
+
+1. Provide keyword arguments for Driver in dict. For example:
+
+   ```python
+   SELENIUMBASE_DRIVER_KWARGS = {
+   "browser": "chrome",
+   "uc": True,
+   "headless": True,
+   "block_images": True,
+   }
+   ```
+
+2. Add the `SeleniumBaseMiddleware` to the downloader middlewares:
+    ```python
+    DOWNLOADER_MIDDLEWARES = {
+        'scrapy_seleniumbase.SeleniumBaseMiddleware': 800
+    }
+    ```
+
+## Usage
+
+Use the `scrapy_seleniumbase.SeleniumBaseRequest` instead of the scrapy built-in
+`Request` like below:
+
+```python
+from scrapy_seleniumbase import SeleniumBaseRequest
+
+yield SeleniumBaseRequest(url=url, callback=self.parse_result)
+```
+
+The request will be handled by seleniumbase, and the request will have an
+additional `meta` key, named `driver` containing the seleniumbase driver with
+the request processed.
+
+```python
+def parse_result(self, response):
+    print(response.request.meta['driver'].title)
+```
+
+For more information about the available driver methods and attributes, refer to
+the [selenium python documentation][1] (all vanilla selenium driver methods are
+available) and [seleniumbase documentation][2] (look for "driver" specific
+methods, located at the end of the page).
+
+The `selector` response attribute work as usual (but contains the html processed
+by the selenium driver).
+
+```python
+def parse_result(self, response):
+    print(response.selector.xpath('//title/@text'))
+```
+
+### Additional arguments
+
+The `scrapy_selenium.SeleniumBaseRequest` accept 5 additional arguments:
+
+#### `wait_time` / `wait_until`
+
+When used, webdriver will perform an [explicit wait][3] before returning the
+response to the spider.
+
+```python
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+
+yield SeleniumBaseRequest(
+    url=url,
+    callback=self.parse_result,
+    wait_time=10,
+    wait_until=EC.element_to_be_clickable((By.ID, 'someid'))
+)
+```
+
+#### `screenshot`
+
+When used, webdriver will take a screenshot of the page and the binary data of
+the .png captured will be added to the response `meta`:
+
+```python
+yield SeleniumBaseRequest(
+    url=url,
+    callback=self.parse_result,
+    screenshot=True
+)
+
+
+def parse_result(self, response):
+    with open('image.png', 'wb') as image_file:
+        image_file.write(response.meta['screenshot'])
+```
+
+#### `script`
+
+When used, webdriver will execute custom JavaScript code.
+
+```python
+yield SeleniumBaseRequest(
+    url=url,
+    callback=self.parse_result,
+    script='window.scrollTo(0, document.body.scrollHeight);',
+)
+```
+
+#### `driver_methods`
+
+When used, seleniumbase webdriver will execute methods, provided as strings in a
+list, before returning page's html.
+
+```python
+def start_requests(self):
+    for url in self.start_urls:
+        yield SeleniumRequest(
+            url=url,
+            driver_methods=['''.find_element("xpath","some_xpath").click()'''])
+
+)
+```
+
+## License
+
+This project is licensed under the MIT License. It is a fork
+of [Quartz-Core/scrapy-seleniumbase](https://github.com/Quartz-Core/scrapy-seleniumbase)
+which was originally released under the WTFPL.
+
+[1]: http://selenium-python.readthedocs.io/api.html#module-selenium.webdriver.remote.webdriver
+
+[2]: https://seleniumbase.io/help_docs/method_summary/#seleniumbase-methods-api-reference
+
+[3]: http://selenium-python.readthedocs.io/waits.html#explicit-waits

scrapy_seleniumbase_cdp-0.0.1/pyproject.toml

@@ -0,0 +1,24 @@
+[project]
+name = "scrapy-seleniumbase-cdp"
+version = "0.0.1"
+authors = [
+    { name = "nyg" },
+]
+description = "Scrapy downloader middleware that uses SeleniumBase's pure CDP mode to make requests."
+readme = "README.md"
+requires-python = ">=3.8"
+classifiers = [
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent",
+]
+license = { text = "MIT" }
+license-files = ["LICEN[CS]E*"]
+
+[project.urls]
+Homepage = "https://github.com/nyg/scrapy-seleniumbase-cdp"
+Issues = "https://github.com/nyg/scrapy-seleniumbase-cdp/issue"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"

scrapy_seleniumbase_cdp-0.0.1/scrapy_seleniumbase_cdp/__init__.py

@@ -0,0 +1,4 @@
+from .middleware_async import SeleniumBaseAsyncCDPMiddleware
+from .request import SeleniumBaseRequest
+
+__all__ = ['SeleniumBaseAsyncCDPMiddleware', 'SeleniumBaseRequest']

scrapy_seleniumbase_cdp-0.0.1/scrapy_seleniumbase_cdp/middleware_async.py

@@ -0,0 +1,75 @@
+# SeleniumBase middleware using the pure CDP mode instead of the UC mode.
+#
+# Uses the async/await API of SeleniumBase because the event loop of the pure
+# CDP mode conflicts with Scrapy's own event loop. For the moment this means
+# less features.
+#
+# The pure CDP mode does not require any WebDriver and can therefore run on
+# platforms where no such drivers are available (e.g., Raspberry Pi)
+#
+# Based on https://github.com/Quartz-Core/scrapy-seleniumbase.
+#
+# Doc: https://github.com/seleniumbase/SeleniumBase/blob/master/help_docs/syntax_formats.md#sb_sf_24
+#      https://github.com/seleniumbase/SeleniumBase/discussions/3955
+
+from importlib import import_module
+
+from scrapy import signals
+from scrapy.http import HtmlResponse
+from scrapy_seleniumbase import SeleniumBaseRequest
+
+
+class SeleniumBaseAsyncCDPMiddleware:
+    """Scrapy middleware handling the requests using seleniumbase"""
+
+    def __init__(self, driver_kwargs):
+        """Initialize the selenium webdriver
+
+        Parameters
+        ----------
+        driver_kwargs: dict
+            A dictionary of keyword arguments to initialize the driver with.
+        """
+        seleniumbase_cdp = import_module("seleniumbase")
+        cdp_module = getattr(seleniumbase_cdp, 'cdp_driver')
+        self.start_async_driver = getattr(cdp_module, "start_async")
+        self.driver = None
+        self.driver_kwargs = driver_kwargs
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        """Initialize the middleware with the crawler settings"""
+        driver_kwargs = crawler.settings.get('SELENIUMBASE_DRIVER_KWARGS', {})
+        middleware = cls(driver_kwargs)
+        crawler.signals.connect(middleware.spider_opened, signals.spider_opened)
+        crawler.signals.connect(middleware.spider_closed, signals.spider_closed)
+        return middleware
+
+    async def process_request(self, request, spider):
+        """Process a request using the selenium driver if applicable"""
+
+        if not isinstance(request, SeleniumBaseRequest):
+            return None
+
+        page = await self.driver.get(request.url)
+
+        if request.wait_until:
+            try:
+                await page.select(request.wait_until, timeout=request.wait_time if hasattr(request, 'wait_time') else 10)
+            except Exception as e:
+                spider.logger.warning(f'Element not found: {request.wait_until}, {e}')
+
+        request.meta.update({'driver': self.driver})
+
+        page_source = await page.evaluate('document.documentElement.outerHTML')
+        body = str.encode(page_source)
+
+        return HtmlResponse(await page.evaluate('window.location.href'), body=body, encoding='utf-8', request=request)
+
+    async def spider_opened(self, spider):
+        """Start the CDP driver when spider opens"""
+        self.driver = await self.start_async_driver(**self.driver_kwargs)
+
+    def spider_closed(self):
+        """Shutdown the driver when spider is closed"""
+        self.driver.stop()

scrapy_seleniumbase_cdp-0.0.1/scrapy_seleniumbase_cdp/request.py

@@ -0,0 +1,43 @@
+"""This module contains the ``SeleniumBaseRequest`` class"""
+
+from scrapy import Request
+
+
+class SeleniumBaseRequest(Request):
+    """Scrapy ``Request`` subclass providing additional arguments"""
+
+    def __init__(
+        self,
+        wait_time=None,
+        wait_until=None,
+        screenshot=False,
+        script=None,
+        driver_methods=None,
+        *args,
+        **kwargs
+    ):
+        """Initialize a new selenium request
+
+        Parameters
+        ----------
+        wait_time: int
+            The number of seconds to wait.
+        wait_until: method
+            One of the "selenium.webdriver.support.expected_conditions". The response
+            will be returned until the given condition is fulfilled.
+        screenshot: bool
+            If True, a screenshot of the page will be taken and the data of the screenshot
+            will be returned in the response "meta" attribute.
+        script: str
+            JavaScript code to execute.
+        driver_methods: list
+            List of seleniumbase driver methods as strings to execute. (e.g., [".find_element(...).click()", ...])
+
+        """
+
+        self.wait_time = wait_time
+        self.wait_until = wait_until
+        self.screenshot = screenshot
+        self.script = script
+        self.driver_methods = driver_methods
+        super().__init__(*args, **kwargs)