aio-scrapy 2.1.7__tar.gz → 2.1.9__tar.gz

{aio_scrapy-2.1.7/aio_scrapy.egg-info → aio_scrapy-2.1.9}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: aio-scrapy
-Version: 2.1.7
+Version: 2.1.9
 Summary: A high-level Web Crawling and Web Scraping framework based on Asyncio
 Home-page: https://github.com/conlin-huang/aio-scrapy.git
 Author: conlin
@@ -41,6 +41,8 @@ Requires-Dist: XlsxWriter>=3.1.2; extra == "all"
 Requires-Dist: pillow>=9.4.0; extra == "all"
 Requires-Dist: requests>=2.28.2; extra == "all"
 Requires-Dist: curl_cffi; extra == "all"
+Requires-Dist: sbcdp; extra == "all"
+Requires-Dist: DrissionPage; extra == "all"
 Provides-Extra: aiomysql
 Requires-Dist: aiomysql>=0.1.1; extra == "aiomysql"
 Requires-Dist: cryptography; extra == "aiomysql"
@@ -52,6 +54,10 @@ Provides-Extra: mongo
 Requires-Dist: motor>=2.1.0; extra == "mongo"
 Provides-Extra: playwright
 Requires-Dist: playwright>=1.31.1; extra == "playwright"
+Provides-Extra: sbcdp
+Requires-Dist: sbcdp; extra == "sbcdp"
+Provides-Extra: dp
+Requires-Dist: DrissionPage; extra == "dp"
 Provides-Extra: pyhttpx
 Requires-Dist: pyhttpx>=2.10.4; extra == "pyhttpx"
 Provides-Extra: curl-cffi
@@ -71,6 +77,7 @@ Dynamic: description-content-type
 Dynamic: home-page
 Dynamic: keywords
 Dynamic: license
+Dynamic: license-file
 Dynamic: provides-extra
 Dynamic: requires-dist
 Dynamic: requires-python
@@ -84,7 +91,7 @@ AioScrapy is a powerful asynchronous web crawling framework built on Python's as
 ## 特性 | Features
 
 - **完全异步**：基于Python的asyncio库，实现高效的并发爬取
-- **多种下载处理程序**：支持多种HTTP客户端，包括aiohttp、httpx、requests、pyhttpx、curl_cffi、DrissionPage和playwright
+- **多种下载处理程序**：支持多种HTTP客户端，包括aiohttp、httpx、requests、pyhttpx、curl_cffi、DrissionPage、playwright和sbcdp
 - **灵活的中间件系统**：轻松添加自定义功能和处理逻辑
 - **强大的数据处理管道**：支持多种数据库存储选项
 - **内置信号系统**：方便的事件处理机制
@@ -94,7 +101,7 @@ AioScrapy is a powerful asynchronous web crawling framework built on Python's as
 
 
 - **Fully Asynchronous**: Built on Python's asyncio for efficient concurrent crawling
-- **Multiple Download Handlers**: Support for various HTTP clients including aiohttp, httpx, requests, pyhttpx, curl_cffi, DrissionPage and playwright
+- **Multiple Download Handlers**: Support for various HTTP clients including aiohttp, httpx, requests, pyhttpx, curl_cffi, DrissionPage, playwright and sbcdp
 - **Flexible Middleware System**: Easily add custom functionality and processing logic
 - **Powerful Data Processing Pipelines**: Support for various database storage options
 - **Built-in Signal System**: Convenient event handling mechanism
@@ -117,6 +124,49 @@ pip install aio-scrapy
 # pip install git+https://github.com/ConlinH/aio-scrapy
 ```
 
+### 开始 | Start
+```python
+from aioscrapy import Spider, logger
+
+
+class MyspiderSpider(Spider):
+    name = 'myspider'
+    custom_settings = {
+        "CLOSE_SPIDER_ON_IDLE": True
+    }
+    start_urls = ["https://quotes.toscrape.com"]
+
+    @staticmethod
+    async def process_request(request, spider):
+        """ request middleware """
+        pass
+
+    @staticmethod
+    async def process_response(request, response, spider):
+        """ response middleware """
+        return response
+
+    @staticmethod
+    async def process_exception(request, exception, spider):
+        """ exception middleware """
+        pass
+
+    async def parse(self, response):
+        for quote in response.css('div.quote'):
+            item = {
+                'author': quote.xpath('span/small/text()').get(),
+                'text': quote.css('span.text::text').get(),
+            }
+            yield item
+
+    async def process_item(self, item):
+        logger.info(item)
+
+
+if __name__ == '__main__':
+    MyspiderSpider.start()
+```
+
 ## 文档 | Documentation
 
 ## 文档目录 | Documentation Contents

{aio_scrapy-2.1.7 → aio_scrapy-2.1.9}/README.md

@@ -6,7 +6,7 @@ AioScrapy is a powerful asynchronous web crawling framework built on Python's as
 ## 特性 | Features
 
 - **完全异步**：基于Python的asyncio库，实现高效的并发爬取
-- **多种下载处理程序**：支持多种HTTP客户端，包括aiohttp、httpx、requests、pyhttpx、curl_cffi、DrissionPage和playwright
+- **多种下载处理程序**：支持多种HTTP客户端，包括aiohttp、httpx、requests、pyhttpx、curl_cffi、DrissionPage、playwright和sbcdp
 - **灵活的中间件系统**：轻松添加自定义功能和处理逻辑
 - **强大的数据处理管道**：支持多种数据库存储选项
 - **内置信号系统**：方便的事件处理机制
@@ -16,7 +16,7 @@ AioScrapy is a powerful asynchronous web crawling framework built on Python's as
 
 
 - **Fully Asynchronous**: Built on Python's asyncio for efficient concurrent crawling
-- **Multiple Download Handlers**: Support for various HTTP clients including aiohttp, httpx, requests, pyhttpx, curl_cffi, DrissionPage and playwright
+- **Multiple Download Handlers**: Support for various HTTP clients including aiohttp, httpx, requests, pyhttpx, curl_cffi, DrissionPage, playwright and sbcdp
 - **Flexible Middleware System**: Easily add custom functionality and processing logic
 - **Powerful Data Processing Pipelines**: Support for various database storage options
 - **Built-in Signal System**: Convenient event handling mechanism
@@ -39,6 +39,49 @@ pip install aio-scrapy
 # pip install git+https://github.com/ConlinH/aio-scrapy
 ```
 
+### 开始 | Start
+```python
+from aioscrapy import Spider, logger
+
+
+class MyspiderSpider(Spider):
+    name = 'myspider'
+    custom_settings = {
+        "CLOSE_SPIDER_ON_IDLE": True
+    }
+    start_urls = ["https://quotes.toscrape.com"]
+
+    @staticmethod
+    async def process_request(request, spider):
+        """ request middleware """
+        pass
+
+    @staticmethod
+    async def process_response(request, response, spider):
+        """ response middleware """
+        return response
+
+    @staticmethod
+    async def process_exception(request, exception, spider):
+        """ exception middleware """
+        pass
+
+    async def parse(self, response):
+        for quote in response.css('div.quote'):
+            item = {
+                'author': quote.xpath('span/small/text()').get(),
+                'text': quote.css('span.text::text').get(),
+            }
+            yield item
+
+    async def process_item(self, item):
+        logger.info(item)
+
+
+if __name__ == '__main__':
+    MyspiderSpider.start()
+```
+
 ## 文档 | Documentation
 
 ## 文档目录 | Documentation Contents

{aio_scrapy-2.1.7 → aio_scrapy-2.1.9/aio_scrapy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: aio-scrapy
-Version: 2.1.7
+Version: 2.1.9
 Summary: A high-level Web Crawling and Web Scraping framework based on Asyncio
 Home-page: https://github.com/conlin-huang/aio-scrapy.git
 Author: conlin
@@ -41,6 +41,8 @@ Requires-Dist: XlsxWriter>=3.1.2; extra == "all"
 Requires-Dist: pillow>=9.4.0; extra == "all"
 Requires-Dist: requests>=2.28.2; extra == "all"
 Requires-Dist: curl_cffi; extra == "all"
+Requires-Dist: sbcdp; extra == "all"
+Requires-Dist: DrissionPage; extra == "all"
 Provides-Extra: aiomysql
 Requires-Dist: aiomysql>=0.1.1; extra == "aiomysql"
 Requires-Dist: cryptography; extra == "aiomysql"
@@ -52,6 +54,10 @@ Provides-Extra: mongo
 Requires-Dist: motor>=2.1.0; extra == "mongo"
 Provides-Extra: playwright
 Requires-Dist: playwright>=1.31.1; extra == "playwright"
+Provides-Extra: sbcdp
+Requires-Dist: sbcdp; extra == "sbcdp"
+Provides-Extra: dp
+Requires-Dist: DrissionPage; extra == "dp"
 Provides-Extra: pyhttpx
 Requires-Dist: pyhttpx>=2.10.4; extra == "pyhttpx"
 Provides-Extra: curl-cffi
@@ -71,6 +77,7 @@ Dynamic: description-content-type
 Dynamic: home-page
 Dynamic: keywords
 Dynamic: license
+Dynamic: license-file
 Dynamic: provides-extra
 Dynamic: requires-dist
 Dynamic: requires-python
@@ -84,7 +91,7 @@ AioScrapy is a powerful asynchronous web crawling framework built on Python's as
 ## 特性 | Features
 
 - **完全异步**：基于Python的asyncio库，实现高效的并发爬取
-- **多种下载处理程序**：支持多种HTTP客户端，包括aiohttp、httpx、requests、pyhttpx、curl_cffi、DrissionPage和playwright
+- **多种下载处理程序**：支持多种HTTP客户端，包括aiohttp、httpx、requests、pyhttpx、curl_cffi、DrissionPage、playwright和sbcdp
 - **灵活的中间件系统**：轻松添加自定义功能和处理逻辑
 - **强大的数据处理管道**：支持多种数据库存储选项
 - **内置信号系统**：方便的事件处理机制
@@ -94,7 +101,7 @@ AioScrapy is a powerful asynchronous web crawling framework built on Python's as
 
 
 - **Fully Asynchronous**: Built on Python's asyncio for efficient concurrent crawling
-- **Multiple Download Handlers**: Support for various HTTP clients including aiohttp, httpx, requests, pyhttpx, curl_cffi, DrissionPage and playwright
+- **Multiple Download Handlers**: Support for various HTTP clients including aiohttp, httpx, requests, pyhttpx, curl_cffi, DrissionPage, playwright and sbcdp
 - **Flexible Middleware System**: Easily add custom functionality and processing logic
 - **Powerful Data Processing Pipelines**: Support for various database storage options
 - **Built-in Signal System**: Convenient event handling mechanism
@@ -117,6 +124,49 @@ pip install aio-scrapy
 # pip install git+https://github.com/ConlinH/aio-scrapy
 ```
 
+### 开始 | Start
+```python
+from aioscrapy import Spider, logger
+
+
+class MyspiderSpider(Spider):
+    name = 'myspider'
+    custom_settings = {
+        "CLOSE_SPIDER_ON_IDLE": True
+    }
+    start_urls = ["https://quotes.toscrape.com"]
+
+    @staticmethod
+    async def process_request(request, spider):
+        """ request middleware """
+        pass
+
+    @staticmethod
+    async def process_response(request, response, spider):
+        """ response middleware """
+        return response
+
+    @staticmethod
+    async def process_exception(request, exception, spider):
+        """ exception middleware """
+        pass
+
+    async def parse(self, response):
+        for quote in response.css('div.quote'):
+            item = {
+                'author': quote.xpath('span/small/text()').get(),
+                'text': quote.css('span.text::text').get(),
+            }
+            yield item
+
+    async def process_item(self, item):
+        logger.info(item)
+
+
+if __name__ == '__main__':
+    MyspiderSpider.start()
+```
+
 ## 文档 | Documentation
 
 ## 文档目录 | Documentation Contents

{aio_scrapy-2.1.7 → aio_scrapy-2.1.9}/aio_scrapy.egg-info/SOURCES.txt

@@ -46,6 +46,7 @@ aioscrapy/core/downloader/handlers/webdriver/__init__.py
 aioscrapy/core/downloader/handlers/webdriver/drissionpage.py
 aioscrapy/core/downloader/handlers/webdriver/driverpool.py
 aioscrapy/core/downloader/handlers/webdriver/playwright.py
+aioscrapy/core/downloader/handlers/webdriver/sbcdp.py
 aioscrapy/db/__init__.py
 aioscrapy/db/absmanager.py
 aioscrapy/db/aiomongo.py
@@ -86,6 +87,7 @@ aioscrapy/libs/pipelines/excel.py
 aioscrapy/libs/pipelines/mongo.py
 aioscrapy/libs/pipelines/mysql.py
 aioscrapy/libs/pipelines/pg.py
+aioscrapy/libs/pipelines/redis.py
 aioscrapy/libs/spider/__init__.py
 aioscrapy/libs/spider/depth.py
 aioscrapy/libs/spider/httperror.py

{aio_scrapy-2.1.7 → aio_scrapy-2.1.9}/aio_scrapy.egg-info/requires.txt

@@ -28,10 +28,15 @@ XlsxWriter>=3.1.2
 pillow>=9.4.0
 requests>=2.28.2
 curl_cffi
+sbcdp
+DrissionPage
 
 [curl_cffi]
 curl_cffi>=0.6.1
 
+[dp]
+DrissionPage
+
 [execl]
 XlsxWriter>=3.1.2
 pillow>=9.4.0
@@ -53,3 +58,6 @@ pyhttpx>=2.10.4
 
 [requests]
 requests>=2.28.2
+
+[sbcdp]
+sbcdp

aio_scrapy-2.1.9/aioscrapy/VERSION

@@ -0,0 +1 @@
+2.1.9

{aio_scrapy-2.1.7 → aio_scrapy-2.1.9}/aioscrapy/core/downloader/handlers/curl_cffi.py

@@ -8,8 +8,9 @@ It supports features like browser impersonation, proxies, and cookies.
 它支持浏览器模拟、代理和Cookie等功能。
 """
 
+import asyncio
 from curl_cffi.curl import CurlError
-from curl_cffi.requests import AsyncSession
+from curl_cffi.requests import AsyncSession, Session
 
 from aioscrapy import Request
 from aioscrapy.core.downloader.handlers import BaseDownloadHandler
@@ -50,6 +51,9 @@ class CurlCffiDownloadHandler(BaseDownloadHandler):
         # SSL验证设置
         self.verify_ssl: bool = self.settings.get("VERIFY_SSL", True)
 
+        # 是否在线程中执行
+        self.use_thread: bool = self.settings.get("USE_THREAD", False)
+
     @classmethod
     def from_settings(cls, settings: Settings):
         """
@@ -160,8 +164,14 @@ class CurlCffiDownloadHandler(BaseDownloadHandler):
 
         # Perform the request
         # 执行请求
-        async with AsyncSession(**session_args) as session:
-            response = await session.request(request.method, request.url, **kwargs)
+        if self.use_thread:
+            with Session(**session_args) as session:
+                # Run the synchronous curl-cffi request in a thread pool
+                # 在线程池中运行同步的curl-cffi请求
+                response = await asyncio.to_thread(session.request, request.method, request.url, **kwargs)
+        else:
+            async with AsyncSession(**session_args) as session:
+                response = await session.request(request.method, request.url, **kwargs)
 
         # Convert curl_cffi response to HtmlResponse
         # 将curl_cffi响应转换为HtmlResponse

{aio_scrapy-2.1.7 → aio_scrapy-2.1.9}/aioscrapy/core/downloader/handlers/webdriver/__init__.py

@@ -1,2 +1,3 @@
 from .playwright import PlaywrightDownloadHandler, PlaywrightDriver
 from .drissionpage import DrissionPageDownloadHandler, DrissionPageDriver
+from .sbcdp import SbcdpDownloadHandler, SbcdpDriver