aio-scrapy 2.1.6__py3-none-any.whl → 2.1.8__py3-none-any.whl

{aio_scrapy-2.1.6.dist-info → aio_scrapy-2.1.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: aio-scrapy
-Version: 2.1.6
+Version: 2.1.8
 Summary: A high-level Web Crawling and Web Scraping framework based on Asyncio
 Home-page: https://github.com/conlin-huang/aio-scrapy.git
 Author: conlin
@@ -76,115 +76,58 @@ Dynamic: requires-dist
 Dynamic: requires-python
 Dynamic: summary
 
-<!--
-![aio-scrapy](./doc/images/aio-scrapy.png)
--->
-### aio-scrapy
+# AioScrapy
 
-An asyncio + aiolibs crawler  imitate scrapy framework
+AioScrapy是一个基于Python异步IO的强大网络爬虫框架。它的设计理念源自Scrapy，但完全基于异步IO实现，提供更高的性能和更灵活的配置选项。</br>
+AioScrapy is a powerful asynchronous web crawling framework built on Python's asyncio library. It is inspired by Scrapy but completely reimplemented with asynchronous IO, offering higher performance and more flexible configuration options.
 
-English | [中文](./doc/README_ZH.md)
+## 特性 | Features
 
-### Overview
-- aio-scrapy framework is base on opensource project Scrapy & scrapy_redis.
-- aio-scrapy implements compatibility with scrapyd.
-- aio-scrapy implements redis queue and rabbitmq queue.
-- aio-scrapy is a fast high-level web crawling and web scraping framework, used to crawl websites and extract structured data from their pages.
-- Distributed crawling/scraping.
-### Requirements
+- **完全异步**：基于Python的asyncio库，实现高效的并发爬取
+- **多种下载处理程序**：支持多种HTTP客户端，包括aiohttp、httpx、requests、pyhttpx、curl_cffi、DrissionPage和playwright
+- **灵活的中间件系统**：轻松添加自定义功能和处理逻辑
+- **强大的数据处理管道**：支持多种数据库存储选项
+- **内置信号系统**：方便的事件处理机制
+- **丰富的配置选项**：高度可定制的爬虫行为
+- **分布式爬取**：支持使用Redis和RabbitMQ进行分布式爬取
+- **数据库集成**：内置支持Redis、MySQL、MongoDB、PostgreSQL和RabbitMQ
 
-- Python 3.9+
-- Works on Linux, Windows, macOS, BSD
-
-### Install
-
-The quick way:
-
-```shell
-# Install the latest aio-scrapy
-pip install git+https://github.com/ConlinH/aio-scrapy
-
-# default
-pip install aio-scrapy
-
-# Install all dependencies
-pip install aio-scrapy[all]
-
-# When you need to use mysql/httpx/rabbitmq/mongo
-pip install aio-scrapy[aiomysql,httpx,aio-pika,mongo]
-```
-
-### Usage
-
-#### create project spider:
-
-```shell
-aioscrapy startproject project_quotes
-```
-
-```
-cd project_quotes
-aioscrapy genspider quotes
-```
-
-quotes.py
-
-```python
-from aioscrapy.spiders import Spider
-
-
-class QuotesMemorySpider(Spider):
-    name = 'QuotesMemorySpider'
-
-    start_urls = ['https://quotes.toscrape.com']
-
-    async def parse(self, response):
-        for quote in response.css('div.quote'):
-            yield {
-                'author': quote.xpath('span/small/text()').get(),
-                'text': quote.css('span.text::text').get(),
-            }
 
-        next_page = response.css('li.next a::attr("href")').get()
-        if next_page is not None:
-            yield response.follow(next_page, self.parse)
+- **Fully Asynchronous**: Built on Python's asyncio for efficient concurrent crawling
+- **Multiple Download Handlers**: Support for various HTTP clients including aiohttp, httpx, requests, pyhttpx, curl_cffi, DrissionPage and playwright
+- **Flexible Middleware System**: Easily add custom functionality and processing logic
+- **Powerful Data Processing Pipelines**: Support for various database storage options
+- **Built-in Signal System**: Convenient event handling mechanism
+- **Rich Configuration Options**: Highly customizable crawler behavior
+- **Distributed Crawling**: Support for distributed crawling using Redis and RabbitMQ
+- **Database Integration**: Built-in support for Redis, MySQL, MongoDB, PostgreSQL, and RabbitMQ
 
+## 安装 | Installation
 
-if __name__ == '__main__':
-    QuotesMemorySpider.start()
+### 要求 | Requirements
 
-```
-
-run the spider:
+- Python 3.9+
 
-```shell
-aioscrapy crawl quotes
-```
+### 使用pip安装 | Install with pip
 
-#### create single script spider:
+```bash
+pip install aio-scrapy
 
-```shell
-aioscrapy genspider single_quotes -t single
+# Install the latest aio-scrapy
+# pip install git+https://github.com/ConlinH/aio-scrapy
 ```
 
-single_quotes.py:
-
+### 开始 | Start
 ```python
-from aioscrapy.spiders import Spider
+from aioscrapy import Spider, logger
 
 
-class QuotesMemorySpider(Spider):
-    name = 'QuotesMemorySpider'
+class MyspiderSpider(Spider):
+    name = 'myspider'
     custom_settings = {
-        "USER_AGENT": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36",
-        'CLOSE_SPIDER_ON_IDLE': True,
-        # 'DOWNLOAD_DELAY': 3,
-        # 'RANDOMIZE_DOWNLOAD_DELAY': True,
-        # 'CONCURRENT_REQUESTS': 1,
-        # 'LOG_LEVEL': 'INFO'
+        "CLOSE_SPIDER_ON_IDLE": True
     }
-
-    start_urls = ['https://quotes.toscrape.com']
+    start_urls = ["https://quotes.toscrape.com"]
 
     @staticmethod
     async def process_request(request, spider):
@@ -203,49 +146,45 @@ class QuotesMemorySpider(Spider):
 
     async def parse(self, response):
         for quote in response.css('div.quote'):
-            yield {
+            item = {
                 'author': quote.xpath('span/small/text()').get(),
                 'text': quote.css('span.text::text').get(),
             }
-
-        next_page = response.css('li.next a::attr("href")').get()
-        if next_page is not None:
-            yield response.follow(next_page, self.parse)
+            yield item
 
     async def process_item(self, item):
-        print(item)
+        logger.info(item)
 
 
 if __name__ == '__main__':
-    QuotesMemorySpider.start()
-
+    MyspiderSpider.start()
 ```
 
-run the spider:
-
-```shell
-aioscrapy runspider quotes.py
-```
-
-
-### more commands:
-
-```shell
-aioscrapy -h
-```
-
-#### [more example](./example)
-
-### Documentation
-[doc](./doc/documentation.md)
-
-### Ready
-
-Please submit your suggestions to the owner by creating an issue
-
-## Thanks
-
-[aiohttp](https://github.com/aio-libs/aiohttp/)
-
-[scrapy](https://github.com/scrapy/scrapy)
-
+## 文档 | Documentation
+
+## 文档目录 | Documentation Contents
+- [安装指南 | Installation Guide](docs/installation.md)
+- [快速入门 | Quick Start](docs/quickstart.md)
+- [核心概念 | Core Concepts](docs/concepts.md)
+- [爬虫指南 | Spider Guide](docs/spiders.md)
+- [下载器 | Downloaders](docs/downloaders.md)
+- [中间件 | Middlewares](docs/middlewares.md)
+- [管道 | Pipelines](docs/pipelines.md)
+- [队列 | Queues](docs/queues.md)
+- [请求过滤器 | Request Filters](docs/dupefilters.md)
+- [代理 | Proxy](docs/proxy.md)
+- [数据库连接 | Database Connections](docs/databases.md)
+- [分布式部署 | Distributed Deployment](docs/distributed.md)
+- [配置参考 | Settings Reference](docs/settings.md)
+- [API参考 | API Reference](docs/api.md)
+- [示例 | Example](example)
+
+## 许可证 | License
+
+本项目采用MIT许可证 - 详情请查看LICENSE文件。</br>
+This project is licensed under the MIT License - see the LICENSE file for details.
+
+
+## 联系
+QQ: 995018884 </br>
+WeChat: h995018884

{aio_scrapy-2.1.6.dist-info → aio_scrapy-2.1.8.dist-info}/RECORD

@@ -1,4 +1,4 @@
-aioscrapy/VERSION,sha256=JPUCseOr-o6i21WmLdCf175ZHUFbkfgq8M6QzXpEMGM,5
+aioscrapy/VERSION,sha256=n5_8BdsibVJ4nz-ATeq6LbtB6k2zft54bCvByqyoWG8,5
 aioscrapy/__init__.py,sha256=esJeH66Mz9WV7XbotvZEjNn49jc589YZ_L2DKoD0JvA,858
 aioscrapy/__main__.py,sha256=rvTdJ0cQwbi29aucPj3jJRpccx5SBzvRcV7qvxvX2NQ,80
 aioscrapy/cmdline.py,sha256=0pusLJXryZAxU9qk6QqN89IO6Kv20gkfJBnZ8UKVg_A,22302
@@ -26,15 +26,15 @@ aioscrapy/core/scheduler.py,sha256=qF_VptLGuFa8E7mXz86tjX5vww6OJTKPxE_g8XsPqsc,2
 aioscrapy/core/scraper.py,sha256=ugO2z-ZJr8xB0S1BhGOpM3zio82a6PNykTrfbAdpd68,34045
 aioscrapy/core/downloader/__init__.py,sha256=LXjkOSuP6wj2lGgmJIH3nbQVf4r9RrlqenSZtWyZvzU,31522
 aioscrapy/core/downloader/handlers/__init__.py,sha256=Rxhrkj3QBo73HY2kb7goApfNKlfc3Mqn5olmoWxT98Q,11006
-aioscrapy/core/downloader/handlers/aiohttp.py,sha256=QQ6WzOZo2Ea_Prck37G7g3RmtfJqeIBZLboEl-8AnkM,13523
-aioscrapy/core/downloader/handlers/curl_cffi.py,sha256=MQJ-7iAZP4jCI6-D-lKHgBEPaWXN7Vy5IeROWp7FZKY,7901
-aioscrapy/core/downloader/handlers/httpx.py,sha256=bEFE8xxhZWz-1Bd1WzihBY9kSdo_k9RIgHKuk0XD_2s,8835
-aioscrapy/core/downloader/handlers/pyhttpx.py,sha256=E32REQf0p6EI6yC_36TTB-OfGbWwuQ7MrDPbdFXxwmA,8455
+aioscrapy/core/downloader/handlers/aiohttp.py,sha256=V9UenrXzdn7jr0LxpsnFZE_smwncbK76gXW4DEE4EIA,13463
+aioscrapy/core/downloader/handlers/curl_cffi.py,sha256=OmQl0RqWmlPI59FBD7h1mHqQi6e_VBbntJ35ui4IbY8,7864
+aioscrapy/core/downloader/handlers/httpx.py,sha256=tsbrhmZfTqTNhxlH4vFU6_0VvPtOPAyKJlhPBiloZOg,8790
+aioscrapy/core/downloader/handlers/pyhttpx.py,sha256=f1q5e2Cfq8jW-X2wn4ncsCgRnOQk-fqXLHy_UMxrO40,8519
 aioscrapy/core/downloader/handlers/requests.py,sha256=n0KTgbRzgNLnw2PiK2NRAC7lNHTF0d1-ZnHkFNQY41A,7795
-aioscrapy/core/downloader/handlers/webdriver/__init__.py,sha256=mzXpySCSLyzvMyLYVPUpxUNGb3zC4hLsojAaCY7gboM,127
-aioscrapy/core/downloader/handlers/webdriver/drissionpage.py,sha256=tFbUBC07Gj48bjGRWAVFmDVSeT_nx0QdU-ZawgzWcgM,21820
+aioscrapy/core/downloader/handlers/webdriver/__init__.py,sha256=TxietLeEdQfNO0hAhh6oEKmHPV72s4Z3UXcEwu-w9sw,144
+aioscrapy/core/downloader/handlers/webdriver/drissionpage.py,sha256=J_OwFHICR4ZQNxYO8Wfg2AQL4z66hu6amD24y_3XB94,21821
 aioscrapy/core/downloader/handlers/webdriver/driverpool.py,sha256=_NoCL_cRFCJtdJwkjArNfdhhfSAUWdmoZ0k7eCx7QwI,8981
-aioscrapy/core/downloader/handlers/webdriver/playwright.py,sha256=uaSUmFpGocMfHKA8hHKj428M-uwq_a4WJWg-4W_R7_w,22518
+aioscrapy/core/downloader/handlers/webdriver/playwright.py,sha256=O_GQ3Xfs7php7ezwvWK5MReoR4JjV6tiWvh7__-XF3A,22519
 aioscrapy/db/__init__.py,sha256=d3X5cqYBkV6MCXIJa8s88Yli27GKQTX94IEJyK0Gj0w,8575
 aioscrapy/db/absmanager.py,sha256=onGxA2eQJ-kC6JsKhR5afaa6tw_UVstDHyh-kkSiW-o,8480
 aioscrapy/db/aiomongo.py,sha256=MOHqy3uIwWJDXxUGZv7fwup7em7mCsZhGzQB3dEauaI,14750
@@ -47,7 +47,7 @@ aioscrapy/dupefilters/disk.py,sha256=CIOhxJ8M2-caoMIZebnAcSjQC0Pr5RIA-69_Cb2k4BA
 aioscrapy/dupefilters/redis.py,sha256=6MUpIrJsgmWMd-1Xp_oF5dD3BQO2uKuTMu3UDUPKvn4,33223
 aioscrapy/http/__init__.py,sha256=_WrJLH4NQsyG1nUhrpnecWpcy7Bf6ZTfT7xZUIcL_SM,596
 aioscrapy/http/headers.py,sha256=FyIQnUvU2n39l3cDPez5VvtYLvVCWkSpjrUkzP58UTQ,9990
-aioscrapy/http/request/__init__.py,sha256=qEFVQUHFj6WUzEjNDTBWIxxeIMxmPt4147Mbf7K6wYA,16831
+aioscrapy/http/request/__init__.py,sha256=bRUmUzyjOzsaV1wdywiGcT8VT0xfrlbjUUKwZ9vRqhQ,19990
 aioscrapy/http/request/form.py,sha256=W8Img6A6PyjIiJCOskUF442LLC-0fYnoDKWxExRjVbw,5123
 aioscrapy/http/request/json_request.py,sha256=XVuGHGkd8LLLNjQnW8TaAiqcZaM9X4N7MEEOMrp17kY,7563
 aioscrapy/http/response/__init__.py,sha256=ep9OnMNgEYF0lE6H7HghA7ziEmAOwCRNsujVXMaTsa4,17934
@@ -96,7 +96,7 @@ aioscrapy/queue/redis.py,sha256=KU31ZNciLI9xxZDxsDhtOPLtmkxZQlRPOx_1z8afdwY,4788
 aioscrapy/scrapyd/__init__.py,sha256=Ey14RVLUP7typ2XqP8RWcUum2fuFyigdhuhBBiEheIo,68
 aioscrapy/scrapyd/runner.py,sha256=L0VpRkZD6IOE9MD_QI1A9ipxu3F5mKkqwyl7QxGctFs,6747
 aioscrapy/settings/__init__.py,sha256=3TWbIDf8vtDosXo8QTiypTUGr3z3o8csKJZQnMHnZrs,37108
-aioscrapy/settings/default_settings.py,sha256=GELRr1VQWTE0Nt64euHU31ENOHFM3N7ZzDQX2uAH_2c,16023
+aioscrapy/settings/default_settings.py,sha256=5iSMC1lLoHTkqGNbQY-eQl5X5xnZFbit26aOfHuPUHE,16040
 aioscrapy/spiders/__init__.py,sha256=U5ZrpW_I-YPUUSi7VDh47mt_rwtzxpj5R0CoYFi0N18,13527
 aioscrapy/templates/project/aioscrapy.cfg,sha256=_nRHP5wtPnZaBi7wCmjWv5BgUu5NYFJZhvCTRVSipyM,112
 aioscrapy/templates/project/module/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -126,9 +126,9 @@ aioscrapy/utils/template.py,sha256=7tyOvgY7HJJLqBOPcHqpZxtuBjgUwTKkDMZkSyE_4MY,4
 aioscrapy/utils/tools.py,sha256=JdAQM4eqBVMYM5LaOqvV6GURVedcSKOTvquLcHTPWXk,10563
 aioscrapy/utils/trackref.py,sha256=umHeYm9Td8h8OtzyvOiAY6GcTna0QUm8J6PwcV_NMgU,10002
 aioscrapy/utils/url.py,sha256=K0zyUoWoeh2EseYVNe3VswnbCr6-Nj2gukhaxCFvJ9w,19669
-aio_scrapy-2.1.6.dist-info/LICENSE,sha256=QbrHw1tuFHRfXCws2HUcrsOPH93sEJ7F4JO6PcjbMiQ,1083
-aio_scrapy-2.1.6.dist-info/METADATA,sha256=IgYA1l9iPOvyBak7oQNRdDNYrfBDc_1pRcf_f2A7xzU,6767
-aio_scrapy-2.1.6.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
-aio_scrapy-2.1.6.dist-info/entry_points.txt,sha256=WWhoVHZvqhW8a5uFg97K0EP_GjG3uuCIFLkyqDICgaw,56
-aio_scrapy-2.1.6.dist-info/top_level.txt,sha256=8l08KyMt22wfX_5BmhrGH0PgwZdzZIPq-hBUa1GNir4,10
-aio_scrapy-2.1.6.dist-info/RECORD,,
+aio_scrapy-2.1.8.dist-info/LICENSE,sha256=QbrHw1tuFHRfXCws2HUcrsOPH93sEJ7F4JO6PcjbMiQ,1083
+aio_scrapy-2.1.8.dist-info/METADATA,sha256=Bezyln2dU_Sp6aU1fgPvDHmzWp4kqFxZd45EbkO6UrQ,7137
+aio_scrapy-2.1.8.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
+aio_scrapy-2.1.8.dist-info/entry_points.txt,sha256=WWhoVHZvqhW8a5uFg97K0EP_GjG3uuCIFLkyqDICgaw,56
+aio_scrapy-2.1.8.dist-info/top_level.txt,sha256=8l08KyMt22wfX_5BmhrGH0PgwZdzZIPq-hBUa1GNir4,10
+aio_scrapy-2.1.8.dist-info/RECORD,,

aioscrapy/VERSION

@@ -1 +1 @@
-2.1.6
+2.1.8

aioscrapy/core/downloader/handlers/aiohttp.py

@@ -50,7 +50,7 @@ class AioHttpDownloadHandler(BaseDownloadHandler):
 
         # Arguments to pass to aiohttp.ClientSession constructor
         # 传递给aiohttp.ClientSession构造函数的参数
-        self.aiohttp_client_session_args: dict = settings.getdict('AIOHTTP_CLIENT_SESSION_ARGS')
+        self.aiohttp_args: dict = settings.getdict('AIOHTTP_ARGS')
 
         # SSL verification setting
         # SSL验证设置
@@ -228,13 +228,13 @@ class AioHttpDownloadHandler(BaseDownloadHandler):
         if self.use_session:
             # Not recommended to use session, The abnormal phenomena will occurs when using tunnel proxy
             # 不建议使用会话，使用隧道代理时会出现异常现象
-            session = self.get_session(**self.aiohttp_client_session_args)
+            session = self.get_session(**self.aiohttp_args)
             async with session.request(request.method, request.url, **kwargs) as response:
                 content: bytes = await response.read()
         else:
             # Create a new session for each request (recommended)
             # 为每个请求创建一个新会话（推荐）
-            async with aiohttp.ClientSession(**self.aiohttp_client_session_args) as session:
+            async with aiohttp.ClientSession(**self.aiohttp_args) as session:
                 async with session.request(request.method, request.url, **kwargs) as response:
                     content: bytes = await response.read()
 

aioscrapy/core/downloader/handlers/curl_cffi.py

@@ -44,7 +44,7 @@ class CurlCffiDownloadHandler(BaseDownloadHandler):
 
         # Arguments to pass to curl_cffi AsyncSession constructor
         # 传递给curl_cffi AsyncSession构造函数的参数
-        self.httpx_client_session_args: dict = self.settings.get('CURL_CFFI_CLIENT_SESSION_ARGS', {})
+        self.curl_cffi_args: dict = self.settings.get('CURL_CFFI_ARGS', {})
 
         # SSL verification setting
         # SSL验证设置
@@ -156,7 +156,7 @@ class CurlCffiDownloadHandler(BaseDownloadHandler):
 
         # Configure curl_cffi session
         # 配置curl_cffi会话
-        session_args = self.httpx_client_session_args.copy()
+        session_args = self.curl_cffi_args.copy()
 
         # Perform the request
         # 执行请求

aioscrapy/core/downloader/handlers/httpx.py

@@ -46,7 +46,7 @@ class HttpxDownloadHandler(BaseDownloadHandler):
 
         # Arguments to pass to httpx AsyncClient constructor
         # 传递给httpx AsyncClient构造函数的参数
-        self.httpx_client_session_args: dict = self.settings.get('HTTPX_CLIENT_SESSION_ARGS', {})
+        self.httpx_args: dict = self.settings.get('HTTPX_ARGS', {})
 
         # SSL verification setting
         # SSL验证设置
@@ -147,7 +147,7 @@ class HttpxDownloadHandler(BaseDownloadHandler):
 
         # Configure httpx client session
         # 配置httpx客户端会话
-        session_args = self.httpx_client_session_args.copy()
+        session_args = self.httpx_args.copy()
         session_args.setdefault('http2', True)  # Enable HTTP/2 by default
                                                # 默认启用HTTP/2
         session_args.update({

aioscrapy/core/downloader/handlers/pyhttpx.py

@@ -46,7 +46,7 @@ class PyhttpxDownloadHandler(BaseDownloadHandler):
 
         # Arguments to pass to pyhttpx HttpSession constructor
         # 传递给pyhttpx HttpSession构造函数的参数
-        self.pyhttpx_client_args: dict = self.settings.get('PYHTTPX_CLIENT_ARGS', {})
+        self.pyhttpx_args: dict = self.settings.get('PYHTTPX_ARGS', {})
 
         # SSL verification setting
         # SSL验证设置
@@ -161,10 +161,13 @@ class PyhttpxDownloadHandler(BaseDownloadHandler):
 
         # Configure pyhttpx session
         # 配置pyhttpx会话
-        session_args = self.pyhttpx_client_args.copy()
+        session_args = self.pyhttpx_args.copy()
         session_args.setdefault('http2', True)  # Enable HTTP/2 by default
                                                # 默认启用HTTP/2
 
+        if ja3 := request.meta.get("ja3"):
+            session_args['ja3'] = ja3
+
         # Execute the request in a thread pool since pyhttpx is synchronous
         # 由于pyhttpx是同步的，在线程池中执行请求
         with pyhttpx.HttpSession(**session_args) as session:

aioscrapy/core/downloader/handlers/webdriver/__init__.py

@@ -1,2 +1,2 @@
-from .playwright import PlaywrightDriver, PlaywrightDriver
-from .drissionpage import DrissionPageHandler, DrissionPageDriver
+from .playwright import PlaywrightDownloadHandler, PlaywrightDriver
+from .drissionpage import DrissionPageDownloadHandler, DrissionPageDriver

aioscrapy/core/downloader/handlers/webdriver/drissionpage.py

@@ -273,7 +273,7 @@ class DrissionPageDriver(WebDriverBase):
             self.page.set.cookies(cookies)
 
 
-class DrissionPageHandler(BaseDownloadHandler):
+class DrissionPageDownloadHandler(BaseDownloadHandler):
     """
     Download handler that uses DrissionPage to perform browser-based HTTP requests.
     使用DrissionPage执行基于浏览器的HTTP请求的下载处理程序。
@@ -298,7 +298,7 @@ class DrissionPageHandler(BaseDownloadHandler):
 
         # Get DrissionPage client arguments from settings
         # 从设置中获取DrissionPage客户端参数
-        client_args = settings.getdict('DP_CLIENT_ARGS', {})
+        client_args = settings.getdict('DP_ARGS', {})
 
         # Configure the pool size for browser instances
         # 配置浏览器实例的池大小

aioscrapy/core/downloader/handlers/webdriver/playwright.py

@@ -278,7 +278,7 @@ class PlaywrightDriver(WebDriverBase):
         ])
 
 
-class PlaywrightHandler(BaseDownloadHandler):
+class PlaywrightDownloadHandler(BaseDownloadHandler):
     """
     Download handler that uses Playwright to perform browser-based HTTP requests.
     使用Playwright执行基于浏览器的HTTP请求的下载处理程序。
@@ -303,7 +303,7 @@ class PlaywrightHandler(BaseDownloadHandler):
 
         # Get Playwright client arguments from settings
         # 从设置中获取Playwright客户端参数
-        playwright_client_args = settings.getdict('PLAYWRIGHT_CLIENT_ARGS')
+        playwright_client_args = settings.getdict('PLAYWRIGHT_ARGS')
 
         # Set the default page load event to wait for
         # 设置要等待的默认页面加载事件

aioscrapy/http/request/__init__.py

@@ -11,9 +11,11 @@ It handles URL normalization, fingerprinting, serialization, and other request-r
 
 import hashlib
 import inspect
-import json
-from typing import Callable, List, Optional, Tuple, Type, TypeVar
+from collections import Counter
+from typing import Callable, List, Optional, Tuple, Type, TypeVar, Union
+from urllib.parse import ParseResult, parse_qsl, urlencode, urlparse
 
+import ujson
 from w3lib.url import canonicalize_url
 from w3lib.url import safe_url_string
 
@@ -23,11 +25,67 @@ from aioscrapy.utils.curl import curl_to_request_kwargs
 from aioscrapy.utils.python import to_unicode
 from aioscrapy.utils.url import escape_ajax
 
+
 # Type variable for Request class to use in class methods
 # 用于在类方法中使用的Request类的类型变量
 RequestTypeVar = TypeVar("RequestTypeVar", bound="Request")
 
 
+def _update_url_params(url: str, params: Union[dict, list, tuple]) -> str:
+    """Add URL query params to provided URL being aware of existing.
+
+    Args:
+        url: string of target URL
+        params: dict containing requested params to be added
+
+    Returns:
+        string with updated URL
+
+    >> url = 'http://stackoverflow.com/test?answers=true'
+    >> new_params = {'answers': False, 'data': ['some','values']}
+    >> update_url_params(url, new_params)
+    'http://stackoverflow.com/test?data=some&data=values&answers=false'
+    """
+    # No need to unquote, since requote_uri will be called later.
+    parsed_url = urlparse(url)
+
+    # Extracting URL arguments from parsed URL, NOTE the result is a list, not dict
+    parsed_get_args = parse_qsl(parsed_url.query, keep_blank_values=True)
+
+    # Merging URL arguments dict with new params
+    old_args_counter = Counter(x[0] for x in parsed_get_args)
+    if isinstance(params, dict):
+        params = list(params.items())
+    new_args_counter = Counter(x[0] for x in params)
+    for key, value in params:
+        # Bool and Dict values should be converted to json-friendly values
+        if isinstance(value, (bool, dict)):
+            value = ujson.dumps(value)
+        # 1 to 1 mapping, we have to search and update it.
+        if old_args_counter.get(key) == 1 and new_args_counter.get(key) == 1:
+            parsed_get_args = [
+                (x if x[0] != key else (key, value)) for x in parsed_get_args
+            ]
+        else:
+            parsed_get_args.append((key, value))
+
+    # Converting URL argument to proper query string
+    encoded_get_args = urlencode(parsed_get_args, doseq=True)
+
+    # Creating new parsed result object based on provided with new
+    # URL arguments. Same thing happens inside of urlparse.
+    new_url = ParseResult(
+        parsed_url.scheme,
+        parsed_url.netloc,
+        parsed_url.path,
+        parsed_url.params,
+        encoded_get_args,
+        parsed_url.fragment,
+    ).geturl()
+
+    return new_url
+
+
 class Request(object):
     attributes: Tuple[str, ...] = (
         "url", "callback", "method", "headers", "body",
@@ -42,7 +100,10 @@ class Request(object):
             callback: Optional[Callable] = None,
             method: str = 'GET',
             headers: Optional[dict] = None,
+            params: Optional[Union[dict, list, tuple]] = None,
             body: Optional[str] = None,
+            data: Optional[Union[dict[str, str], list[tuple], str, bytes]] = None,
+            json: Optional[dict | list] = None,
             cookies: Optional[dict] = None,
             meta: Optional[dict] = None,
             encoding: str = 'utf-8',
@@ -77,8 +138,32 @@ class Request(object):
         """
         self._encoding = encoding
         self.method = str(method).upper()
+
+        self.headers = Headers(headers or {})
+
+        # url
+        if params:
+            url = _update_url_params(url, params)
         self._set_url(url)
+
+        # body/data/json
+        if data is not None:
+            if isinstance(data, (dict, list, tuple)):
+                body = urlencode(data)
+            elif isinstance(data, str):
+                body = data
+            elif isinstance(data, bytes):
+                body = data.decode(self._encoding)
+            self.headers.setdefault('Content-Type', 'application/x-www-form-urlencoded')
+
+        if json is not None:
+            body = ujson.dumps(json, separators=(",", ":"))
+            # Set default headers for JSON content
+            # 设置JSON内容的默认头部
+            self.headers.setdefault('Content-Type', 'application/json')
+
         self._set_body(body)
+
         assert isinstance(priority, int), f"Request priority not an integer: {priority!r}"
         self.priority = priority
 
@@ -86,7 +171,6 @@ class Request(object):
         self.errback = errback
 
         self.cookies = cookies or {}
-        self.headers = Headers(headers or {})
         self.dont_filter = dont_filter
         self.use_proxy = use_proxy
 
@@ -207,7 +291,7 @@ class Request(object):
         """
         return self._body
 
-    def _set_body(self, body: str) -> None:
+    def _set_body(self, body: Optional[str]) -> None:
         """
         Set the request body.
         设置请求体。
@@ -361,7 +445,7 @@ class Request(object):
             The request fingerprint. 请求指纹。
         """
         return hashlib.sha1(
-            json.dumps({
+            ujson.dumps({
                 'method': to_unicode(self.method),
                 'url': canonicalize_url(self.url, keep_fragments=keep_fragments),
                 'body': self.body,

aioscrapy/settings/default_settings.py

@@ -167,15 +167,15 @@ DOWNLOAD_HANDLERS_MAP = {
     # playwright handlers (for JavaScript rendering)
     # playwright处理程序（用于JavaScript渲染）
     'playwright': {
-        'http': 'aioscrapy.core.downloader.handlers.webdriver.playwright.PlaywrightHandler',
-        'https': 'aioscrapy.core.downloader.handlers.webdriver.playwright.PlaywrightHandler',
+        'http': 'aioscrapy.core.downloader.handlers.webdriver.playwright.PlaywrightDownloadHandler',
+        'https': 'aioscrapy.core.downloader.handlers.webdriver.playwright.PlaywrightDownloadHandler',
     },
 
-    # DrissionPageHandler handlers (for JavaScript rendering)
-    # DrissionPageHandler处理程序（用于JavaScript渲染）
+    # DrissionPage handlers (for JavaScript rendering)
+    # DrissionPage处理程序（用于JavaScript渲染）
     'dp': {
-        'http': 'aioscrapy.core.downloader.handlers.webdriver.drissionpage.DrissionPageHandler',
-        'https': 'aioscrapy.core.downloader.handlers.webdriver.drissionpage.DrissionPageHandler',
+        'http': 'aioscrapy.core.downloader.handlers.webdriver.drissionpage.DrissionPageDownloadHandler',
+        'https': 'aioscrapy.core.downloader.handlers.webdriver.drissionpage.DrissionPageDownloadHandler',
     },
 
     # curl_cffi handlers
@@ -480,4 +480,4 @@ URLLENGTH_LIMIT = 2083
 
 # Whether to close the spider when it becomes idle (no more requests)
 # 当爬虫变为空闲状态（没有更多请求）时是否关闭爬虫
-CLOSE_SPIDER_ON_IDLE = False
+CLOSE_SPIDER_ON_IDLE = True