aio-scrapy 2.0.10__py3-none-any.whl → 2.1.2__py3-none-any.whl

{aio_scrapy-2.0.10.dist-info → aio_scrapy-2.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: aio-scrapy
-Version: 2.0.10
+Version: 2.1.2
 Summary: A high-level Web Crawling and Web Scraping framework based on Asyncio
 Home-page: https://github.com/conlin-huang/aio-scrapy.git
 Author: conlin
@@ -38,19 +38,22 @@ Requires-Dist: aiomysql >=0.1.1 ; extra == 'all'
 Requires-Dist: httpx[http2] >=0.23.0 ; extra == 'all'
 Requires-Dist: aio-pika >=8.1.1 ; extra == 'all'
 Requires-Dist: cryptography ; extra == 'all'
-Requires-Dist: motor >=3.1.1 ; extra == 'all'
+Requires-Dist: motor >=2.1.0 ; extra == 'all'
 Requires-Dist: pyhttpx >=2.10.1 ; extra == 'all'
 Requires-Dist: asyncpg >=0.27.0 ; extra == 'all'
 Requires-Dist: XlsxWriter >=3.1.2 ; extra == 'all'
 Requires-Dist: pillow >=9.4.0 ; extra == 'all'
 Requires-Dist: requests >=2.28.2 ; extra == 'all'
+Requires-Dist: curl-cffi ; extra == 'all'
+Provides-Extra: curl_cffi
+Requires-Dist: curl-cffi >=0.6.1 ; extra == 'curl_cffi'
 Provides-Extra: execl
 Requires-Dist: XlsxWriter >=3.1.2 ; extra == 'execl'
 Requires-Dist: pillow >=9.4.0 ; extra == 'execl'
 Provides-Extra: httpx
 Requires-Dist: httpx[http2] >=0.23.0 ; extra == 'httpx'
 Provides-Extra: mongo
-Requires-Dist: motor >=3.1.1 ; extra == 'mongo'
+Requires-Dist: motor >=2.1.0 ; extra == 'mongo'
 Provides-Extra: pg
 Requires-Dist: asyncpg >=0.27.0 ; extra == 'pg'
 Provides-Extra: playwright
@@ -86,7 +89,7 @@ The quick way:
 
 ```shell
 # Install the latest aio-scrapy
-pip install git+https://github.com/conlin-huang/aio-scrapy
+pip install git+https://github.com/ConlinH/aio-scrapy
 
 # default
 pip install aio-scrapy

{aio_scrapy-2.0.10.dist-info → aio_scrapy-2.1.2.dist-info}/RECORD

@@ -1,9 +1,9 @@
-aioscrapy/VERSION,sha256=bkksF7-FeZMTR8EfltCUKJZNQaHaQkySSXYbwvc2qdw,6
+aioscrapy/VERSION,sha256=P_yDzF7mglQfHrNrG3774VSyqYLfjq7rdsUEPwejXH0,5
 aioscrapy/__init__.py,sha256=esJeH66Mz9WV7XbotvZEjNn49jc589YZ_L2DKoD0JvA,858
 aioscrapy/__main__.py,sha256=rvTdJ0cQwbi29aucPj3jJRpccx5SBzvRcV7qvxvX2NQ,80
 aioscrapy/cmdline.py,sha256=1qhNg2Edl-Obmf2re2K4V8pJG7ubGfZZCzcHdKtdE_s,5159
-aioscrapy/crawler.py,sha256=k24cWw8tev93obQHNqhjdLMTPX3jVGoHRfS29n56etk,10109
-aioscrapy/exceptions.py,sha256=NjA2Rx1KZsjMgH7IOdNpxuRkh-RwylRCYvEhwgXKIb8,2027
+aioscrapy/crawler.py,sha256=6-ptivIjIGKdojOlZqXV0hV3x1Gont81tOC5u5JqIME,10330
+aioscrapy/exceptions.py,sha256=k1daw1hV_aqsaIKKibdyqcNPyVn5oUb07wmB2DRxfjs,2111
 aioscrapy/link.py,sha256=fXMqsHvYEzsuYi-sNDcElS7jV6Lusq0tjPkPUGOlyZw,1867
 aioscrapy/logformatter.py,sha256=y3etd28ACbpTbcGprJ_cQ086gxQY3k_QX_yxYFoF1AU,3028
 aioscrapy/process.py,sha256=uFkj2wzaBu0Vs3pGFKdJ4R-0Gn7hROX6EU-B5zddnyQ,1603
@@ -21,28 +21,29 @@ aioscrapy/commands/settings.py,sha256=sc0rwwfBQNySKX8uV3iJqv3i7SelFwNcrlHYxDupKO
 aioscrapy/commands/startproject.py,sha256=Rcc7JkN75Jp2t2aZIxBzPsWbLXChNAUSByDhcW_6Ig8,4001
 aioscrapy/commands/version.py,sha256=yqqTMlZkkiQhtbU9w_IqUWLMOAjqYlv24friEkPRQYM,485
 aioscrapy/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-aioscrapy/core/engine.py,sha256=zW3GPigPqyrWJ_Jk7SUxD0ueV1HTuxUvweFUU4WFG-0,10926
-aioscrapy/core/scheduler.py,sha256=YCRw9j79ZOL8bijDa3IdRaw0YlMTrwXuJGzaApkN7lc,5737
-aioscrapy/core/scraper.py,sha256=M_bcizLUzWuECe7sIIZ_HJLNrPzL7dX2o-tN5nvFnCs,10304
-aioscrapy/core/downloader/__init__.py,sha256=22TC0z49BX3YvDUPl6DKMrOonECpY5tjaWJGGEV7RbU,9574
+aioscrapy/core/engine.py,sha256=h02-K2lQqlCxvNIlURgPpnhHCbyiJRIWrFJt5Ys7vZY,9843
+aioscrapy/core/scheduler.py,sha256=czCx5oHknXuHadpISTfoEMSKXXrlwJTmLTUQtHdtaTc,7407
+aioscrapy/core/scraper.py,sha256=dh06xcSI5SHC15psF41Y7RiDxq7V59E2VtUqjInmM8g,10539
+aioscrapy/core/downloader/__init__.py,sha256=QqBDokvvEgJMRJuQ7Xs_HJpAbbS2A0Z_75HOwVXyxPo,9747
 aioscrapy/core/downloader/handlers/__init__.py,sha256=CriaX2Cp4jUqzDDGZDB7HiIEgUWt2pnYVho6HMV6sJ0,3198
-aioscrapy/core/downloader/handlers/aiohttp.py,sha256=dFVVeGgJ1WZcE1zI4fQOZIzmrkC6l1WZcYstHmB3qYg,3942
-aioscrapy/core/downloader/handlers/httpx.py,sha256=-DfjYgfrjxMhaMpTgEOFlQRONasCXV0g6UgH3WmWcfs,3041
-aioscrapy/core/downloader/handlers/pyhttpx.py,sha256=fgD6Kz_gfB17KHbnkFtUHJDjfYR-c9P2LhuYX4hcva8,2228
-aioscrapy/core/downloader/handlers/requests.py,sha256=I49YnAxFGf-_a_YR-1AOG8vPLMmKiMtdmP4Xn-c0dPw,1996
-aioscrapy/core/downloader/handlers/playwright/__init__.py,sha256=lTI7Strp7SSZxM5IfZVgJGVwv9UfhTA4LdFpsfvzZ5k,4160
+aioscrapy/core/downloader/handlers/aiohttp.py,sha256=hoQhdsOsj77HVx7Lf1CEzmwf07C1pRdO87xiazQQ5dE,4263
+aioscrapy/core/downloader/handlers/curl_cffi.py,sha256=emvIuOgW8m4dXHVchBdP-lRlQwd6c-KZ7KEq3rxHkEw,2576
+aioscrapy/core/downloader/handlers/httpx.py,sha256=aMgqlQEBaDfwNCz9uzqn4DTcs-vqy8WVRmLoHL8tSSI,3384
+aioscrapy/core/downloader/handlers/pyhttpx.py,sha256=XWe838pJxwRocIV5x1qbWXSmaczdFI_cOvHIeB0lYdg,2559
+aioscrapy/core/downloader/handlers/requests.py,sha256=XB6XAa91NUhxYJHUA0z-E-QTW8nU6BcLVZgYgD0vKA8,2362
+aioscrapy/core/downloader/handlers/playwright/__init__.py,sha256=6r8ieFGkKGqJzPMs-vjDcpbJNWzHof-dhu2qVLaf7AE,4501
 aioscrapy/core/downloader/handlers/playwright/driverpool.py,sha256=qfIdGjORdn1MookO-ucIJ8NOeLrIQ0y0UJY_xuMzM_8,1374
 aioscrapy/core/downloader/handlers/playwright/webdriver.py,sha256=QFtAT--2Ea_Gg4x1EhMidyOwQjbqljUl4sKGB_hAA00,3530
 aioscrapy/db/__init__.py,sha256=ISBXM_-cCf5CgTLc3i_emLxV163-ZAbgttkQiRxokD0,2456
 aioscrapy/db/absmanager.py,sha256=6vlPcjDHOtZCHePiUYPe6ezRnM-TB4XLhmuw7APaWDk,1162
-aioscrapy/db/aiomongo.py,sha256=A9NjJy7_jI1J3hj8rw-o1PuXhXwFghHs4xSnsO5-ZfE,2745
+aioscrapy/db/aiomongo.py,sha256=t4JpRPBBisF7_rz02Kp6AejrphLvLWg5rF-yYLIe2MI,3071
 aioscrapy/db/aiomysql.py,sha256=-xCLfeH7RzvghY1jqREAb_Qnz9q_dVjxoHGfz7sCqbU,3799
 aioscrapy/db/aiopg.py,sha256=WG4s_2X0b8LQHbZpoIrwZeuGHNolKj-SvmvAZQlCk00,3213
 aioscrapy/db/aiorabbitmq.py,sha256=tNKl4Kx7KM7H_lOj8xfeA0uD8PuBTVzySApTEn5TyAE,5583
 aioscrapy/db/aioredis.py,sha256=UOoTRTQUvghnq29bVL8v1HvksMXYOzHaS8Btgbpn0bY,2966
-aioscrapy/dupefilters/__init__.py,sha256=17s6Hyr_lWDFPto6wLEvRfT2TbGU2RIssTDuChzrDNA,1498
+aioscrapy/dupefilters/__init__.py,sha256=KPLIs9RMcl_8oD9gb3wQMRtbmRv_rTdfCp-qlekh1Qg,1684
 aioscrapy/dupefilters/disk.py,sha256=EMgxeC2a6aYCGKgp4QOs5xwHp33LUsOZ8pliKBTFx1c,1551
-aioscrapy/dupefilters/redis.py,sha256=cUuM68dEM1_ki2eOzZ6pAvmLZlAP_tC4lx73Ufmg_Bs,4812
+aioscrapy/dupefilters/redis.py,sha256=KrI_SjH--yTNlLmJRGVp1N1BgyKsy6JtT9p02LxTWTc,6969
 aioscrapy/http/__init__.py,sha256=yeQTT5W1iwr6dKznTS5d9vnx2hsB47i9roPM57wQp_0,597
 aioscrapy/http/headers.py,sha256=H-RJ6KqOsFFFAXORfvoyz3V-ud0I8TAj5Jt5fAACcLc,1573
 aioscrapy/http/request/__init__.py,sha256=PFoFU3ncTN-gj6Rx01rjVa_744Qfv3EH29mooW6JX9U,7121
@@ -58,7 +59,7 @@ aioscrapy/libs/downloader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJW
 aioscrapy/libs/downloader/defaultheaders.py,sha256=tg_ULA0Y-41bZKG607mowFJQGVfnZ45LdR044DsjA_A,563
 aioscrapy/libs/downloader/downloadtimeout.py,sha256=hNh3OEj7rC0ceQrv_yrhR5lb5AvfxJ6cspj3qsQWj4o,704
 aioscrapy/libs/downloader/ja3fingerprint.py,sha256=DgTw74GXC_Bp94eD_bwoG6A_DphUHTt7bH4glBNXyV8,1058
-aioscrapy/libs/downloader/retry.py,sha256=eaMig7JpSyr6QQBD6FNYpcttuGK811Dm4tJGTUIi3q8,5191
+aioscrapy/libs/downloader/retry.py,sha256=0670bPz5lc4wUsWmYlhYdGZdeflsQdFhJbnwK1g0c84,4441
 aioscrapy/libs/downloader/stats.py,sha256=FlkS8Zm4j3SBjHb6caXwq08HvvZ37VKORGCAjlA2U38,1376
 aioscrapy/libs/downloader/useragent.py,sha256=E5x5dk9AxsSCGDDICJlTXwWXRkqAibWgesqG0VhAG8M,743
 aioscrapy/libs/extensions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -67,10 +68,10 @@ aioscrapy/libs/extensions/corestats.py,sha256=WCZ4nnk6LUP7AdGx9mnuVm96iWMxHozxdN
 aioscrapy/libs/extensions/logstats.py,sha256=wSLbN9tmsw5I1FBxHjLfIdQo85fxJI7TmOefispaxc4,1844
 aioscrapy/libs/extensions/metric.py,sha256=cx9UnSdj6akzrPe_uwWHh_QKTNzD82VRrEjiiHOoAuc,5479
 aioscrapy/libs/extensions/throttle.py,sha256=yos2D3XZgH40G52kltMKv5_GeAK4MqpRwTu6FCErUh0,3512
-aioscrapy/libs/pipelines/__init__.py,sha256=x24am2am-aUjeX4XlYJxWQT0IS-jhKkQOL1MM-iWwzs,5709
+aioscrapy/libs/pipelines/__init__.py,sha256=XW5Ur6bhvGLo-w-tdUeIB4jkFpZxqUU9mbajfAAztb0,5642
 aioscrapy/libs/pipelines/csv.py,sha256=-PEZOt-3ndF0ePO7EnqjEqeCYMJR9wHv3XcpSq6QswI,2454
 aioscrapy/libs/pipelines/execl.py,sha256=a8sfgQCHUc0MIja9cPP4TZ6ghfkxYZuAzLDIK4_nQuo,6284
-aioscrapy/libs/pipelines/mongo.py,sha256=jiPyC3C0mNb-zlS0ecEBgl883gBtBQBFEeBR8DOcmmI,2001
+aioscrapy/libs/pipelines/mongo.py,sha256=B3dhvspxc4lmPh2noqARYV-rFuHfivdSfZ7ZlPKnk7c,2323
 aioscrapy/libs/pipelines/mysql.py,sha256=gN4DnyuXTQvDvy9Gu-v8F6sT8l7GZEa45AD0d-Ckv8s,1022
 aioscrapy/libs/pipelines/pg.py,sha256=la-SflXtGFw4IQYlOn75Brw2IfmtOUcCh0gUSz_Jg-0,990
 aioscrapy/libs/spider/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -94,8 +95,8 @@ aioscrapy/queue/redis.py,sha256=KU31ZNciLI9xxZDxsDhtOPLtmkxZQlRPOx_1z8afdwY,4788
 aioscrapy/scrapyd/__init__.py,sha256=Ey14RVLUP7typ2XqP8RWcUum2fuFyigdhuhBBiEheIo,68
 aioscrapy/scrapyd/runner.py,sha256=tewEkdNTMrBoredCbhmdrswSrF-GWsU3MLgC__ntnzQ,1777
 aioscrapy/settings/__init__.py,sha256=GuiVhezV8U2J1B-WJwSvxxeH_1YWYD_Wighr9owC4HU,15781
-aioscrapy/settings/default_settings.py,sha256=ffGA1SKEBQtmRC7UaFcNBlZrVW9PjUwukDiARqVfTXs,5432
-aioscrapy/spiders/__init__.py,sha256=vAfod_sqXs85E-QRNji_Qhf7SyWx1kXgJD8n3AhAj1g,3934
+aioscrapy/settings/default_settings.py,sha256=PrUOFYNnPIS8eCdqvRylMLBK-4tT-2MYuU6Nn8dQrx0,5639
+aioscrapy/spiders/__init__.py,sha256=oM_FzqWa46P6cjzarOO1cfDTQD2AuIPgaWZrmdMcuTI,4085
 aioscrapy/templates/project/aioscrapy.cfg,sha256=_nRHP5wtPnZaBi7wCmjWv5BgUu5NYFJZhvCTRVSipyM,112
 aioscrapy/templates/project/module/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 aioscrapy/templates/project/module/middlewares.py.tmpl,sha256=0eEf2LC0vYcWPH82HNqieYSORyUuIo3Bgl5t-neRAJ4,3469
@@ -103,7 +104,7 @@ aioscrapy/templates/project/module/pipelines.py.tmpl,sha256=-MYA7MFAffH8FTG1VGAk
 aioscrapy/templates/project/module/settings.py.tmpl,sha256=AO2jmyokUhuhFqxMvsMihPgSY4ZrldsMs-BuOEVfvQY,1421
 aioscrapy/templates/project/module/spiders/__init__.py,sha256=Zg1uss1vaNjvld9s9Ccua50SxVZwpFTPwqpBHoCrWdU,164
 aioscrapy/templates/spiders/basic.tmpl,sha256=oO1vh7-TZLjvpwdrYC49TGe-A6Kulc8UIG4Sa0QhDfI,375
-aioscrapy/templates/spiders/single.tmpl,sha256=nqB7vP_L9icsxJWiSpsSSRZ1nTxHWowTjieRIcYaYIw,942
+aioscrapy/templates/spiders/single.tmpl,sha256=Ptmo_uFDGEffvpEMyxec7sxIyBbP05x0Grhn5u6lZbQ,1011
 aioscrapy/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 aioscrapy/utils/conf.py,sha256=NkSmKjOE7xVvrAWQu4ne3jOzNGucgZdWHPhGbpz8dPU,7208
 aioscrapy/utils/curl.py,sha256=I8eZWFNgvyUiJ2YS9-s3HltGNVG8XMMU0HPhlMxuxdA,3295
@@ -114,7 +115,7 @@ aioscrapy/utils/log.py,sha256=NRDivw8w21J77qEUeqqLdC4sgdIKaj2UAP6lDvWGotM,1697
 aioscrapy/utils/misc.py,sha256=9NOssEl7CP_c6R9skxyXwmz4bd-nZ_gkw6F0EybeLTQ,3509
 aioscrapy/utils/ossignal.py,sha256=jAsCIKu17KV45-9dZwEkFJHF31Y13KP_zxY0x49j1jo,896
 aioscrapy/utils/project.py,sha256=cT98HaR5JaNmm-Y1UzSuzXj6B5S7GlmMshUfMhjpjJY,2905
-aioscrapy/utils/python.py,sha256=fMV3Y2s7AnbQ7TChBoQodqPNzGEdVA3J89W-arwswd4,4577
+aioscrapy/utils/python.py,sha256=38oD-OSjeGb3XZFJn3bt74PwGbejnBfLWC5-lkUL0g8,4462
 aioscrapy/utils/reqser.py,sha256=qjrYut6KtvGpLLd-HDM0cncNzWCtXgpH6NyERu_5A9g,487
 aioscrapy/utils/request.py,sha256=bkFaLDeebAOp7pF-7vta9LKOB2OR2s7V9jVKfA-XlqA,2418
 aioscrapy/utils/response.py,sha256=UPR1wTTAYZkLGiiIs28kJLhlF7WPrgLuW31l9LZuYKM,1341
@@ -124,9 +125,9 @@ aioscrapy/utils/template.py,sha256=HR97X4lpv2WuqhuPfzTgaBN66fYnzHVpP6zQ5IoTwcI,8
 aioscrapy/utils/tools.py,sha256=WJowViZB8XEs2CFqjVvbqXK3H5Uvf4BgWgBD_RcHMaM,2319
 aioscrapy/utils/trackref.py,sha256=0nIpelT1d5WYxALl8SGA8vHNYsh-jS0Z2lwVEAhwx8E,2019
 aioscrapy/utils/url.py,sha256=8W8tAhU7lgfPOfzKp3ejJGEcLj1i_PnA_53Jv5LpxiY,5464
-aio_scrapy-2.0.10.dist-info/LICENSE,sha256=L-UoAEM3fQSjKA7FVWxQM7gwSCbeue6gZRAnpRS_UCo,1088
-aio_scrapy-2.0.10.dist-info/METADATA,sha256=qMfSjJmZpj8xAaoGdjEC-oNULa4wYcWFwgJJm8wBQ3U,6385
-aio_scrapy-2.0.10.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-aio_scrapy-2.0.10.dist-info/entry_points.txt,sha256=WWhoVHZvqhW8a5uFg97K0EP_GjG3uuCIFLkyqDICgaw,56
-aio_scrapy-2.0.10.dist-info/top_level.txt,sha256=8l08KyMt22wfX_5BmhrGH0PgwZdzZIPq-hBUa1GNir4,10
-aio_scrapy-2.0.10.dist-info/RECORD,,
+aio_scrapy-2.1.2.dist-info/LICENSE,sha256=L-UoAEM3fQSjKA7FVWxQM7gwSCbeue6gZRAnpRS_UCo,1088
+aio_scrapy-2.1.2.dist-info/METADATA,sha256=jsrkNFC96CZ99taB-pViCiU0SVOa4FOqBbSXziaXwNg,6506
+aio_scrapy-2.1.2.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+aio_scrapy-2.1.2.dist-info/entry_points.txt,sha256=WWhoVHZvqhW8a5uFg97K0EP_GjG3uuCIFLkyqDICgaw,56
+aio_scrapy-2.1.2.dist-info/top_level.txt,sha256=8l08KyMt22wfX_5BmhrGH0PgwZdzZIPq-hBUa1GNir4,10
+aio_scrapy-2.1.2.dist-info/RECORD,,

aioscrapy/VERSION

@@ -1 +1 @@
-2.0.10
+2.1.2

aioscrapy/core/downloader/__init__.py

@@ -138,14 +138,14 @@ class Downloader(BaseDownloader):
 
     @classmethod
     async def from_crawler(cls, crawler) -> "Downloader":
+        df = crawler.settings.get('DUPEFILTER_CLASS') and await load_instance(crawler.settings['DUPEFILTER_CLASS'], crawler=crawler)
+        crawler.spider.dupefilter = df  # 将指纹绑定到Spider 在解析成功的时候 调用DUPEFILTER_CLASS的success方法
         return cls(
             crawler,
             await call_helper(DownloadHandlerManager.for_crawler, crawler),
             await call_helper(DownloaderMiddlewareManager.from_crawler, crawler),
-            proxy=crawler.settings.get("PROXY_HANDLER") and await load_instance(crawler.settings["PROXY_HANDLER"],
-                                                                                crawler=crawler),
-            dupefilter=crawler.settings.get('DUPEFILTER_CLASS') and await load_instance(
-                crawler.settings['DUPEFILTER_CLASS'], crawler=crawler)
+            proxy=crawler.settings.get("PROXY_HANDLER") and await load_instance(crawler.settings["PROXY_HANDLER"], crawler=crawler),
+            dupefilter=df
         )
 
     async def fetch(self, request: Request) -> None:
@@ -204,6 +204,7 @@ class Downloader(BaseDownloader):
             slot.transferring.remove(request)
             slot.active.remove(request)
             self.active.remove(request)
+            self.dupefilter and not request.dont_filter and await self.dupefilter.done(request, done_type="request_done")
             if isinstance(result, Response):
                 await self.signals.send_catch_log(signal=signals.response_downloaded,
                                                   response=result,

aioscrapy/core/downloader/handlers/aiohttp.py

@@ -4,9 +4,11 @@ import ssl
 from typing import Optional
 
 import aiohttp
+from aiohttp.client_exceptions import ClientError
 
 from aioscrapy import Request
 from aioscrapy.core.downloader.handlers import BaseDownloadHandler
+from aioscrapy.exceptions import DownloadError
 from aioscrapy.http import HtmlResponse
 from aioscrapy.settings import Settings
 from aioscrapy.utils.log import logger
@@ -32,6 +34,12 @@ class AioHttpDownloadHandler(BaseDownloadHandler):
         return self.session
 
     async def download_request(self, request: Request, _) -> HtmlResponse:
+        try:
+            return await self._download_request(request)
+        except ClientError as e:
+            raise DownloadError from e
+
+    async def _download_request(self, request: Request) -> HtmlResponse:
         kwargs = {
             'verify_ssl': request.meta.get('verify_ssl', self.verify_ssl),
             'timeout': request.meta.get('download_timeout', 180),

aioscrapy/core/downloader/handlers/curl_cffi.py

@@ -0,0 +1,67 @@
+from curl_cffi.curl import CurlError
+from curl_cffi.requests import AsyncSession
+
+from aioscrapy import Request
+from aioscrapy.core.downloader.handlers import BaseDownloadHandler
+from aioscrapy.exceptions import DownloadError
+from aioscrapy.http import HtmlResponse
+from aioscrapy.settings import Settings
+from aioscrapy.utils.log import logger
+
+
+class CurlCffiDownloadHandler(BaseDownloadHandler):
+
+    def __init__(self, settings):
+        self.settings: Settings = settings
+        self.httpx_client_session_args: dict = self.settings.get('CURL_CFFI_CLIENT_SESSION_ARGS', {})
+        self.verify_ssl: bool = self.settings.get("VERIFY_SSL", True)
+
+    @classmethod
+    def from_settings(cls, settings: Settings):
+        return cls(settings)
+
+    async def download_request(self, request: Request, _) -> HtmlResponse:
+        try:
+            return await self._download_request(request)
+        except CurlError as e:
+            raise DownloadError from e
+
+    async def _download_request(self, request: Request) -> HtmlResponse:
+        kwargs = {
+            'timeout': self.settings.get('DOWNLOAD_TIMEOUT'),
+            'cookies': dict(request.cookies),
+            'verify': request.meta.get('verify_ssl', self.verify_ssl),
+            'allow_redirects': self.settings.getbool('REDIRECT_ENABLED', True) if request.meta.get(
+                'dont_redirect') is None else request.meta.get('dont_redirect'),
+            'impersonate': request.meta.get('impersonate'),
+        }
+        post_data = request.body or None
+        if isinstance(post_data, dict):
+            kwargs['json'] = post_data
+        else:
+            kwargs['data'] = post_data
+
+        headers = request.headers or self.settings.get('DEFAULT_REQUEST_HEADERS')
+        kwargs['headers'] = headers
+
+        proxy = request.meta.get("proxy")
+        if proxy:
+            kwargs["proxies"] = {'http': proxy, 'https': proxy}
+            logger.debug(f"use proxy {proxy}: {request.url}")
+
+        session_args = self.httpx_client_session_args.copy()
+
+        async with AsyncSession(**session_args) as session:
+            response = await session.request(request.method, request.url, **kwargs)
+
+        return HtmlResponse(
+            str(response.url),
+            status=response.status_code,
+            headers=response.headers,
+            body=response.content,
+            cookies={j.name: j.value or '' for j in response.cookies.jar},
+            encoding=response.encoding
+        )
+
+    async def close(self):
+        pass

aioscrapy/core/downloader/handlers/httpx.py

@@ -1,9 +1,11 @@
 import ssl
 
 import httpx
+from httpx import HTTPError as HttpxError
 
 from aioscrapy import Request
 from aioscrapy.core.downloader.handlers import BaseDownloadHandler
+from aioscrapy.exceptions import DownloadError
 from aioscrapy.http import HtmlResponse
 from aioscrapy.settings import Settings
 from aioscrapy.utils.log import logger
@@ -27,6 +29,12 @@ class HttpxDownloadHandler(BaseDownloadHandler):
         return cls(settings)
 
     async def download_request(self, request: Request, _) -> HtmlResponse:
+        try:
+            return await self._download_request(request)
+        except HttpxError as e:
+            raise DownloadError from e
+
+    async def _download_request(self, request: Request) -> HtmlResponse:
         kwargs = {
             'timeout': self.settings.get('DOWNLOAD_TIMEOUT'),
             'cookies': dict(request.cookies),
@@ -68,7 +76,7 @@ class HttpxDownloadHandler(BaseDownloadHandler):
             status=response.status_code,
             headers=response.headers,
             body=content,
-            cookies=dict(response.cookies),
+            cookies={j.name: j.value or '' for j in response.cookies.jar},
             encoding=response.encoding
         )
 

aioscrapy/core/downloader/handlers/playwright/__init__.py

@@ -1,11 +1,13 @@
 from functools import wraps
 
+from playwright._impl._api_types import Error
 from playwright.async_api._generated import Response as EventResponse
 
-from aioscrapy import Request
+from aioscrapy import Request, Spider
 from aioscrapy.core.downloader.handlers import BaseDownloadHandler
 from aioscrapy.core.downloader.handlers.playwright.driverpool import WebDriverPool
 from aioscrapy.core.downloader.handlers.playwright.webdriver import PlaywrightDriver
+from aioscrapy.exceptions import DownloadError
 from aioscrapy.http import PlaywrightResponse
 from aioscrapy.settings import Settings
 from aioscrapy.utils.tools import call_helper
@@ -24,7 +26,13 @@ class PlaywrightHandler(BaseDownloadHandler):
     def from_settings(cls, settings: Settings):
         return cls(settings)
 
-    async def download_request(self, request: Request, spider) -> PlaywrightResponse:
+    async def download_request(self, request: Request, spider: Spider) -> PlaywrightResponse:
+        try:
+            return await self._download_request(request, spider)
+        except Error as e:
+            raise DownloadError from e
+
+    async def _download_request(self, request: Request, spider) -> PlaywrightResponse:
         cookies = dict(request.cookies)
         timeout = request.meta.get('download_timeout', 30) * 1000
         user_agent = request.headers.get("User-Agent")

aioscrapy/core/downloader/handlers/pyhttpx.py

@@ -1,9 +1,11 @@
 import asyncio
 
 import pyhttpx
+from pyhttpx.exception import BaseExpetion as PyHttpxError
 
 from aioscrapy import Request
 from aioscrapy.core.downloader.handlers import BaseDownloadHandler
+from aioscrapy.exceptions import DownloadError
 from aioscrapy.http import HtmlResponse
 from aioscrapy.settings import Settings
 from aioscrapy.utils.log import logger
@@ -22,6 +24,12 @@ class PyhttpxDownloadHandler(BaseDownloadHandler):
         return cls(settings)
 
     async def download_request(self, request: Request, _) -> HtmlResponse:
+        try:
+            return await self._download_request(request)
+        except PyHttpxError as e:
+            raise DownloadError from e
+
+    async def _download_request(self, request: Request) -> HtmlResponse:
         kwargs = {
             'timeout': self.settings.get('DOWNLOAD_TIMEOUT'),
             'cookies': dict(request.cookies),

aioscrapy/core/downloader/handlers/requests.py

@@ -1,9 +1,11 @@
 import asyncio
 
 import requests
+from requests.exceptions import RequestException as RequestsError
 
 from aioscrapy import Request
 from aioscrapy.core.downloader.handlers import BaseDownloadHandler
+from aioscrapy.exceptions import DownloadError
 from aioscrapy.http import HtmlResponse
 from aioscrapy.settings import Settings
 from aioscrapy.utils.log import logger
@@ -21,6 +23,12 @@ class RequestsDownloadHandler(BaseDownloadHandler):
         return cls(settings)
 
     async def download_request(self, request: Request, _) -> HtmlResponse:
+        try:
+            return await self._download_request(request)
+        except RequestsError as e:
+            raise DownloadError from e
+
+    async def _download_request(self, request: Request) -> HtmlResponse:
         kwargs = {
             'timeout': self.settings.get('DOWNLOAD_TIMEOUT'),
             'cookies': dict(request.cookies),
@@ -48,7 +56,7 @@ class RequestsDownloadHandler(BaseDownloadHandler):
             status=response.status_code,
             headers=response.headers,
             body=response.content,
-            cookies=dict(response.cookies),
+            cookies={k: v or '' for k, v in response.cookies.items()},
             encoding=response.encoding
         )
 

aioscrapy/core/engine.py

@@ -42,8 +42,6 @@ class ExecutionEngine(object):
         self.signals = crawler.signals
         self.logformatter = crawler.logformatter
 
-        self.enqueue_cache_num = self.settings.getint("ENQUEUE_CACHE_NUM")
-        self.enqueue_cache: Queue = Queue(self.enqueue_cache_num)
         self.slot: Optional[Slot] = None
         self.spider: Optional[Spider] = None
         self.downloader: Optional[DownloaderTV] = None
@@ -53,7 +51,6 @@ class ExecutionEngine(object):
         self.running: bool = False
         self.unlock: bool = True
         self.finish: bool = False
-        self.enqueue_unlock: bool = True
 
     async def start(
             self,
@@ -70,7 +67,6 @@ class ExecutionEngine(object):
         while not self.finish:
             self.running and await self._next_request()
             await asyncio.sleep(1)
-            self.enqueue_cache_num != 1 and create_task(self._crawl())
             self.running and await self._spider_idle(self.spider)
 
     async def stop(self, reason: str = 'shutdown') -> None:
@@ -81,7 +77,6 @@ class ExecutionEngine(object):
 
         while not self.is_idle():
             await asyncio.sleep(0.2)
-            self.enqueue_cache_num != 1 and create_task(self._crawl())
         await self.close_spider(self.spider, reason=reason)
         await self.signals.send_catch_log_deferred(signal=signals.engine_stopped)
         self.finish = True
@@ -212,27 +207,8 @@ class ExecutionEngine(object):
         return True
 
     async def crawl(self, request: Request) -> None:
-        if self.enqueue_cache_num == 1:
-            await self.scheduler.enqueue_request(request)
-            create_task(self._next_request())
-        else:
-            await self.enqueue_cache.put(request)
-
-    async def _crawl(self) -> None:
-        if not self.enqueue_unlock:
-            return
-        self.enqueue_unlock = False
-        requests = []
-        for _ in range(self.enqueue_cache.qsize()):
-            try:
-                request = self.enqueue_cache.get_nowait()
-                requests.append(request)
-            except QueueEmpty:
-                break
-        if requests:
-            await call_helper(self.scheduler.enqueue_request_batch, requests)
-            create_task(self._next_request())
-        self.enqueue_unlock = True
+        await self.scheduler.enqueue_request(request)
+        # create_task(self._next_request())
 
     async def close_spider(self, spider: Spider, reason: str = 'cancelled') -> None:
         """Close (cancel) spider and clear all its outstanding requests"""
@@ -276,7 +252,6 @@ class ExecutionEngine(object):
         # method of 'has_pending_requests' has IO, so method of 'is_idle' execute twice
         if self.is_idle() \
                 and self.slot.start_requests is None \
-                and self.enqueue_unlock and self.enqueue_cache.empty() \
                 and not await self.scheduler.has_pending_requests() \
                 and self.is_idle():
             await self.stop(reason='finished')

aioscrapy/core/scheduler.py

@@ -31,7 +31,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta):
     @classmethod
     async def from_crawler(cls, crawler: "aioscrapy.Crawler") -> "BaseScheduler":
         """
-        Factory method which receives the current :class:`~scrapy.crawler.Crawler` object as argument.
+        Factory method which receives the current :class:`~aioscrapy.crawler.Crawler` object as argument.
         """
         return cls()
 
@@ -103,20 +103,27 @@ class Scheduler(BaseScheduler):
             queue: AbsQueue,
             spider: aioscrapy.Spider,
             stats=Optional[StatsCollector],
-            persist: bool = True
+            persist: bool = True,
+            cache_queue: Optional[AbsQueue] = None
     ):
+
         self.queue = queue
+        self.cache_queue = cache_queue
         self.spider = spider
         self.stats = stats
         self.persist = persist
 
     @classmethod
     async def from_crawler(cls: Type[SchedulerTV], crawler: "aioscrapy.Crawler") -> SchedulerTV:
+        cache_queue = None
+        if crawler.settings.getbool('USE_SCHEDULER_QUEUE_CACHE', False):
+            cache_queue = await load_instance('aioscrapy.queue.memory.SpiderPriorityQueue', spider=crawler.spider)
         instance = cls(
             await load_instance(crawler.settings['SCHEDULER_QUEUE_CLASS'], spider=crawler.spider),
             crawler.spider,
             stats=crawler.stats,
-            persist=crawler.settings.getbool('SCHEDULER_PERSIST', True)
+            persist=crawler.settings.getbool('SCHEDULER_PERSIST', True),
+            cache_queue=cache_queue
         )
 
         if crawler.settings.getbool('SCHEDULER_FLUSH_ON_START', False):
@@ -128,8 +135,20 @@ class Scheduler(BaseScheduler):
         return instance
 
     async def close(self, reason: str) -> None:
+
         if not self.persist:
             await self.flush()
+            return
+
+        # 如果持久化，将缓存中的任务放回到redis等分布式队列中
+        if self.cache_queue is not None:
+            while True:
+                temp = []
+                async for request in self.cache_queue.pop(2000):
+                    temp.append(request)
+                temp and await self.queue.push_batch(temp)
+                if len(temp) < 2000:
+                    break
 
     async def flush(self) -> None:
         await call_helper(self.queue.clear)
@@ -141,16 +160,37 @@ class Scheduler(BaseScheduler):
         return True
 
     async def enqueue_request(self, request: aioscrapy.Request) -> bool:
-        await call_helper(self.queue.push, request)
+        """
+        如果启用了缓存队列(USE_SCHEDULER_QUEUE_CACHE)，则优先将任务放到缓存队列中
+        """
+        if self.cache_queue is not None:
+            await call_helper(self.cache_queue.push, request)
+        else:
+            await call_helper(self.queue.push, request)
         if self.stats:
             self.stats.inc_value(self.queue.inc_key, spider=self.spider)
         return True
 
     async def next_request(self, count: int = 1) -> Optional[aioscrapy.Request]:
+        """
+        如果启用了缓存队列(USE_SCHEDULER_QUEUE_CACHE)，则优先从缓存队列中获取任务，然后从redis等分布式队列中获取任务
+        """
+        flag = False
+        if self.cache_queue is not None:
+            async for request in self.cache_queue.pop(count):
+                if request and self.stats:
+                    self.stats.inc_value(self.queue.inc_key, spider=self.spider)
+                yield request
+                flag = True
+
+        if flag:
+            return
+
         async for request in self.queue.pop(count):
             if request and self.stats:
                 self.stats.inc_value(self.queue.inc_key, spider=self.spider)
             yield request
 
     async def has_pending_requests(self) -> bool:
-        return await call_helper(self.queue.len) > 0
+        return await call_helper(self.queue.len) if self.cache_queue is None \
+            else (await call_helper(self.queue.len) + await call_helper(self.cache_queue.len)) > 0

aioscrapy/core/scraper.py

@@ -110,8 +110,8 @@ class Scraper:
                     await self.handle_spider_error(e, request, result)
                 else:
                     await self.handle_spider_output(output, request, result)
-            except BaseException:
-                logger.exception('Scraper bug processing %(request)s' % {'request': request})
+            except BaseException as e:
+                await self.handle_spider_error(e, request, result)
             finally:
                 if isinstance(result, PlaywrightResponse):
                     await result.release()
@@ -161,17 +161,23 @@ class Scraper:
         """Iter each Request/Item (given in the output parameter) returned from the given spider"""
         if not result:
             return
-
+        parser_successful = True
         while True:
             try:
                 output = await result.__anext__()
             except StopAsyncIteration:
                 break
             except Exception as e:
+                parser_successful = False
                 await self.handle_spider_error(e, request, response)
             else:
                 await self._process_spidermw_output(output, request, response)
 
+        self.spider.dupefilter and \
+        not request.dont_filter and \
+        parser_successful and \
+        await self.spider.dupefilter.done(request, done_type="parse_done")
+
     async def _process_spidermw_output(self, output: Any, request: Request, response: Response) -> None:
         """Process each Request/Item (given in the output parameter) returned from the given spider"""
 

aioscrapy/crawler.py

@@ -234,9 +234,12 @@ class CrawlerProcess(CrawlerRunner):
         finally:
             await self.recycle_db_connect()
 
-    def start(self) -> None:
+    def start(self, use_windows_selector_eventLoop: bool = False) -> None:
         if sys.platform.startswith('win'):
-            asyncio.set_event_loop(asyncio.windows_events.ProactorEventLoop())
+            if use_windows_selector_eventLoop:
+                asyncio.set_event_loop_policy(asyncio.windows_events.WindowsSelectorEventLoopPolicy())
+            else:
+                asyncio.set_event_loop(asyncio.windows_events.ProactorEventLoop())
         else:
             try:
                 import uvloop

aioscrapy/db/aiomongo.py

@@ -1,7 +1,9 @@
 from motor.motor_asyncio import AsyncIOMotorClient
+from pymongo.errors import NetworkTimeout
 
 import aioscrapy
 from aioscrapy.db.absmanager import AbsDBPoolManager
+from loguru import logger
 
 
 class MongoExecutor:
@@ -9,10 +11,16 @@ class MongoExecutor:
         self.alias = alias
         self.pool_manager = pool_manager
 
-    async def insert(self, table_name, values, db_name=None):
+    async def insert(self, table_name, values, db_name=None, ordered=False, retry_times=3):
         client, db_name_default = self.pool_manager.get_pool(self.alias)
         db_name = db_name or db_name_default
-        return await client[f'{db_name}'][f'{table_name}'].insert_many(values)
+        for _ in range(retry_times):
+            try:
+                return await client[f'{db_name}'][f'{table_name}'].insert_many(values, ordered=ordered)
+            except NetworkTimeout:
+                logger.warning("mongo insert error by NetworkTimeout, retrying...")
+
+        raise NetworkTimeout
 
     def __getattr__(self, table_name: str):
         client, db_name_default = self.pool_manager.get_pool(self.alias)

aioscrapy/dupefilters/__init__.py

@@ -1,3 +1,4 @@
+from typing import Literal
 from abc import ABCMeta, abstractmethod
 
 from aioscrapy import Request, Spider
@@ -37,3 +38,6 @@ class DupeFilterBase(metaclass=ABCMeta):
             self.logdupes = False
 
         spider.crawler.stats.inc_value('dupefilter/filtered', spider=spider)
+
+    async def done(self, request: Request, done_type: Literal["request_done", "parse_done"]) -> None:
+        """ deal fingerprint on task successful """

aioscrapy/dupefilters/redis.py

@@ -1,3 +1,5 @@
+from typing import Literal
+
 from aioscrapy import Request
 from aioscrapy.db import db_manager
 from aioscrapy.dupefilters import DupeFilterBase
@@ -128,5 +130,50 @@ class RedisBloomDupeFilter(RedisRFPDupeFilter):
         return False
 
 
+class RedisBloomSetDupeFilter(RedisBloomDupeFilter):
+
+    def __init__(self, server, key, key_set, ttl, debug, bit, hash_number, keep_on_close, info):
+        super().__init__(server, key, debug, bit, hash_number, keep_on_close, info)
+        self.key_set = key_set
+        self.ttl = ttl
+
+    @classmethod
+    async def from_crawler(cls, crawler: "aioscrapy.crawler.Crawler"):
+        server = db_manager.redis.queue
+        dupefilter_key = crawler.settings.get("SCHEDULER_DUPEFILTER_KEY", '%(spider)s:bloomfilter')
+        keep_on_close = crawler.settings.getbool("KEEP_DUPEFILTER_DATA_ON_CLOSE", True)
+        key = dupefilter_key % {'spider': crawler.spider.name}
+        debug = crawler.settings.getbool('DUPEFILTER_DEBUG', False)
+        info = crawler.settings.getbool('DUPEFILTER_INFO', False)
+        bit = crawler.settings.getint('BLOOMFILTER_BIT', 30)
+        hash_number = crawler.settings.getint('BLOOMFILTER_HASH_NUMBER', 6)
+        ttl = crawler.settings.getint('DUPEFILTER_SET_KEY_TTL', 180)
+        return cls(server, key=key, key_set=key + "_set", ttl=ttl, debug=debug, bit=bit, hash_number=hash_number,
+                   keep_on_close=keep_on_close, info=info)
+
+    async def request_seen(self, request: Request) -> bool:
+        fp = await self.bf.exists(request.fingerprint)
+        if fp:
+            return True
+        async with self.server.pipeline() as pipe:
+            pipe.sadd(self.key_set, request.fingerprint)
+            pipe.expire(self.key_set, self.ttl)
+            ret, _ = await pipe.execute()
+        return ret == 0
+
+    async def done(self, request: Request, done_type: Literal["request_done", "parse_done"]):
+        print(done_type)
+        if done_type == "request_done":
+            await self.server.srem(self.key_set, request.fingerprint)
+        elif done_type == "parse_done":
+            await self.bf.insert(request.fingerprint)
+
+    async def close(self, reason=''):
+        if not self.keep_on_close:
+            await self.clear()
+        await self.server.delete(self.key_set)
+
+
 RFPDupeFilter = RedisRFPDupeFilter
 BloomDupeFilter = RedisBloomDupeFilter
+BloomSetDupeFilter = RedisBloomSetDupeFilter

aioscrapy/exceptions.py

@@ -91,3 +91,8 @@ class ContractFail(AssertionError):
 
 class ProxyException(Exception):
     pass
+
+
+class DownloadError(Exception):
+    """下载页面时发生的错误"""
+    pass

aioscrapy/libs/downloader/retry.py

@@ -10,62 +10,21 @@ Failed pages are collected on the scraping process and rescheduled at the end,
 once the spider has finished crawling all regular (non failed) pages.
 """
 from typing import Optional, Union
-from aioscrapy.exceptions import ProxyException
+
+from anyio import EndOfStream
 
 try:
     from asyncio.exceptions import TimeoutError
 except:
     from concurrent.futures._base import TimeoutError
 
-NEED_RETRY_ERROR = (TimeoutError, ConnectionRefusedError, IOError, ProxyException)
-
-try:
-    from aiohttp.client_exceptions import ClientError
-
-    NEED_RETRY_ERROR += (ClientError,)
-except ImportError:
-    pass
-
-try:
-    from anyio import EndOfStream
-
-    NEED_RETRY_ERROR += (EndOfStream,)
-except ImportError:
-    pass
-
-try:
-    from httpx import HTTPError as HttpxError
-
-    NEED_RETRY_ERROR += (HttpxError,)
-except ImportError:
-    pass
-
-try:
-    from pyhttpx.exception import BaseExpetion as PyHttpxError
-
-    NEED_RETRY_ERROR += (PyHttpxError,)
-except ImportError:
-    pass
-
-try:
-    from requests.exceptions import RequestException as RequestsError
-
-    NEED_RETRY_ERROR += (RequestsError,)
-except ImportError:
-    pass
-
-try:
-    from playwright._impl._api_types import Error as PlaywrightError
-
-    NEED_RETRY_ERROR += (PlaywrightError,)
-except ImportError:
-    pass
-
-from aioscrapy.exceptions import NotConfigured
+from aioscrapy.exceptions import ProxyException, DownloadError, NotConfigured
 from aioscrapy.http.request import Request
 from aioscrapy.spiders import Spider
-from aioscrapy.utils.python import global_object_name
 from aioscrapy.utils.log import logger as retry_logger
+from aioscrapy.utils.python import global_object_name
+
+NEED_RETRY_ERROR = (TimeoutError, ConnectionRefusedError, IOError, ProxyException, DownloadError, EndOfStream)
 
 
 def get_retry_request(

aioscrapy/libs/pipelines/__init__.py

@@ -8,15 +8,13 @@ class SqlFormat:
 
     @staticmethod
     def pg_insert(table: str, fields: list, *args) -> str:
-        fields = ','.join(fields)
         placeholder = ','.join([f'${i + 1}' for i in range(len(fields))])
-        return f'''INSERT INTO {table} ({fields}) VALUES ({placeholder})'''
+        return f'''INSERT INTO {table} ({",".join(fields)}) VALUES ({placeholder})'''
 
     @staticmethod
     def pg_ignore_insert(table: str, fields: list, *args) -> str:
         placeholder = ','.join([f'${i + 1}' for i in range(len(fields))])
-        fields = ','.join(fields)
-        return f'INSERT INTO {table} ({fields}) VALUES ({placeholder}) ON CONFLICT DO NOTHING'
+        return f'''INSERT INTO {table} ({",".join(fields)}) VALUES ({placeholder}) ON CONFLICT DO NOTHING'''
 
     @staticmethod
     def pg_update_insert(table: str, fields: list, update_fields: list, on_conflict: str, *args) -> str:
@@ -25,8 +23,7 @@ class SqlFormat:
         if not update_fields:
             update_fields = fields
         update_fields = ','.join([f"{key} = excluded.{key}" for key in update_fields])
-        fields = ','.join(fields)
-        return f'INSERT INTO {table} ({fields}) VALUES ({placeholder}) ON CONFLICT({on_conflict}) DO UPDATE SET {update_fields}'
+        return f'''INSERT INTO {table} ({",".join(fields)}) VALUES ({placeholder}) ON CONFLICT({on_conflict}) DO UPDATE SET {update_fields}'''
 
     @staticmethod
     def mysql_insert(table: str, fields: list, *args) -> str:

aioscrapy/libs/pipelines/mongo.py

@@ -9,6 +9,8 @@ class MongoPipeline(DBPipelineBase):
     def __init__(self, settings, db_type: str):
         super().__init__(settings, db_type)
         self.db_cache = {}
+        self.ordered_cache = {}
+        self.retry_times = settings.getint("MONGO_TIMEOUT_RETRY_TIMES", 3)
 
     @classmethod
     def from_settings(cls, settings):
@@ -17,17 +19,19 @@ class MongoPipeline(DBPipelineBase):
     def parse_item_to_cache(self, item: dict, save_info: dict):
         db_name = save_info.get('db_name')
         table_name = save_info.get('table_name')
+        ordered = save_info.get('ordered', False)
         assert table_name is not None, 'please set table_name'
         db_alias = save_info.get('db_alias', ['default'])
         if isinstance(db_alias, str):
             db_alias = [db_alias]
 
-        cache_key = ''.join(db_alias) + (db_name or '') + table_name
+        cache_key = ''.join(db_alias) + (db_name or '') + table_name + str(ordered)
 
         if self.table_cache.get(cache_key) is None:
             self.db_alias_cache[cache_key] = db_alias
             self.table_cache[cache_key] = table_name
             self.db_cache[cache_key] = db_name
+            self.ordered_cache[cache_key] = ordered
             self.item_cache[cache_key] = []
 
         self.item_cache[cache_key].append(item)
@@ -40,7 +44,8 @@ class MongoPipeline(DBPipelineBase):
                 try:
                     executor = db_manager.mongo.executor(alias)
                     result = await executor.insert(
-                        table_name, self.item_cache[cache_key], db_name=self.db_cache[cache_key]
+                        table_name, self.item_cache[cache_key], db_name=self.db_cache[cache_key],
+                        ordered=self.ordered_cache[cache_key], retry_times=self.retry_times
                     )
                     logger.info(
                         f'table:{alias}->{table_name} sum:{len(self.item_cache[cache_key])} ok:{len(result.inserted_ids)}'

aioscrapy/settings/default_settings.py

@@ -72,6 +72,10 @@ DOWNLOAD_HANDLERS_MAP = {
         'http': 'aioscrapy.core.downloader.handlers.playwright.PlaywrightHandler',
         'https': 'aioscrapy.core.downloader.handlers.playwright.PlaywrightHandler',
     },
+    'curl_cffi': {
+        'http': 'aioscrapy.core.downloader.handlers.curl_cffi.CurlCffiDownloadHandler',
+        'https': 'aioscrapy.core.downloader.handlers.curl_cffi.CurlCffiDownloadHandler',
+    },
 }
 
 DOWNLOAD_TIMEOUT = 180  # 3mins

aioscrapy/spiders/__init__.py

@@ -22,6 +22,7 @@ class Spider(object):
 
     name: Optional[str] = None
     proxy: Optional["aioscrapy.proxy.AbsProxy"] = None
+    dupefilter: Optional["aioscrapy.dupefilters.DupeFilterBase"] = None
     custom_settings: Optional[dict] = None
     stats: Optional[StatsCollector] = None
 
@@ -77,7 +78,7 @@ class Spider(object):
             yield Request(url)
 
     async def request_from_dict(self, d: dict):
-        """集成后重写改方法，将队列中的json根据情况构建成Request对象"""
+        """继承成后重写改方法，将队列中的json根据情况构建成Request对象"""
         pass
 
     async def _parse(self, response: Response, **kwargs):
@@ -106,7 +107,7 @@ class Spider(object):
     __repr__ = __str__
 
     @classmethod
-    def start(cls, setting_path=None):
+    def start(cls, setting_path=None, use_windows_selector_eventLoop: bool = False):
         from aioscrapy.crawler import CrawlerProcess
         from aioscrapy.utils.project import get_project_settings
 
@@ -115,7 +116,7 @@ class Spider(object):
             settings.setmodule(setting_path)
         cp = CrawlerProcess(settings)
         cp.crawl(cls)
-        cp.start()
+        cp.start(use_windows_selector_eventLoop)
 
     def spider_idle(self):
         if not self.close_on_idle:

aioscrapy/templates/spiders/single.tmpl

@@ -24,11 +24,12 @@ class $classname(Spider):
         pass
 
     async def parse(self, response):
-        item = {
-            'author': quote.xpath('span/small/text()').get(),
-            'text': quote.css('span.text::text').get(),
-        }
-        yield item
+        for quote in response.css('div.quote'):
+            item = {
+                'author': quote.xpath('span/small/text()').get(),
+                'text': quote.css('span.text::text').get(),
+            }
+            yield item
 
     async def process_item(self, item):
         logger.info(item)

aioscrapy/utils/python.py

@@ -1,16 +1,12 @@
 """
 This module contains essential stuff that should've come with Python itself ;)
 """
-import errno
 import gc
-import inspect
 import re
 import sys
-import warnings
 import weakref
-from functools import partial, wraps
+from functools import wraps
 
-from aioscrapy.exceptions import AioScrapyDeprecationWarning
 from aioscrapy.utils.decorators import deprecated
 
 
@@ -150,4 +146,3 @@ if hasattr(sys, "pypy_version_info"):
 else:
     def garbage_collect():
         gc.collect()
-