aio-scrapy 2.1.0__py3-none-any.whl → 2.1.3__py3-none-any.whl

{aio_scrapy-2.1.0.dist-info → aio_scrapy-2.1.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: aio-scrapy
-Version: 2.1.0
+Version: 2.1.3
 Summary: A high-level Web Crawling and Web Scraping framework based on Asyncio
 Home-page: https://github.com/conlin-huang/aio-scrapy.git
 Author: conlin
@@ -38,7 +38,7 @@ Requires-Dist: aiomysql >=0.1.1 ; extra == 'all'
 Requires-Dist: httpx[http2] >=0.23.0 ; extra == 'all'
 Requires-Dist: aio-pika >=8.1.1 ; extra == 'all'
 Requires-Dist: cryptography ; extra == 'all'
-Requires-Dist: motor >=3.1.1 ; extra == 'all'
+Requires-Dist: motor >=2.1.0 ; extra == 'all'
 Requires-Dist: pyhttpx >=2.10.1 ; extra == 'all'
 Requires-Dist: asyncpg >=0.27.0 ; extra == 'all'
 Requires-Dist: XlsxWriter >=3.1.2 ; extra == 'all'
@@ -53,7 +53,7 @@ Requires-Dist: pillow >=9.4.0 ; extra == 'execl'
 Provides-Extra: httpx
 Requires-Dist: httpx[http2] >=0.23.0 ; extra == 'httpx'
 Provides-Extra: mongo
-Requires-Dist: motor >=3.1.1 ; extra == 'mongo'
+Requires-Dist: motor >=2.1.0 ; extra == 'mongo'
 Provides-Extra: pg
 Requires-Dist: asyncpg >=0.27.0 ; extra == 'pg'
 Provides-Extra: playwright
@@ -89,7 +89,7 @@ The quick way:
 
 ```shell
 # Install the latest aio-scrapy
-pip install git+https://github.com/conlin-huang/aio-scrapy
+pip install git+https://github.com/ConlinH/aio-scrapy
 
 # default
 pip install aio-scrapy

{aio_scrapy-2.1.0.dist-info → aio_scrapy-2.1.3.dist-info}/RECORD

@@ -1,9 +1,9 @@
-aioscrapy/VERSION,sha256=gkj3dyaHr_CxA7NomJCN64ISYzf5M-SaWsBIKQk8WP8,5
+aioscrapy/VERSION,sha256=BuGd6tadzBa8VzzJ8ktotpkH9M_Ur7nXA3LMa0YqmGI,5
 aioscrapy/__init__.py,sha256=esJeH66Mz9WV7XbotvZEjNn49jc589YZ_L2DKoD0JvA,858
 aioscrapy/__main__.py,sha256=rvTdJ0cQwbi29aucPj3jJRpccx5SBzvRcV7qvxvX2NQ,80
 aioscrapy/cmdline.py,sha256=1qhNg2Edl-Obmf2re2K4V8pJG7ubGfZZCzcHdKtdE_s,5159
 aioscrapy/crawler.py,sha256=6-ptivIjIGKdojOlZqXV0hV3x1Gont81tOC5u5JqIME,10330
-aioscrapy/exceptions.py,sha256=NjA2Rx1KZsjMgH7IOdNpxuRkh-RwylRCYvEhwgXKIb8,2027
+aioscrapy/exceptions.py,sha256=k1daw1hV_aqsaIKKibdyqcNPyVn5oUb07wmB2DRxfjs,2111
 aioscrapy/link.py,sha256=fXMqsHvYEzsuYi-sNDcElS7jV6Lusq0tjPkPUGOlyZw,1867
 aioscrapy/logformatter.py,sha256=y3etd28ACbpTbcGprJ_cQ086gxQY3k_QX_yxYFoF1AU,3028
 aioscrapy/process.py,sha256=uFkj2wzaBu0Vs3pGFKdJ4R-0Gn7hROX6EU-B5zddnyQ,1603
@@ -22,28 +22,28 @@ aioscrapy/commands/startproject.py,sha256=Rcc7JkN75Jp2t2aZIxBzPsWbLXChNAUSByDhcW
 aioscrapy/commands/version.py,sha256=yqqTMlZkkiQhtbU9w_IqUWLMOAjqYlv24friEkPRQYM,485
 aioscrapy/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 aioscrapy/core/engine.py,sha256=h02-K2lQqlCxvNIlURgPpnhHCbyiJRIWrFJt5Ys7vZY,9843
-aioscrapy/core/scheduler.py,sha256=sHrTfzSkqCVZTVw7zWAyv3vDd6iUwE9xbIsnePUVeZk,7408
-aioscrapy/core/scraper.py,sha256=M_bcizLUzWuECe7sIIZ_HJLNrPzL7dX2o-tN5nvFnCs,10304
-aioscrapy/core/downloader/__init__.py,sha256=22TC0z49BX3YvDUPl6DKMrOonECpY5tjaWJGGEV7RbU,9574
+aioscrapy/core/scheduler.py,sha256=czCx5oHknXuHadpISTfoEMSKXXrlwJTmLTUQtHdtaTc,7407
+aioscrapy/core/scraper.py,sha256=eS_qEX_Q9fXZnK8Ou1wDtBJhRKk9JoUSnbn4c04u1cA,10750
+aioscrapy/core/downloader/__init__.py,sha256=2EUQHGS6Q8fy9eoDT_kmA-eYZEUBV9OkmTppfG05AFA,10039
 aioscrapy/core/downloader/handlers/__init__.py,sha256=CriaX2Cp4jUqzDDGZDB7HiIEgUWt2pnYVho6HMV6sJ0,3198
-aioscrapy/core/downloader/handlers/aiohttp.py,sha256=dFVVeGgJ1WZcE1zI4fQOZIzmrkC6l1WZcYstHmB3qYg,3942
-aioscrapy/core/downloader/handlers/curl_cffi.py,sha256=6jBp9WrGU0PCWd3HfXLD6P3MkMIG_zmKNCKieORVPas,2250
-aioscrapy/core/downloader/handlers/httpx.py,sha256=-DfjYgfrjxMhaMpTgEOFlQRONasCXV0g6UgH3WmWcfs,3041
-aioscrapy/core/downloader/handlers/pyhttpx.py,sha256=fgD6Kz_gfB17KHbnkFtUHJDjfYR-c9P2LhuYX4hcva8,2228
-aioscrapy/core/downloader/handlers/requests.py,sha256=I49YnAxFGf-_a_YR-1AOG8vPLMmKiMtdmP4Xn-c0dPw,1996
-aioscrapy/core/downloader/handlers/playwright/__init__.py,sha256=lTI7Strp7SSZxM5IfZVgJGVwv9UfhTA4LdFpsfvzZ5k,4160
+aioscrapy/core/downloader/handlers/aiohttp.py,sha256=KY04ATlu2cTVF3Uxtvqpx1cGxOn2QVIGWoj7S3_UwF4,4266
+aioscrapy/core/downloader/handlers/curl_cffi.py,sha256=LpwWdHxctDOxVvhzYcgG1qGhPtw9DoqjUKEdrZlerAQ,2579
+aioscrapy/core/downloader/handlers/httpx.py,sha256=4XyLpiaXbO0AtPqctFJgYFX-5rJrKyf469YFpVJdcRY,3387
+aioscrapy/core/downloader/handlers/pyhttpx.py,sha256=phpWXtuvP-9tve3MDnWeto2Dmo25UTxgLYc12QEvMt4,2562
+aioscrapy/core/downloader/handlers/requests.py,sha256=UnV1WDyET8WMwyYYU0DcL_r420uz_0dK1ej5xAl2fwk,2317
+aioscrapy/core/downloader/handlers/playwright/__init__.py,sha256=PXS40Vv3KsV77QoyWCrWcHL6mItXxpTzwaCATB6RXiQ,4504
 aioscrapy/core/downloader/handlers/playwright/driverpool.py,sha256=qfIdGjORdn1MookO-ucIJ8NOeLrIQ0y0UJY_xuMzM_8,1374
 aioscrapy/core/downloader/handlers/playwright/webdriver.py,sha256=QFtAT--2Ea_Gg4x1EhMidyOwQjbqljUl4sKGB_hAA00,3530
 aioscrapy/db/__init__.py,sha256=ISBXM_-cCf5CgTLc3i_emLxV163-ZAbgttkQiRxokD0,2456
 aioscrapy/db/absmanager.py,sha256=6vlPcjDHOtZCHePiUYPe6ezRnM-TB4XLhmuw7APaWDk,1162
-aioscrapy/db/aiomongo.py,sha256=A9NjJy7_jI1J3hj8rw-o1PuXhXwFghHs4xSnsO5-ZfE,2745
+aioscrapy/db/aiomongo.py,sha256=t4JpRPBBisF7_rz02Kp6AejrphLvLWg5rF-yYLIe2MI,3071
 aioscrapy/db/aiomysql.py,sha256=-xCLfeH7RzvghY1jqREAb_Qnz9q_dVjxoHGfz7sCqbU,3799
 aioscrapy/db/aiopg.py,sha256=WG4s_2X0b8LQHbZpoIrwZeuGHNolKj-SvmvAZQlCk00,3213
 aioscrapy/db/aiorabbitmq.py,sha256=tNKl4Kx7KM7H_lOj8xfeA0uD8PuBTVzySApTEn5TyAE,5583
 aioscrapy/db/aioredis.py,sha256=UOoTRTQUvghnq29bVL8v1HvksMXYOzHaS8Btgbpn0bY,2966
-aioscrapy/dupefilters/__init__.py,sha256=17s6Hyr_lWDFPto6wLEvRfT2TbGU2RIssTDuChzrDNA,1498
+aioscrapy/dupefilters/__init__.py,sha256=Dx_CN-wBYiatLj3cXbK0f5d66CTjzzTeex6565L1EsA,1765
 aioscrapy/dupefilters/disk.py,sha256=EMgxeC2a6aYCGKgp4QOs5xwHp33LUsOZ8pliKBTFx1c,1551
-aioscrapy/dupefilters/redis.py,sha256=cUuM68dEM1_ki2eOzZ6pAvmLZlAP_tC4lx73Ufmg_Bs,4812
+aioscrapy/dupefilters/redis.py,sha256=YUUsnRQ326PjdM_FUWAWjgOWw92KwswAGnM-FmN8pv0,7559
 aioscrapy/http/__init__.py,sha256=yeQTT5W1iwr6dKznTS5d9vnx2hsB47i9roPM57wQp_0,597
 aioscrapy/http/headers.py,sha256=H-RJ6KqOsFFFAXORfvoyz3V-ud0I8TAj5Jt5fAACcLc,1573
 aioscrapy/http/request/__init__.py,sha256=PFoFU3ncTN-gj6Rx01rjVa_744Qfv3EH29mooW6JX9U,7121
@@ -59,7 +59,7 @@ aioscrapy/libs/downloader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJW
 aioscrapy/libs/downloader/defaultheaders.py,sha256=tg_ULA0Y-41bZKG607mowFJQGVfnZ45LdR044DsjA_A,563
 aioscrapy/libs/downloader/downloadtimeout.py,sha256=hNh3OEj7rC0ceQrv_yrhR5lb5AvfxJ6cspj3qsQWj4o,704
 aioscrapy/libs/downloader/ja3fingerprint.py,sha256=DgTw74GXC_Bp94eD_bwoG6A_DphUHTt7bH4glBNXyV8,1058
-aioscrapy/libs/downloader/retry.py,sha256=nNhAqudTBhYJES1CEuzo0a-ucmS2WKcj8bOvs3PwPjw,5306
+aioscrapy/libs/downloader/retry.py,sha256=0670bPz5lc4wUsWmYlhYdGZdeflsQdFhJbnwK1g0c84,4441
 aioscrapy/libs/downloader/stats.py,sha256=FlkS8Zm4j3SBjHb6caXwq08HvvZ37VKORGCAjlA2U38,1376
 aioscrapy/libs/downloader/useragent.py,sha256=E5x5dk9AxsSCGDDICJlTXwWXRkqAibWgesqG0VhAG8M,743
 aioscrapy/libs/extensions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -71,7 +71,7 @@ aioscrapy/libs/extensions/throttle.py,sha256=yos2D3XZgH40G52kltMKv5_GeAK4MqpRwTu
 aioscrapy/libs/pipelines/__init__.py,sha256=XW5Ur6bhvGLo-w-tdUeIB4jkFpZxqUU9mbajfAAztb0,5642
 aioscrapy/libs/pipelines/csv.py,sha256=-PEZOt-3ndF0ePO7EnqjEqeCYMJR9wHv3XcpSq6QswI,2454
 aioscrapy/libs/pipelines/execl.py,sha256=a8sfgQCHUc0MIja9cPP4TZ6ghfkxYZuAzLDIK4_nQuo,6284
-aioscrapy/libs/pipelines/mongo.py,sha256=jiPyC3C0mNb-zlS0ecEBgl883gBtBQBFEeBR8DOcmmI,2001
+aioscrapy/libs/pipelines/mongo.py,sha256=B3dhvspxc4lmPh2noqARYV-rFuHfivdSfZ7ZlPKnk7c,2323
 aioscrapy/libs/pipelines/mysql.py,sha256=gN4DnyuXTQvDvy9Gu-v8F6sT8l7GZEa45AD0d-Ckv8s,1022
 aioscrapy/libs/pipelines/pg.py,sha256=la-SflXtGFw4IQYlOn75Brw2IfmtOUcCh0gUSz_Jg-0,990
 aioscrapy/libs/spider/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -96,7 +96,7 @@ aioscrapy/scrapyd/__init__.py,sha256=Ey14RVLUP7typ2XqP8RWcUum2fuFyigdhuhBBiEheIo
 aioscrapy/scrapyd/runner.py,sha256=tewEkdNTMrBoredCbhmdrswSrF-GWsU3MLgC__ntnzQ,1777
 aioscrapy/settings/__init__.py,sha256=GuiVhezV8U2J1B-WJwSvxxeH_1YWYD_Wighr9owC4HU,15781
 aioscrapy/settings/default_settings.py,sha256=PrUOFYNnPIS8eCdqvRylMLBK-4tT-2MYuU6Nn8dQrx0,5639
-aioscrapy/spiders/__init__.py,sha256=KoM3RMOtvWqN5Qfh6AATTWFmj9DIVmhQqrU_EhE1EdI,4010
+aioscrapy/spiders/__init__.py,sha256=oM_FzqWa46P6cjzarOO1cfDTQD2AuIPgaWZrmdMcuTI,4085
 aioscrapy/templates/project/aioscrapy.cfg,sha256=_nRHP5wtPnZaBi7wCmjWv5BgUu5NYFJZhvCTRVSipyM,112
 aioscrapy/templates/project/module/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 aioscrapy/templates/project/module/middlewares.py.tmpl,sha256=0eEf2LC0vYcWPH82HNqieYSORyUuIo3Bgl5t-neRAJ4,3469
@@ -104,7 +104,7 @@ aioscrapy/templates/project/module/pipelines.py.tmpl,sha256=-MYA7MFAffH8FTG1VGAk
 aioscrapy/templates/project/module/settings.py.tmpl,sha256=AO2jmyokUhuhFqxMvsMihPgSY4ZrldsMs-BuOEVfvQY,1421
 aioscrapy/templates/project/module/spiders/__init__.py,sha256=Zg1uss1vaNjvld9s9Ccua50SxVZwpFTPwqpBHoCrWdU,164
 aioscrapy/templates/spiders/basic.tmpl,sha256=oO1vh7-TZLjvpwdrYC49TGe-A6Kulc8UIG4Sa0QhDfI,375
-aioscrapy/templates/spiders/single.tmpl,sha256=nqB7vP_L9icsxJWiSpsSSRZ1nTxHWowTjieRIcYaYIw,942
+aioscrapy/templates/spiders/single.tmpl,sha256=Ptmo_uFDGEffvpEMyxec7sxIyBbP05x0Grhn5u6lZbQ,1011
 aioscrapy/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 aioscrapy/utils/conf.py,sha256=NkSmKjOE7xVvrAWQu4ne3jOzNGucgZdWHPhGbpz8dPU,7208
 aioscrapy/utils/curl.py,sha256=I8eZWFNgvyUiJ2YS9-s3HltGNVG8XMMU0HPhlMxuxdA,3295
@@ -115,7 +115,7 @@ aioscrapy/utils/log.py,sha256=NRDivw8w21J77qEUeqqLdC4sgdIKaj2UAP6lDvWGotM,1697
 aioscrapy/utils/misc.py,sha256=9NOssEl7CP_c6R9skxyXwmz4bd-nZ_gkw6F0EybeLTQ,3509
 aioscrapy/utils/ossignal.py,sha256=jAsCIKu17KV45-9dZwEkFJHF31Y13KP_zxY0x49j1jo,896
 aioscrapy/utils/project.py,sha256=cT98HaR5JaNmm-Y1UzSuzXj6B5S7GlmMshUfMhjpjJY,2905
-aioscrapy/utils/python.py,sha256=fMV3Y2s7AnbQ7TChBoQodqPNzGEdVA3J89W-arwswd4,4577
+aioscrapy/utils/python.py,sha256=38oD-OSjeGb3XZFJn3bt74PwGbejnBfLWC5-lkUL0g8,4462
 aioscrapy/utils/reqser.py,sha256=qjrYut6KtvGpLLd-HDM0cncNzWCtXgpH6NyERu_5A9g,487
 aioscrapy/utils/request.py,sha256=bkFaLDeebAOp7pF-7vta9LKOB2OR2s7V9jVKfA-XlqA,2418
 aioscrapy/utils/response.py,sha256=UPR1wTTAYZkLGiiIs28kJLhlF7WPrgLuW31l9LZuYKM,1341
@@ -125,9 +125,9 @@ aioscrapy/utils/template.py,sha256=HR97X4lpv2WuqhuPfzTgaBN66fYnzHVpP6zQ5IoTwcI,8
 aioscrapy/utils/tools.py,sha256=WJowViZB8XEs2CFqjVvbqXK3H5Uvf4BgWgBD_RcHMaM,2319
 aioscrapy/utils/trackref.py,sha256=0nIpelT1d5WYxALl8SGA8vHNYsh-jS0Z2lwVEAhwx8E,2019
 aioscrapy/utils/url.py,sha256=8W8tAhU7lgfPOfzKp3ejJGEcLj1i_PnA_53Jv5LpxiY,5464
-aio_scrapy-2.1.0.dist-info/LICENSE,sha256=L-UoAEM3fQSjKA7FVWxQM7gwSCbeue6gZRAnpRS_UCo,1088
-aio_scrapy-2.1.0.dist-info/METADATA,sha256=ZZlawN0H5Ngxljj10IgIy9O7RU-y_RxJ9iTEkc_YNR8,6511
-aio_scrapy-2.1.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-aio_scrapy-2.1.0.dist-info/entry_points.txt,sha256=WWhoVHZvqhW8a5uFg97K0EP_GjG3uuCIFLkyqDICgaw,56
-aio_scrapy-2.1.0.dist-info/top_level.txt,sha256=8l08KyMt22wfX_5BmhrGH0PgwZdzZIPq-hBUa1GNir4,10
-aio_scrapy-2.1.0.dist-info/RECORD,,
+aio_scrapy-2.1.3.dist-info/LICENSE,sha256=L-UoAEM3fQSjKA7FVWxQM7gwSCbeue6gZRAnpRS_UCo,1088
+aio_scrapy-2.1.3.dist-info/METADATA,sha256=ldFQ8PbukunEAMxkUNbPQsA2Iuo6VfyYwPQbjc7KETk,6506
+aio_scrapy-2.1.3.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+aio_scrapy-2.1.3.dist-info/entry_points.txt,sha256=WWhoVHZvqhW8a5uFg97K0EP_GjG3uuCIFLkyqDICgaw,56
+aio_scrapy-2.1.3.dist-info/top_level.txt,sha256=8l08KyMt22wfX_5BmhrGH0PgwZdzZIPq-hBUa1GNir4,10
+aio_scrapy-2.1.3.dist-info/RECORD,,

aioscrapy/VERSION

@@ -1 +1 @@
-2.1.0
+2.1.3

aioscrapy/core/downloader/__init__.py

@@ -138,14 +138,16 @@ class Downloader(BaseDownloader):
 
     @classmethod
     async def from_crawler(cls, crawler) -> "Downloader":
+        df = crawler.settings.get('DUPEFILTER_CLASS') and await load_instance(crawler.settings['DUPEFILTER_CLASS'],
+                                                                              crawler=crawler)
+        crawler.spider.dupefilter = df  # 将指纹绑定到Spider 在解析成功的时候 调用DUPEFILTER_CLASS的success方法
         return cls(
             crawler,
             await call_helper(DownloadHandlerManager.for_crawler, crawler),
             await call_helper(DownloaderMiddlewareManager.from_crawler, crawler),
             proxy=crawler.settings.get("PROXY_HANDLER") and await load_instance(crawler.settings["PROXY_HANDLER"],
                                                                                 crawler=crawler),
-            dupefilter=crawler.settings.get('DUPEFILTER_CLASS') and await load_instance(
-                crawler.settings['DUPEFILTER_CLASS'], crawler=crawler)
+            dupefilter=df
         )
 
     async def fetch(self, request: Request) -> None:
@@ -204,11 +206,17 @@ class Downloader(BaseDownloader):
             slot.transferring.remove(request)
             slot.active.remove(request)
             self.active.remove(request)
+
             if isinstance(result, Response):
                 await self.signals.send_catch_log(signal=signals.response_downloaded,
                                                   response=result,
                                                   request=request,
                                                   spider=self.spider)
+            #  控制指纹是否移除
+            self.dupefilter and \
+                not request.dont_filter and \
+                await self.dupefilter.done(request, done_type="request_ok" if isinstance(result, Response) else "request_err")
+
             await self._call_engine(result, request)
             await self._process_queue(slot)
 

aioscrapy/core/downloader/handlers/aiohttp.py

@@ -4,9 +4,11 @@ import ssl
 from typing import Optional
 
 import aiohttp
+from aiohttp.client_exceptions import ClientError
 
 from aioscrapy import Request
 from aioscrapy.core.downloader.handlers import BaseDownloadHandler
+from aioscrapy.exceptions import DownloadError
 from aioscrapy.http import HtmlResponse
 from aioscrapy.settings import Settings
 from aioscrapy.utils.log import logger
@@ -32,6 +34,12 @@ class AioHttpDownloadHandler(BaseDownloadHandler):
         return self.session
 
     async def download_request(self, request: Request, _) -> HtmlResponse:
+        try:
+            return await self._download_request(request)
+        except ClientError as e:
+            raise DownloadError(e) from e
+
+    async def _download_request(self, request: Request) -> HtmlResponse:
         kwargs = {
             'verify_ssl': request.meta.get('verify_ssl', self.verify_ssl),
             'timeout': request.meta.get('download_timeout', 180),

aioscrapy/core/downloader/handlers/curl_cffi.py

@@ -1,9 +1,9 @@
-import ssl
-
+from curl_cffi.curl import CurlError
 from curl_cffi.requests import AsyncSession
 
 from aioscrapy import Request
 from aioscrapy.core.downloader.handlers import BaseDownloadHandler
+from aioscrapy.exceptions import DownloadError
 from aioscrapy.http import HtmlResponse
 from aioscrapy.settings import Settings
 from aioscrapy.utils.log import logger
@@ -21,6 +21,12 @@ class CurlCffiDownloadHandler(BaseDownloadHandler):
         return cls(settings)
 
     async def download_request(self, request: Request, _) -> HtmlResponse:
+        try:
+            return await self._download_request(request)
+        except CurlError as e:
+            raise DownloadError(e) from e
+
+    async def _download_request(self, request: Request) -> HtmlResponse:
         kwargs = {
             'timeout': self.settings.get('DOWNLOAD_TIMEOUT'),
             'cookies': dict(request.cookies),
@@ -52,8 +58,8 @@ class CurlCffiDownloadHandler(BaseDownloadHandler):
             str(response.url),
             status=response.status_code,
             headers=response.headers,
-            body=response.text,
-            cookies=dict(response.cookies),
+            body=response.content,
+            cookies={j.name: j.value or '' for j in response.cookies.jar},
             encoding=response.encoding
         )
 

aioscrapy/core/downloader/handlers/httpx.py

@@ -1,9 +1,11 @@
 import ssl
 
 import httpx
+from httpx import HTTPError as HttpxError
 
 from aioscrapy import Request
 from aioscrapy.core.downloader.handlers import BaseDownloadHandler
+from aioscrapy.exceptions import DownloadError
 from aioscrapy.http import HtmlResponse
 from aioscrapy.settings import Settings
 from aioscrapy.utils.log import logger
@@ -27,6 +29,12 @@ class HttpxDownloadHandler(BaseDownloadHandler):
         return cls(settings)
 
     async def download_request(self, request: Request, _) -> HtmlResponse:
+        try:
+            return await self._download_request(request)
+        except HttpxError as e:
+            raise DownloadError(e) from e
+
+    async def _download_request(self, request: Request) -> HtmlResponse:
         kwargs = {
             'timeout': self.settings.get('DOWNLOAD_TIMEOUT'),
             'cookies': dict(request.cookies),
@@ -68,7 +76,7 @@ class HttpxDownloadHandler(BaseDownloadHandler):
             status=response.status_code,
             headers=response.headers,
             body=content,
-            cookies=dict(response.cookies),
+            cookies={j.name: j.value or '' for j in response.cookies.jar},
             encoding=response.encoding
         )
 

aioscrapy/core/downloader/handlers/playwright/__init__.py

@@ -1,11 +1,13 @@
 from functools import wraps
 
+from playwright._impl._api_types import Error
 from playwright.async_api._generated import Response as EventResponse
 
-from aioscrapy import Request
+from aioscrapy import Request, Spider
 from aioscrapy.core.downloader.handlers import BaseDownloadHandler
 from aioscrapy.core.downloader.handlers.playwright.driverpool import WebDriverPool
 from aioscrapy.core.downloader.handlers.playwright.webdriver import PlaywrightDriver
+from aioscrapy.exceptions import DownloadError
 from aioscrapy.http import PlaywrightResponse
 from aioscrapy.settings import Settings
 from aioscrapy.utils.tools import call_helper
@@ -24,7 +26,13 @@ class PlaywrightHandler(BaseDownloadHandler):
     def from_settings(cls, settings: Settings):
         return cls(settings)
 
-    async def download_request(self, request: Request, spider) -> PlaywrightResponse:
+    async def download_request(self, request: Request, spider: Spider) -> PlaywrightResponse:
+        try:
+            return await self._download_request(request, spider)
+        except Error as e:
+            raise DownloadError(e) from e
+
+    async def _download_request(self, request: Request, spider) -> PlaywrightResponse:
         cookies = dict(request.cookies)
         timeout = request.meta.get('download_timeout', 30) * 1000
         user_agent = request.headers.get("User-Agent")

aioscrapy/core/downloader/handlers/pyhttpx.py

@@ -1,9 +1,11 @@
 import asyncio
 
 import pyhttpx
+from pyhttpx.exception import BaseExpetion as PyHttpxError
 
 from aioscrapy import Request
 from aioscrapy.core.downloader.handlers import BaseDownloadHandler
+from aioscrapy.exceptions import DownloadError
 from aioscrapy.http import HtmlResponse
 from aioscrapy.settings import Settings
 from aioscrapy.utils.log import logger
@@ -22,6 +24,12 @@ class PyhttpxDownloadHandler(BaseDownloadHandler):
         return cls(settings)
 
     async def download_request(self, request: Request, _) -> HtmlResponse:
+        try:
+            return await self._download_request(request)
+        except PyHttpxError as e:
+            raise DownloadError(e) from e
+
+    async def _download_request(self, request: Request) -> HtmlResponse:
         kwargs = {
             'timeout': self.settings.get('DOWNLOAD_TIMEOUT'),
             'cookies': dict(request.cookies),

aioscrapy/core/downloader/handlers/requests.py

@@ -1,9 +1,11 @@
 import asyncio
 
 import requests
+from requests.exceptions import RequestException as RequestsError
 
 from aioscrapy import Request
 from aioscrapy.core.downloader.handlers import BaseDownloadHandler
+from aioscrapy.exceptions import DownloadError
 from aioscrapy.http import HtmlResponse
 from aioscrapy.settings import Settings
 from aioscrapy.utils.log import logger
@@ -14,13 +16,18 @@ class RequestsDownloadHandler(BaseDownloadHandler):
     def __init__(self, settings):
         self.settings: Settings = settings
         self.verify_ssl: bool = self.settings.get("VERIFY_SSL", True)
-        self.loop = asyncio.get_running_loop()
 
     @classmethod
     def from_settings(cls, settings: Settings):
         return cls(settings)
 
     async def download_request(self, request: Request, _) -> HtmlResponse:
+        try:
+            return await self._download_request(request)
+        except RequestsError as e:
+            raise DownloadError(e) from e
+
+    async def _download_request(self, request: Request) -> HtmlResponse:
         kwargs = {
             'timeout': self.settings.get('DOWNLOAD_TIMEOUT'),
             'cookies': dict(request.cookies),
@@ -48,7 +55,7 @@ class RequestsDownloadHandler(BaseDownloadHandler):
             status=response.status_code,
             headers=response.headers,
             body=response.content,
-            cookies=dict(response.cookies),
+            cookies={k: v or '' for k, v in response.cookies.items()},
             encoding=response.encoding
         )
 

aioscrapy/core/scheduler.py

@@ -191,7 +191,6 @@ class Scheduler(BaseScheduler):
                 self.stats.inc_value(self.queue.inc_key, spider=self.spider)
             yield request
 
-
     async def has_pending_requests(self) -> bool:
         return await call_helper(self.queue.len) if self.cache_queue is None \
             else (await call_helper(self.queue.len) + await call_helper(self.cache_queue.len)) > 0

aioscrapy/core/scraper.py

@@ -110,9 +110,14 @@ class Scraper:
                     await self.handle_spider_error(e, request, result)
                 else:
                     await self.handle_spider_output(output, request, result)
-            except BaseException:
-                logger.exception('Scraper bug processing %(request)s' % {'request': request})
+            except BaseException as e:
+                await self.handle_spider_error(e, request, result)
             finally:
+                # 控制指纹是否移除
+                self.spider.dupefilter and \
+                    not request.dont_filter and \
+                    await self.spider.dupefilter.done(request, done_type="parse_ok" if getattr(request, "parse_ok", False) else "parse_err")
+
                 if isinstance(result, PlaywrightResponse):
                     await result.release()
 
@@ -162,16 +167,22 @@ class Scraper:
         if not result:
             return
 
+        parse_ok = True
         while True:
             try:
                 output = await result.__anext__()
             except StopAsyncIteration:
                 break
             except Exception as e:
+                parse_ok = False
                 await self.handle_spider_error(e, request, response)
             else:
                 await self._process_spidermw_output(output, request, response)
 
+        self.spider.dupefilter and \
+            not request.dont_filter and \
+            setattr(request, "parse_ok", parse_ok)
+
     async def _process_spidermw_output(self, output: Any, request: Request, response: Response) -> None:
         """Process each Request/Item (given in the output parameter) returned from the given spider"""
 

aioscrapy/db/aiomongo.py

@@ -1,7 +1,9 @@
 from motor.motor_asyncio import AsyncIOMotorClient
+from pymongo.errors import NetworkTimeout
 
 import aioscrapy
 from aioscrapy.db.absmanager import AbsDBPoolManager
+from loguru import logger
 
 
 class MongoExecutor:
@@ -9,10 +11,16 @@ class MongoExecutor:
         self.alias = alias
         self.pool_manager = pool_manager
 
-    async def insert(self, table_name, values, db_name=None):
+    async def insert(self, table_name, values, db_name=None, ordered=False, retry_times=3):
         client, db_name_default = self.pool_manager.get_pool(self.alias)
         db_name = db_name or db_name_default
-        return await client[f'{db_name}'][f'{table_name}'].insert_many(values)
+        for _ in range(retry_times):
+            try:
+                return await client[f'{db_name}'][f'{table_name}'].insert_many(values, ordered=ordered)
+            except NetworkTimeout:
+                logger.warning("mongo insert error by NetworkTimeout, retrying...")
+
+        raise NetworkTimeout
 
     def __getattr__(self, table_name: str):
         client, db_name_default = self.pool_manager.get_pool(self.alias)

aioscrapy/dupefilters/__init__.py

@@ -1,3 +1,4 @@
+from typing import Literal
 from abc import ABCMeta, abstractmethod
 
 from aioscrapy import Request, Spider
@@ -37,3 +38,10 @@ class DupeFilterBase(metaclass=ABCMeta):
             self.logdupes = False
 
         spider.crawler.stats.inc_value('dupefilter/filtered', spider=spider)
+
+    async def done(
+            self,
+            request: Request,
+            done_type: Literal["request_ok", "request_err", "parse_ok", "parse_err"]
+    ) -> None:
+        """ 根据done_type的状态 控制指纹的移除 """

aioscrapy/dupefilters/redis.py

@@ -1,3 +1,5 @@
+from typing import Literal
+
 from aioscrapy import Request
 from aioscrapy.db import db_manager
 from aioscrapy.dupefilters import DupeFilterBase
@@ -128,5 +130,68 @@ class RedisBloomDupeFilter(RedisRFPDupeFilter):
         return False
 
 
+class ExRedisBloomDupeFilter(RedisBloomDupeFilter):
+
+    def __init__(self, server, key, key_set, ttl, debug, bit, hash_number, keep_on_close, info):
+        super().__init__(server, key, debug, bit, hash_number, keep_on_close, info)
+        self.key_set = key_set
+        self.ttl = ttl
+
+    @classmethod
+    async def from_crawler(cls, crawler: "aioscrapy.crawler.Crawler"):
+        server = db_manager.redis.queue
+        dupefilter_key = crawler.settings.get("SCHEDULER_DUPEFILTER_KEY", '%(spider)s:bloomfilter')
+        keep_on_close = crawler.settings.getbool("KEEP_DUPEFILTER_DATA_ON_CLOSE", True)
+        key = dupefilter_key % {'spider': crawler.spider.name}
+        debug = crawler.settings.getbool('DUPEFILTER_DEBUG', False)
+        info = crawler.settings.getbool('DUPEFILTER_INFO', False)
+        bit = crawler.settings.getint('BLOOMFILTER_BIT', 30)
+        hash_number = crawler.settings.getint('BLOOMFILTER_HASH_NUMBER', 6)
+        ttl = crawler.settings.getint('DUPEFILTER_SET_KEY_TTL', 180)
+        return cls(server, key=key, key_set=key + "_set", ttl=ttl, debug=debug, bit=bit, hash_number=hash_number,
+                   keep_on_close=keep_on_close, info=info)
+
+    async def request_seen(self, request: Request) -> bool:
+        fp = await self.bf.exists(request.fingerprint)
+        if fp:
+            return True
+        async with self.server.pipeline() as pipe:
+            pipe.sadd(self.key_set, request.fingerprint)
+            pipe.expire(self.key_set, self.ttl)
+            ret, _ = await pipe.execute()
+        return ret == 0
+
+    async def done(
+            self,
+            request: Request,
+            done_type: Literal["request_ok", "request_err", "parse_ok", "parse_err"]
+    ):
+        if done_type == "request_ok" or done_type == "request_err":
+            await self.server.srem(self.key_set, request.fingerprint)
+        elif done_type == "parse_ok":
+            await self.bf.insert(request.fingerprint)
+
+    async def close(self, reason=''):
+        if not self.keep_on_close:
+            await self.clear()
+        await self.server.delete(self.key_set)
+
+
+class ExRedisRFPDupeFilter(RedisRFPDupeFilter):
+
+    async def done(
+            self,
+            request: Request,
+            done_type: Literal["request_ok", "request_err", "parse_ok", "parse_err"]
+    ):
+        # 当请求失败或解析失败的时候 从Redis的Set中移除指纹
+        if done_type == "request_err" or done_type == "parse_err":
+            await self.server.srem(self.key, request.fingerprint)
+
+
 RFPDupeFilter = RedisRFPDupeFilter
+ExRFPDupeFilter = ExRedisRFPDupeFilter
 BloomDupeFilter = RedisBloomDupeFilter
+ExBloomDupeFilter = ExRedisBloomDupeFilter
+BloomSetDupeFilter = ExRedisBloomDupeFilter
+

aioscrapy/exceptions.py

@@ -91,3 +91,8 @@ class ContractFail(AssertionError):
 
 class ProxyException(Exception):
     pass
+
+
+class DownloadError(Exception):
+    """下载页面时发生的错误"""
+    pass

aioscrapy/libs/downloader/retry.py

@@ -10,70 +10,21 @@ Failed pages are collected on the scraping process and rescheduled at the end,
 once the spider has finished crawling all regular (non failed) pages.
 """
 from typing import Optional, Union
-from aioscrapy.exceptions import ProxyException
+
+from anyio import EndOfStream
 
 try:
     from asyncio.exceptions import TimeoutError
 except:
     from concurrent.futures._base import TimeoutError
 
-NEED_RETRY_ERROR = (TimeoutError, ConnectionRefusedError, IOError, ProxyException)
-
-try:
-    from aiohttp.client_exceptions import ClientError
-
-    NEED_RETRY_ERROR += (ClientError,)
-except ImportError:
-    pass
-
-try:
-    from anyio import EndOfStream
-
-    NEED_RETRY_ERROR += (EndOfStream,)
-except ImportError:
-    pass
-
-try:
-    from httpx import HTTPError as HttpxError
-
-    NEED_RETRY_ERROR += (HttpxError,)
-except ImportError:
-    pass
-
-try:
-    from pyhttpx.exception import BaseExpetion as PyHttpxError
-
-    NEED_RETRY_ERROR += (PyHttpxError,)
-except ImportError:
-    pass
-
-try:
-    from requests.exceptions import RequestException as RequestsError
-
-    NEED_RETRY_ERROR += (RequestsError,)
-except ImportError:
-    pass
-
-try:
-    from playwright._impl._api_types import Error as PlaywrightError
-
-    NEED_RETRY_ERROR += (PlaywrightError,)
-except ImportError:
-    pass
-
-
-try:
-    from curl_cffi.curl import CurlError
-
-    NEED_RETRY_ERROR += (CurlError,)
-except ImportError:
-    pass
-
-from aioscrapy.exceptions import NotConfigured
+from aioscrapy.exceptions import ProxyException, DownloadError, NotConfigured
 from aioscrapy.http.request import Request
 from aioscrapy.spiders import Spider
-from aioscrapy.utils.python import global_object_name
 from aioscrapy.utils.log import logger as retry_logger
+from aioscrapy.utils.python import global_object_name
+
+NEED_RETRY_ERROR = (TimeoutError, ConnectionRefusedError, IOError, ProxyException, DownloadError, EndOfStream)
 
 
 def get_retry_request(

aioscrapy/libs/pipelines/mongo.py

@@ -9,6 +9,8 @@ class MongoPipeline(DBPipelineBase):
     def __init__(self, settings, db_type: str):
         super().__init__(settings, db_type)
         self.db_cache = {}
+        self.ordered_cache = {}
+        self.retry_times = settings.getint("MONGO_TIMEOUT_RETRY_TIMES", 3)
 
     @classmethod
     def from_settings(cls, settings):
@@ -17,17 +19,19 @@ class MongoPipeline(DBPipelineBase):
     def parse_item_to_cache(self, item: dict, save_info: dict):
         db_name = save_info.get('db_name')
         table_name = save_info.get('table_name')
+        ordered = save_info.get('ordered', False)
         assert table_name is not None, 'please set table_name'
         db_alias = save_info.get('db_alias', ['default'])
         if isinstance(db_alias, str):
             db_alias = [db_alias]
 
-        cache_key = ''.join(db_alias) + (db_name or '') + table_name
+        cache_key = ''.join(db_alias) + (db_name or '') + table_name + str(ordered)
 
         if self.table_cache.get(cache_key) is None:
             self.db_alias_cache[cache_key] = db_alias
             self.table_cache[cache_key] = table_name
             self.db_cache[cache_key] = db_name
+            self.ordered_cache[cache_key] = ordered
             self.item_cache[cache_key] = []
 
         self.item_cache[cache_key].append(item)
@@ -40,7 +44,8 @@ class MongoPipeline(DBPipelineBase):
                 try:
                     executor = db_manager.mongo.executor(alias)
                     result = await executor.insert(
-                        table_name, self.item_cache[cache_key], db_name=self.db_cache[cache_key]
+                        table_name, self.item_cache[cache_key], db_name=self.db_cache[cache_key],
+                        ordered=self.ordered_cache[cache_key], retry_times=self.retry_times
                     )
                     logger.info(
                         f'table:{alias}->{table_name} sum:{len(self.item_cache[cache_key])} ok:{len(result.inserted_ids)}'

aioscrapy/spiders/__init__.py

@@ -22,6 +22,7 @@ class Spider(object):
 
     name: Optional[str] = None
     proxy: Optional["aioscrapy.proxy.AbsProxy"] = None
+    dupefilter: Optional["aioscrapy.dupefilters.DupeFilterBase"] = None
     custom_settings: Optional[dict] = None
     stats: Optional[StatsCollector] = None
 
@@ -77,7 +78,7 @@ class Spider(object):
             yield Request(url)
 
     async def request_from_dict(self, d: dict):
-        """集成后重写改方法，将队列中的json根据情况构建成Request对象"""
+        """继承成后重写改方法，将队列中的json根据情况构建成Request对象"""
         pass
 
     async def _parse(self, response: Response, **kwargs):

aioscrapy/templates/spiders/single.tmpl

@@ -24,11 +24,12 @@ class $classname(Spider):
         pass
 
     async def parse(self, response):
-        item = {
-            'author': quote.xpath('span/small/text()').get(),
-            'text': quote.css('span.text::text').get(),
-        }
-        yield item
+        for quote in response.css('div.quote'):
+            item = {
+                'author': quote.xpath('span/small/text()').get(),
+                'text': quote.css('span.text::text').get(),
+            }
+            yield item
 
     async def process_item(self, item):
         logger.info(item)

aioscrapy/utils/python.py

@@ -1,16 +1,12 @@
 """
 This module contains essential stuff that should've come with Python itself ;)
 """
-import errno
 import gc
-import inspect
 import re
 import sys
-import warnings
 import weakref
-from functools import partial, wraps
+from functools import wraps
 
-from aioscrapy.exceptions import AioScrapyDeprecationWarning
 from aioscrapy.utils.decorators import deprecated
 
 
@@ -150,4 +146,3 @@ if hasattr(sys, "pypy_version_info"):
 else:
     def garbage_collect():
         gc.collect()
-