aio-scrapy 2.0.7__py3-none-any.whl → 2.0.9__py3-none-any.whl

{aio_scrapy-2.0.7.dist-info → aio_scrapy-2.0.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: aio-scrapy
-Version: 2.0.7
+Version: 2.0.9
 Summary: A high-level Web Crawling and Web Scraping framework based on Asyncio
 Home-page: https://github.com/conlin-huang/aio-scrapy.git
 Author: conlin
@@ -21,44 +21,44 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: aiohttp
 Requires-Dist: ujson
-Requires-Dist: w3lib (>=1.17.0)
-Requires-Dist: parsel (>=1.5.0)
-Requires-Dist: PyDispatcher (>=2.0.5)
-Requires-Dist: zope.interface (>=5.1.0)
-Requires-Dist: redis (>=4.3.1)
-Requires-Dist: aiomultiprocess (>=0.9.0)
-Requires-Dist: loguru (>=0.7.0)
+Requires-Dist: w3lib >=1.17.0
+Requires-Dist: parsel >=1.5.0
+Requires-Dist: PyDispatcher >=2.0.5
+Requires-Dist: zope.interface >=5.1.0
+Requires-Dist: redis >=4.3.1
+Requires-Dist: aiomultiprocess >=0.9.0
+Requires-Dist: loguru >=0.7.0
 Provides-Extra: aio-pika
-Requires-Dist: aio-pika (>=8.1.1) ; extra == 'aio-pika'
+Requires-Dist: aio-pika >=8.1.1 ; extra == 'aio-pika'
 Provides-Extra: aiomysql
-Requires-Dist: aiomysql (>=0.1.1) ; extra == 'aiomysql'
+Requires-Dist: aiomysql >=0.1.1 ; extra == 'aiomysql'
 Requires-Dist: cryptography ; extra == 'aiomysql'
 Provides-Extra: all
-Requires-Dist: aiomysql (>=0.1.1) ; extra == 'all'
-Requires-Dist: httpx[http2] (>=0.23.0) ; extra == 'all'
-Requires-Dist: aio-pika (>=8.1.1) ; extra == 'all'
+Requires-Dist: aiomysql >=0.1.1 ; extra == 'all'
+Requires-Dist: httpx[http2] >=0.23.0 ; extra == 'all'
+Requires-Dist: aio-pika >=8.1.1 ; extra == 'all'
 Requires-Dist: cryptography ; extra == 'all'
-Requires-Dist: motor (>=3.1.1) ; extra == 'all'
-Requires-Dist: pyhttpx (>=2.10.1) ; extra == 'all'
-Requires-Dist: asyncpg (>=0.27.0) ; extra == 'all'
-Requires-Dist: XlsxWriter (>=3.1.2) ; extra == 'all'
-Requires-Dist: pillow (>=9.4.0) ; extra == 'all'
-Requires-Dist: requests (>=2.28.2) ; extra == 'all'
+Requires-Dist: motor >=3.1.1 ; extra == 'all'
+Requires-Dist: pyhttpx >=2.10.1 ; extra == 'all'
+Requires-Dist: asyncpg >=0.27.0 ; extra == 'all'
+Requires-Dist: XlsxWriter >=3.1.2 ; extra == 'all'
+Requires-Dist: pillow >=9.4.0 ; extra == 'all'
+Requires-Dist: requests >=2.28.2 ; extra == 'all'
 Provides-Extra: execl
-Requires-Dist: XlsxWriter (>=3.1.2) ; extra == 'execl'
-Requires-Dist: pillow (>=9.4.0) ; extra == 'execl'
+Requires-Dist: XlsxWriter >=3.1.2 ; extra == 'execl'
+Requires-Dist: pillow >=9.4.0 ; extra == 'execl'
 Provides-Extra: httpx
-Requires-Dist: httpx[http2] (>=0.23.0) ; extra == 'httpx'
+Requires-Dist: httpx[http2] >=0.23.0 ; extra == 'httpx'
 Provides-Extra: mongo
-Requires-Dist: motor (>=3.1.1) ; extra == 'mongo'
+Requires-Dist: motor >=3.1.1 ; extra == 'mongo'
 Provides-Extra: pg
-Requires-Dist: asyncpg (>=0.27.0) ; extra == 'pg'
+Requires-Dist: asyncpg >=0.27.0 ; extra == 'pg'
 Provides-Extra: playwright
-Requires-Dist: playwright (>=1.31.1) ; extra == 'playwright'
+Requires-Dist: playwright >=1.31.1 ; extra == 'playwright'
 Provides-Extra: pyhttpx
-Requires-Dist: pyhttpx (>=2.10.4) ; extra == 'pyhttpx'
+Requires-Dist: pyhttpx >=2.10.4 ; extra == 'pyhttpx'
 Provides-Extra: requests
-Requires-Dist: requests (>=2.28.2) ; extra == 'requests'
+Requires-Dist: requests >=2.28.2 ; extra == 'requests'
 
 <!--
 ![aio-scrapy](./doc/images/aio-scrapy.png)

{aio_scrapy-2.0.7.dist-info → aio_scrapy-2.0.9.dist-info}/RECORD

@@ -1,4 +1,4 @@
-aioscrapy/VERSION,sha256=WbMmlVasurlWz2dT8BlCvGppjXF1UPN-qH_4eMFie-Q,5
+aioscrapy/VERSION,sha256=Qd60-DGk0CyAsfZTOK4DTPjIJ6aXFjyqch4-b7ff6f0,5
 aioscrapy/__init__.py,sha256=esJeH66Mz9WV7XbotvZEjNn49jc589YZ_L2DKoD0JvA,858
 aioscrapy/__main__.py,sha256=rvTdJ0cQwbi29aucPj3jJRpccx5SBzvRcV7qvxvX2NQ,80
 aioscrapy/cmdline.py,sha256=1qhNg2Edl-Obmf2re2K4V8pJG7ubGfZZCzcHdKtdE_s,5159
@@ -40,9 +40,9 @@ aioscrapy/db/aiomysql.py,sha256=-xCLfeH7RzvghY1jqREAb_Qnz9q_dVjxoHGfz7sCqbU,3799
 aioscrapy/db/aiopg.py,sha256=WG4s_2X0b8LQHbZpoIrwZeuGHNolKj-SvmvAZQlCk00,3213
 aioscrapy/db/aiorabbitmq.py,sha256=tNKl4Kx7KM7H_lOj8xfeA0uD8PuBTVzySApTEn5TyAE,5583
 aioscrapy/db/aioredis.py,sha256=UOoTRTQUvghnq29bVL8v1HvksMXYOzHaS8Btgbpn0bY,2966
-aioscrapy/dupefilters/__init__.py,sha256=bxOMWqqXuOZS8r5OXqK9JSB8Sf-LNDqfKtllTCi-Gls,725
-aioscrapy/dupefilters/disk.py,sha256=z4PcayzR1zMxXkCR1pJAJvzcvzsfpHM64ibYQltnOMo,2178
-aioscrapy/dupefilters/redis.py,sha256=rYmKnjoSZlOmAABkdrFa9HUSy3F6VBre_7z6u8YpMCs,5160
+aioscrapy/dupefilters/__init__.py,sha256=17s6Hyr_lWDFPto6wLEvRfT2TbGU2RIssTDuChzrDNA,1498
+aioscrapy/dupefilters/disk.py,sha256=EMgxeC2a6aYCGKgp4QOs5xwHp33LUsOZ8pliKBTFx1c,1551
+aioscrapy/dupefilters/redis.py,sha256=1bDqB1avfDRR9b9doDXAyxwL1Fa8LEpMYlV7YoBXBvw,4723
 aioscrapy/http/__init__.py,sha256=yeQTT5W1iwr6dKznTS5d9vnx2hsB47i9roPM57wQp_0,597
 aioscrapy/http/headers.py,sha256=H-RJ6KqOsFFFAXORfvoyz3V-ud0I8TAj5Jt5fAACcLc,1573
 aioscrapy/http/request/__init__.py,sha256=PFoFU3ncTN-gj6Rx01rjVa_744Qfv3EH29mooW6JX9U,7121
@@ -58,7 +58,7 @@ aioscrapy/libs/downloader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJW
 aioscrapy/libs/downloader/defaultheaders.py,sha256=tg_ULA0Y-41bZKG607mowFJQGVfnZ45LdR044DsjA_A,563
 aioscrapy/libs/downloader/downloadtimeout.py,sha256=hNh3OEj7rC0ceQrv_yrhR5lb5AvfxJ6cspj3qsQWj4o,704
 aioscrapy/libs/downloader/ja3fingerprint.py,sha256=DgTw74GXC_Bp94eD_bwoG6A_DphUHTt7bH4glBNXyV8,1058
-aioscrapy/libs/downloader/retry.py,sha256=onItmt4s_diH3v1iMJiTitMn12DflwerSK2fIgJyBsY,5082
+aioscrapy/libs/downloader/retry.py,sha256=eaMig7JpSyr6QQBD6FNYpcttuGK811Dm4tJGTUIi3q8,5191
 aioscrapy/libs/downloader/stats.py,sha256=FlkS8Zm4j3SBjHb6caXwq08HvvZ37VKORGCAjlA2U38,1376
 aioscrapy/libs/downloader/useragent.py,sha256=E5x5dk9AxsSCGDDICJlTXwWXRkqAibWgesqG0VhAG8M,743
 aioscrapy/libs/extensions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -87,15 +87,15 @@ aioscrapy/middleware/itempipeline.py,sha256=_Htfrs3vzIUfajTOzQLdGaX_4xTFtSSoFzyh
 aioscrapy/middleware/spider.py,sha256=QvV5dchOlskBn1sXKd5dj6s9zSZmlT6LibydCjfmYjU,6361
 aioscrapy/proxy/__init__.py,sha256=Cwua97Z-ezxtDSlud7mCOAV-iExY7RX_8O1oP5PS__k,1807
 aioscrapy/proxy/redis.py,sha256=LFfnnkihf6Wq1-HeRzPLVEiy5e5wxJbMY7htU-C_Pd8,2711
-aioscrapy/queue/__init__.py,sha256=-5h-Mk4Lxfyv5FIPuoMWYkl_TQUOc5Ftn83zXW9ib9Y,2025
-aioscrapy/queue/memory.py,sha256=7tOj6Ut4pKxd0YhfUUrxTjfQIc06fzyL8YfHFC1Yz1k,3128
-aioscrapy/queue/rabbitmq.py,sha256=z_Zjqsgi_nmpouX1BAgyAcvpQKKOvzoUmnBEd9YLVy8,2510
-aioscrapy/queue/redis.py,sha256=9TbAjDe8aAMjapwgKiUaDSdEH5xcVTK1OSKL-SlGfhg,4770
+aioscrapy/queue/__init__.py,sha256=MKHNOgcZRAjFHAxoKLujvsBCkB_Ne1-gz5DqNbdTYNA,2037
+aioscrapy/queue/memory.py,sha256=_Dvkd-HXwdR8B-wsEPlHNWbAgaMH9M_UGZhF21LbnHA,3140
+aioscrapy/queue/rabbitmq.py,sha256=rE1GCoIGxoaV4KAi_9Umt623A00FaHHIFV5ZH_nypzY,2516
+aioscrapy/queue/redis.py,sha256=KU31ZNciLI9xxZDxsDhtOPLtmkxZQlRPOx_1z8afdwY,4788
 aioscrapy/scrapyd/__init__.py,sha256=Ey14RVLUP7typ2XqP8RWcUum2fuFyigdhuhBBiEheIo,68
 aioscrapy/scrapyd/runner.py,sha256=tewEkdNTMrBoredCbhmdrswSrF-GWsU3MLgC__ntnzQ,1777
 aioscrapy/settings/__init__.py,sha256=GuiVhezV8U2J1B-WJwSvxxeH_1YWYD_Wighr9owC4HU,15781
 aioscrapy/settings/default_settings.py,sha256=ffGA1SKEBQtmRC7UaFcNBlZrVW9PjUwukDiARqVfTXs,5432
-aioscrapy/spiders/__init__.py,sha256=t7CVEHOCWRQ06CfUXlWFI_yQg9ojES4f_N265mnDS1A,3928
+aioscrapy/spiders/__init__.py,sha256=vAfod_sqXs85E-QRNji_Qhf7SyWx1kXgJD8n3AhAj1g,3934
 aioscrapy/templates/project/aioscrapy.cfg,sha256=_nRHP5wtPnZaBi7wCmjWv5BgUu5NYFJZhvCTRVSipyM,112
 aioscrapy/templates/project/module/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 aioscrapy/templates/project/module/middlewares.py.tmpl,sha256=0eEf2LC0vYcWPH82HNqieYSORyUuIo3Bgl5t-neRAJ4,3469
@@ -110,13 +110,13 @@ aioscrapy/utils/curl.py,sha256=I8eZWFNgvyUiJ2YS9-s3HltGNVG8XMMU0HPhlMxuxdA,3295
 aioscrapy/utils/decorators.py,sha256=gMQArNxF9QQc1bENA0IqDchAjqmfWvHGKOyUdjXdg6A,794
 aioscrapy/utils/deprecate.py,sha256=STy55Q8kZI8q8CQUfxK4QQVu1Rs4En3rlhzWc7p7T00,5467
 aioscrapy/utils/httpobj.py,sha256=ytec7IZzsQY_GwR___051hdbOWs1ZM6S57HwcNiu2es,708
-aioscrapy/utils/log.py,sha256=U4BqJQ3p-s_lYbqgln_9FHeUegKF7T1MTrN1KTtn92o,1592
+aioscrapy/utils/log.py,sha256=NRDivw8w21J77qEUeqqLdC4sgdIKaj2UAP6lDvWGotM,1697
 aioscrapy/utils/misc.py,sha256=9NOssEl7CP_c6R9skxyXwmz4bd-nZ_gkw6F0EybeLTQ,3509
 aioscrapy/utils/ossignal.py,sha256=jAsCIKu17KV45-9dZwEkFJHF31Y13KP_zxY0x49j1jo,896
 aioscrapy/utils/project.py,sha256=cT98HaR5JaNmm-Y1UzSuzXj6B5S7GlmMshUfMhjpjJY,2905
 aioscrapy/utils/python.py,sha256=fMV3Y2s7AnbQ7TChBoQodqPNzGEdVA3J89W-arwswd4,4577
-aioscrapy/utils/reqser.py,sha256=Xg-0asXQsA5eHEDLx82tk-0MX8CyukrBMVf0ZvO3NgM,475
-aioscrapy/utils/request.py,sha256=1SihbyMHEiXm-q473-WoyjEcukGLxJ_ZftDbd-SvFmQ,2406
+aioscrapy/utils/reqser.py,sha256=qjrYut6KtvGpLLd-HDM0cncNzWCtXgpH6NyERu_5A9g,487
+aioscrapy/utils/request.py,sha256=bkFaLDeebAOp7pF-7vta9LKOB2OR2s7V9jVKfA-XlqA,2418
 aioscrapy/utils/response.py,sha256=UPR1wTTAYZkLGiiIs28kJLhlF7WPrgLuW31l9LZuYKM,1341
 aioscrapy/utils/signal.py,sha256=bkqRgGMqQ82dly_D4tDe_0pHBbc9QUxBJqSsH9RSQf0,2282
 aioscrapy/utils/spider.py,sha256=Usq3UlCaDUvXGp0ojFt39UPKFrR2rbInlJc_q0Xk7Qc,610
@@ -124,9 +124,9 @@ aioscrapy/utils/template.py,sha256=HR97X4lpv2WuqhuPfzTgaBN66fYnzHVpP6zQ5IoTwcI,8
 aioscrapy/utils/tools.py,sha256=WJowViZB8XEs2CFqjVvbqXK3H5Uvf4BgWgBD_RcHMaM,2319
 aioscrapy/utils/trackref.py,sha256=0nIpelT1d5WYxALl8SGA8vHNYsh-jS0Z2lwVEAhwx8E,2019
 aioscrapy/utils/url.py,sha256=8W8tAhU7lgfPOfzKp3ejJGEcLj1i_PnA_53Jv5LpxiY,5464
-aio_scrapy-2.0.7.dist-info/LICENSE,sha256=L-UoAEM3fQSjKA7FVWxQM7gwSCbeue6gZRAnpRS_UCo,1088
-aio_scrapy-2.0.7.dist-info/METADATA,sha256=K3ualF3s3obq6vgvz6Gp5o6vysBqhIuRcEiQR7AsKbw,6436
-aio_scrapy-2.0.7.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-aio_scrapy-2.0.7.dist-info/entry_points.txt,sha256=WWhoVHZvqhW8a5uFg97K0EP_GjG3uuCIFLkyqDICgaw,56
-aio_scrapy-2.0.7.dist-info/top_level.txt,sha256=8l08KyMt22wfX_5BmhrGH0PgwZdzZIPq-hBUa1GNir4,10
-aio_scrapy-2.0.7.dist-info/RECORD,,
+aio_scrapy-2.0.9.dist-info/LICENSE,sha256=L-UoAEM3fQSjKA7FVWxQM7gwSCbeue6gZRAnpRS_UCo,1088
+aio_scrapy-2.0.9.dist-info/METADATA,sha256=AzYfL1fSr0PvPyJDV7QgQ2rm3DzWaIN5WwRhtNsT8Ik,6384
+aio_scrapy-2.0.9.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+aio_scrapy-2.0.9.dist-info/entry_points.txt,sha256=WWhoVHZvqhW8a5uFg97K0EP_GjG3uuCIFLkyqDICgaw,56
+aio_scrapy-2.0.9.dist-info/top_level.txt,sha256=8l08KyMt22wfX_5BmhrGH0PgwZdzZIPq-hBUa1GNir4,10
+aio_scrapy-2.0.9.dist-info/RECORD,,

{aio_scrapy-2.0.7.dist-info → aio_scrapy-2.0.9.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.40.0)
+Generator: bdist_wheel (0.42.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

aioscrapy/VERSION

@@ -1 +1 @@
-2.0.7
+2.0.9

aioscrapy/dupefilters/__init__.py

@@ -1,6 +1,7 @@
 from abc import ABCMeta, abstractmethod
 
 from aioscrapy import Request, Spider
+from aioscrapy.utils.log import logger
 
 
 class DupeFilterBase(metaclass=ABCMeta):
@@ -19,6 +20,20 @@ class DupeFilterBase(metaclass=ABCMeta):
     async def close(self, reason: str = '') -> None:
         """ Delete data on close """
 
-    @abstractmethod
-    def log(self, request: Request, spider: Spider) -> None:
-        """ Logs given request """
+    def log(self, request: Request, spider: Spider):
+        if self.info:
+            logger.info("Filtered duplicate request: %(request)s" % {
+                'request': request.meta.get('dupefilter_msg') or request
+            })
+        elif self.debug:
+            logger.debug("Filtered duplicate request: %(request)s" % {
+                'request': request.meta.get('dupefilter_msg') or request
+            })
+        elif self.logdupes:
+            msg = ("Filtered duplicate request: %(request)s"
+                   " - no more duplicates will be shown"
+                   " (see DUPEFILTER_DEBUG to show all duplicates)")
+            logger.debug(msg % {'request': request.meta.get('dupefilter_msg') or request})
+            self.logdupes = False
+
+        spider.crawler.stats.inc_value('dupefilter/filtered', spider=spider)

aioscrapy/dupefilters/disk.py

@@ -1,20 +1,19 @@
 import os
 from typing import Optional, Set
 
-from aioscrapy import Request, Spider
+from aioscrapy import Request
 from aioscrapy.dupefilters import DupeFilterBase
-from aioscrapy.utils.log import logger
-from aioscrapy.utils.request import referer_str
 
 
 class DiskRFPDupeFilter(DupeFilterBase):
     """Request Fingerprint duplicates filter built with Disk storage"""
 
-    def __init__(self, path: Optional[str] = None, debug: bool = False):
+    def __init__(self, path: Optional[str] = None, debug: bool = False, info: bool = False):
         self.file: Optional["File object"] = None
         self.debug = debug
         self.fingerprints: Set = set()
         self.logdupes: bool = True
+        self.info: bool = info
         if path:
             self.file = open(os.path.join(path, 'requests.seen'), 'a+')
             self.file.seek(0)
@@ -23,10 +22,11 @@ class DiskRFPDupeFilter(DupeFilterBase):
     @classmethod
     def from_crawler(cls, crawler: "aioscrapy.crawler.Crawler"):
         debug = crawler.settings.getbool('DUPEFILTER_DEBUG')
+        info = crawler.settings.getbool('DUPEFILTER_INFO')
         path = crawler.settings.get('JOBDIR', './job_dir')
         if path and not os.path.exists(path):
             os.makedirs(path)
-        return cls(path, debug)
+        return cls(path, debug, info)
 
     async def request_seen(self, request: Request) -> bool:
         if request.fingerprint in self.fingerprints:
@@ -40,19 +40,5 @@ class DiskRFPDupeFilter(DupeFilterBase):
         if self.file:
             self.file.close()
 
-    def log(self, request: Request, spider: Spider):
-        if self.debug:
-            logger.debug("Filtered duplicate request: %(request)s (referer: %(referer)s)" % {
-                'request': request, 'referer': referer_str(request)
-            })
-        elif self.logdupes:
-            msg = ("Filtered duplicate request: %(request)s"
-                   " - no more duplicates will be shown"
-                   " (see DUPEFILTER_DEBUG to show all duplicates)")
-            logger.debug(msg % {'request': request})
-            self.logdupes = False
-
-        spider.crawler.stats.inc_value('dupefilter/filtered', spider=spider)
-
 
 RFPDupeFilter = DiskRFPDupeFilter

aioscrapy/dupefilters/redis.py

@@ -2,8 +2,6 @@ from aioscrapy import Request
 from aioscrapy.db import db_manager
 from aioscrapy.dupefilters import DupeFilterBase
 
-from aioscrapy.utils.log import logger
-
 
 class RedisRFPDupeFilter(DupeFilterBase):
     """Request Fingerprint duplicates filter built with Set of Redis"""
@@ -13,13 +11,15 @@ class RedisRFPDupeFilter(DupeFilterBase):
             server: "redis.asyncio.Redis",
             key: str,
             debug: bool = False,
-            keep_on_close: bool = True
+            keep_on_close: bool = True,
+            info: bool = False,
     ):
         self.server = server
         self.key = key
         self.debug = debug
         self.keep_on_close = keep_on_close
         self.logdupes: bool = True
+        self.info: bool = info
 
     @classmethod
     def from_crawler(cls, crawler: "aioscrapy.crawler.Crawler"):
@@ -28,7 +28,8 @@ class RedisRFPDupeFilter(DupeFilterBase):
         keep_on_close = crawler.settings.getbool("KEEP_DUPEFILTER_DATA_ON_CLOSE", True)
         key = dupefilter_key % {'spider': crawler.spider.name}
         debug = crawler.settings.getbool('DUPEFILTER_DEBUG', False)
-        instance = cls(server, key=key, debug=debug, keep_on_close=keep_on_close)
+        info = crawler.settings.getbool('DUPEFILTER_DEBUG', False)
+        instance = cls(server, key=key, debug=debug, keep_on_close=keep_on_close, info=info)
         return instance
 
     async def request_seen(self, request: Request):
@@ -41,17 +42,6 @@ class RedisRFPDupeFilter(DupeFilterBase):
     async def clear(self):
         await self.server.delete(self.key)
 
-    def log(self, request, spider):
-        if self.debug:
-            logger.debug("Filtered duplicate request: %(request)s" % {'request': request})
-        elif self.logdupes:
-            msg = ("Filtered duplicate request %(request)s"
-                   " - no more duplicates will be shown"
-                   " (see DUPEFILTER_DEBUG to show all duplicates)")
-            logger.debug(msg % {'request': request})
-            self.logdupes = False
-        spider.crawler.stats.inc_value('dupefilter/filtered', spider=spider)
-
 
 class HashMap(object):
     def __init__(self, m, seed):

aioscrapy/libs/downloader/retry.py

@@ -26,6 +26,13 @@ try:
 except ImportError:
     pass
 
+try:
+    from anyio import EndOfStream
+
+    NEED_RETRY_ERROR += (EndOfStream,)
+except ImportError:
+    pass
+
 try:
     from httpx import HTTPError as HttpxError
 

aioscrapy/queue/__init__.py

@@ -37,10 +37,10 @@ class AbsQueue(metaclass=ABCMeta):
         obj = request.to_dict(spider=self.spider)
         return self.serializer.dumps(obj)
 
-    def _decode_request(self, encoded_request: Any) -> aioscrapy.Request:
+    async def _decode_request(self, encoded_request: Any) -> aioscrapy.Request:
         """Decode an request previously encoded"""
         obj = self.serializer.loads(encoded_request)
-        return request_from_dict(obj, spider=self.spider)
+        return await request_from_dict(obj, spider=self.spider)
 
     def __len__(self) -> None:
         """Return the length of the queue"""

aioscrapy/queue/memory.py

@@ -58,7 +58,7 @@ class MemoryQueueBase(AbsQueue):
                 data = self.container.get_nowait()
             except QueueEmpty:
                 break
-            yield self._decode_request(data)
+            yield await self._decode_request(data)
 
     async def clear(self, timeout: int = 0) -> None:
         self.container = self.get_queue(self.max_size)
@@ -93,7 +93,7 @@ class MemoryPriorityQueue(MemoryFifoQueue):
                 score, data = self.container.get_nowait()
             except QueueEmpty:
                 break
-            yield self._decode_request(data)
+            yield await self._decode_request(data)
 
 
 SpiderQueue = MemoryFifoQueue

aioscrapy/queue/rabbitmq.py

@@ -57,7 +57,7 @@ class RabbitMqPriorityQueue(AbsQueue):
     async def pop(self, count: int = 1) -> Optional[aioscrapy.Request]:
         result = await self.container.get_message(self.key)
         if result:
-            yield self._decode_request(result)
+            yield await self._decode_request(result)
 
     async def clear(self) -> None:
         await self.container.clean_message_queue(self.key)

aioscrapy/queue/redis.py

@@ -67,7 +67,7 @@ class RedisFifoQueue(RedisQueueBase):
             results = await pipe.execute()
         for result in results:
             if result:
-                yield self._decode_request(result)
+                yield await self._decode_request(result)
 
 
 class RedisPriorityQueue(RedisQueueBase):
@@ -97,7 +97,7 @@ class RedisPriorityQueue(RedisQueueBase):
                 .execute()
             )
         for result in results:
-            yield self._decode_request(result)
+            yield await self._decode_request(result)
 
 
 class RedisLifoQueue(RedisQueueBase):
@@ -124,7 +124,7 @@ class RedisLifoQueue(RedisQueueBase):
             results = await pipe.execute()
         for result in results:
             if result:
-                yield self._decode_request(result)
+                yield await self._decode_request(result)
 
 
 SpiderQueue = RedisFifoQueue

aioscrapy/spiders/__init__.py

@@ -76,7 +76,7 @@ class Spider(object):
         for url in self.start_urls:
             yield Request(url)
 
-    def request_from_dict(self, d: dict):
+    async def request_from_dict(self, d: dict):
         """集成后重写改方法，将队列中的json根据情况构建成Request对象"""
         pass
 

aioscrapy/utils/log.py

@@ -7,7 +7,9 @@ from loguru import logger as _logger
 
 from aioscrapy.settings import Settings
 
-_logger.remove(0)
+for _handler in _logger._core.handlers.values():
+    if _handler._name == '<stderr>':
+        _logger.remove(_handler._id)
 
 
 def configure_logging(spider: Type["Spider"], settings: Settings):

aioscrapy/utils/reqser.py

@@ -11,5 +11,5 @@ def request_to_dict(request: "aioscrapy.Request", spider: Optional["aioscrapy.Sp
     return request.to_dict(spider=spider)
 
 
-def request_from_dict(d: dict, spider: Optional["aioscrapy.Spider"] = None) -> "aioscrapy.Request":
-    return _from_dict(d, spider=spider)
+async def request_from_dict(d: dict, spider: Optional["aioscrapy.Spider"] = None) -> "aioscrapy.Request":
+    return await _from_dict(d, spider=spider)

aioscrapy/utils/request.py

@@ -39,13 +39,13 @@ def referer_str(request: Request) -> Optional[str]:
     return to_unicode(referrer, errors='replace')
 
 
-def request_from_dict(d: dict, *, spider: Optional[Spider] = None) -> Request:
+async def request_from_dict(d: dict, *, spider: Optional[Spider] = None) -> Request:
     """Create a :class:`~scrapy.Request` object from a dict.
 
     If a spider is given, it will try to resolve the callbacks looking at the
     spider for methods with the same name.
     """
-    d = spider.request_from_dict(d) or d
+    d = await spider.request_from_dict(d) or d
     if isinstance(d, Request):
         return d