aio-scrapy 2.1.2__tar.gz → 2.1.3__tar.gz

{aio-scrapy-2.1.2/aio_scrapy.egg-info → aio-scrapy-2.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: aio-scrapy
-Version: 2.1.2
+Version: 2.1.3
 Summary: A high-level Web Crawling and Web Scraping framework based on Asyncio
 Home-page: https://github.com/conlin-huang/aio-scrapy.git
 Author: conlin

{aio-scrapy-2.1.2 → aio-scrapy-2.1.3/aio_scrapy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: aio-scrapy
-Version: 2.1.2
+Version: 2.1.3
 Summary: A high-level Web Crawling and Web Scraping framework based on Asyncio
 Home-page: https://github.com/conlin-huang/aio-scrapy.git
 Author: conlin

aio-scrapy-2.1.3/aioscrapy/VERSION

@@ -0,0 +1 @@
+2.1.3

{aio-scrapy-2.1.2 → aio-scrapy-2.1.3}/aioscrapy/core/downloader/__init__.py

@@ -138,13 +138,15 @@ class Downloader(BaseDownloader):
 
     @classmethod
     async def from_crawler(cls, crawler) -> "Downloader":
-        df = crawler.settings.get('DUPEFILTER_CLASS') and await load_instance(crawler.settings['DUPEFILTER_CLASS'], crawler=crawler)
+        df = crawler.settings.get('DUPEFILTER_CLASS') and await load_instance(crawler.settings['DUPEFILTER_CLASS'],
+                                                                              crawler=crawler)
         crawler.spider.dupefilter = df  # 将指纹绑定到Spider 在解析成功的时候 调用DUPEFILTER_CLASS的success方法
         return cls(
             crawler,
             await call_helper(DownloadHandlerManager.for_crawler, crawler),
             await call_helper(DownloaderMiddlewareManager.from_crawler, crawler),
-            proxy=crawler.settings.get("PROXY_HANDLER") and await load_instance(crawler.settings["PROXY_HANDLER"], crawler=crawler),
+            proxy=crawler.settings.get("PROXY_HANDLER") and await load_instance(crawler.settings["PROXY_HANDLER"],
+                                                                                crawler=crawler),
             dupefilter=df
         )
 
@@ -204,12 +206,17 @@ class Downloader(BaseDownloader):
             slot.transferring.remove(request)
             slot.active.remove(request)
             self.active.remove(request)
-            self.dupefilter and not request.dont_filter and await self.dupefilter.done(request, done_type="request_done")
+
             if isinstance(result, Response):
                 await self.signals.send_catch_log(signal=signals.response_downloaded,
                                                   response=result,
                                                   request=request,
                                                   spider=self.spider)
+            #  控制指纹是否移除
+            self.dupefilter and \
+                not request.dont_filter and \
+                await self.dupefilter.done(request, done_type="request_ok" if isinstance(result, Response) else "request_err")
+
             await self._call_engine(result, request)
             await self._process_queue(slot)
 

{aio-scrapy-2.1.2 → aio-scrapy-2.1.3}/aioscrapy/core/downloader/handlers/aiohttp.py

@@ -37,7 +37,7 @@ class AioHttpDownloadHandler(BaseDownloadHandler):
         try:
             return await self._download_request(request)
         except ClientError as e:
-            raise DownloadError from e
+            raise DownloadError(e) from e
 
     async def _download_request(self, request: Request) -> HtmlResponse:
         kwargs = {

{aio-scrapy-2.1.2 → aio-scrapy-2.1.3}/aioscrapy/core/downloader/handlers/curl_cffi.py

@@ -24,7 +24,7 @@ class CurlCffiDownloadHandler(BaseDownloadHandler):
         try:
             return await self._download_request(request)
         except CurlError as e:
-            raise DownloadError from e
+            raise DownloadError(e) from e
 
     async def _download_request(self, request: Request) -> HtmlResponse:
         kwargs = {

{aio-scrapy-2.1.2 → aio-scrapy-2.1.3}/aioscrapy/core/downloader/handlers/httpx.py

@@ -32,7 +32,7 @@ class HttpxDownloadHandler(BaseDownloadHandler):
         try:
             return await self._download_request(request)
         except HttpxError as e:
-            raise DownloadError from e
+            raise DownloadError(e) from e
 
     async def _download_request(self, request: Request) -> HtmlResponse:
         kwargs = {

{aio-scrapy-2.1.2 → aio-scrapy-2.1.3}/aioscrapy/core/downloader/handlers/playwright/__init__.py

@@ -30,7 +30,7 @@ class PlaywrightHandler(BaseDownloadHandler):
         try:
             return await self._download_request(request, spider)
         except Error as e:
-            raise DownloadError from e
+            raise DownloadError(e) from e
 
     async def _download_request(self, request: Request, spider) -> PlaywrightResponse:
         cookies = dict(request.cookies)

{aio-scrapy-2.1.2 → aio-scrapy-2.1.3}/aioscrapy/core/downloader/handlers/pyhttpx.py

@@ -27,7 +27,7 @@ class PyhttpxDownloadHandler(BaseDownloadHandler):
         try:
             return await self._download_request(request)
         except PyHttpxError as e:
-            raise DownloadError from e
+            raise DownloadError(e) from e
 
     async def _download_request(self, request: Request) -> HtmlResponse:
         kwargs = {

{aio-scrapy-2.1.2 → aio-scrapy-2.1.3}/aioscrapy/core/downloader/handlers/requests.py

@@ -16,7 +16,6 @@ class RequestsDownloadHandler(BaseDownloadHandler):
     def __init__(self, settings):
         self.settings: Settings = settings
         self.verify_ssl: bool = self.settings.get("VERIFY_SSL", True)
-        self.loop = asyncio.get_running_loop()
 
     @classmethod
     def from_settings(cls, settings: Settings):
@@ -26,7 +25,7 @@ class RequestsDownloadHandler(BaseDownloadHandler):
         try:
             return await self._download_request(request)
         except RequestsError as e:
-            raise DownloadError from e
+            raise DownloadError(e) from e
 
     async def _download_request(self, request: Request) -> HtmlResponse:
         kwargs = {

{aio-scrapy-2.1.2 → aio-scrapy-2.1.3}/aioscrapy/core/scraper.py

@@ -113,6 +113,11 @@ class Scraper:
             except BaseException as e:
                 await self.handle_spider_error(e, request, result)
             finally:
+                # 控制指纹是否移除
+                self.spider.dupefilter and \
+                    not request.dont_filter and \
+                    await self.spider.dupefilter.done(request, done_type="parse_ok" if getattr(request, "parse_ok", False) else "parse_err")
+
                 if isinstance(result, PlaywrightResponse):
                     await result.release()
 
@@ -161,22 +166,22 @@ class Scraper:
         """Iter each Request/Item (given in the output parameter) returned from the given spider"""
         if not result:
             return
-        parser_successful = True
+
+        parse_ok = True
         while True:
             try:
                 output = await result.__anext__()
             except StopAsyncIteration:
                 break
             except Exception as e:
-                parser_successful = False
+                parse_ok = False
                 await self.handle_spider_error(e, request, response)
             else:
                 await self._process_spidermw_output(output, request, response)
 
         self.spider.dupefilter and \
-        not request.dont_filter and \
-        parser_successful and \
-        await self.spider.dupefilter.done(request, done_type="parse_done")
+            not request.dont_filter and \
+            setattr(request, "parse_ok", parse_ok)
 
     async def _process_spidermw_output(self, output: Any, request: Request, response: Response) -> None:
         """Process each Request/Item (given in the output parameter) returned from the given spider"""

{aio-scrapy-2.1.2 → aio-scrapy-2.1.3}/aioscrapy/dupefilters/__init__.py

@@ -39,5 +39,9 @@ class DupeFilterBase(metaclass=ABCMeta):
 
         spider.crawler.stats.inc_value('dupefilter/filtered', spider=spider)
 
-    async def done(self, request: Request, done_type: Literal["request_done", "parse_done"]) -> None:
-        """ deal fingerprint on task successful """
+    async def done(
+            self,
+            request: Request,
+            done_type: Literal["request_ok", "request_err", "parse_ok", "parse_err"]
+    ) -> None:
+        """ 根据done_type的状态 控制指纹的移除 """

{aio-scrapy-2.1.2 → aio-scrapy-2.1.3}/aioscrapy/dupefilters/redis.py

@@ -130,7 +130,7 @@ class RedisBloomDupeFilter(RedisRFPDupeFilter):
         return False
 
 
-class RedisBloomSetDupeFilter(RedisBloomDupeFilter):
+class ExRedisBloomDupeFilter(RedisBloomDupeFilter):
 
     def __init__(self, server, key, key_set, ttl, debug, bit, hash_number, keep_on_close, info):
         super().__init__(server, key, debug, bit, hash_number, keep_on_close, info)
@@ -161,11 +161,14 @@ class RedisBloomSetDupeFilter(RedisBloomDupeFilter):
             ret, _ = await pipe.execute()
         return ret == 0
 
-    async def done(self, request: Request, done_type: Literal["request_done", "parse_done"]):
-        print(done_type)
-        if done_type == "request_done":
+    async def done(
+            self,
+            request: Request,
+            done_type: Literal["request_ok", "request_err", "parse_ok", "parse_err"]
+    ):
+        if done_type == "request_ok" or done_type == "request_err":
             await self.server.srem(self.key_set, request.fingerprint)
-        elif done_type == "parse_done":
+        elif done_type == "parse_ok":
             await self.bf.insert(request.fingerprint)
 
     async def close(self, reason=''):
@@ -174,6 +177,21 @@ class RedisBloomSetDupeFilter(RedisBloomDupeFilter):
         await self.server.delete(self.key_set)
 
 
+class ExRedisRFPDupeFilter(RedisRFPDupeFilter):
+
+    async def done(
+            self,
+            request: Request,
+            done_type: Literal["request_ok", "request_err", "parse_ok", "parse_err"]
+    ):
+        # 当请求失败或解析失败的时候 从Redis的Set中移除指纹
+        if done_type == "request_err" or done_type == "parse_err":
+            await self.server.srem(self.key, request.fingerprint)
+
+
 RFPDupeFilter = RedisRFPDupeFilter
+ExRFPDupeFilter = ExRedisRFPDupeFilter
 BloomDupeFilter = RedisBloomDupeFilter
-BloomSetDupeFilter = RedisBloomSetDupeFilter
+ExBloomDupeFilter = ExRedisBloomDupeFilter
+BloomSetDupeFilter = ExRedisBloomDupeFilter
+

aio-scrapy-2.1.2/aioscrapy/VERSION

@@ -1 +0,0 @@
-2.1.2