aio-scrapy 2.0.8__tar.gz → 2.0.9__tar.gz

{aio-scrapy-2.0.8/aio_scrapy.egg-info → aio-scrapy-2.0.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: aio-scrapy
-Version: 2.0.8
+Version: 2.0.9
 Summary: A high-level Web Crawling and Web Scraping framework based on Asyncio
 Home-page: https://github.com/conlin-huang/aio-scrapy.git
 Author: conlin

{aio-scrapy-2.0.8 → aio-scrapy-2.0.9/aio_scrapy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: aio-scrapy
-Version: 2.0.8
+Version: 2.0.9
 Summary: A high-level Web Crawling and Web Scraping framework based on Asyncio
 Home-page: https://github.com/conlin-huang/aio-scrapy.git
 Author: conlin

aio-scrapy-2.0.9/aioscrapy/VERSION

@@ -0,0 +1 @@
+2.0.9

aio-scrapy-2.0.9/aioscrapy/dupefilters/__init__.py

@@ -0,0 +1,39 @@
+from abc import ABCMeta, abstractmethod
+
+from aioscrapy import Request, Spider
+from aioscrapy.utils.log import logger
+
+
+class DupeFilterBase(metaclass=ABCMeta):
+    """Request Fingerprint duplicates filter"""
+
+    @classmethod
+    @abstractmethod
+    def from_crawler(cls, crawler: "aioscrapy.crawler.Crawler"):
+        """ Get Instance of RFPDupeFilter from crawler """
+
+    @abstractmethod
+    async def request_seen(self, request: Request) -> bool:
+        """ Check whether fingerprint of request exists """
+
+    @abstractmethod
+    async def close(self, reason: str = '') -> None:
+        """ Delete data on close """
+
+    def log(self, request: Request, spider: Spider):
+        if self.info:
+            logger.info("Filtered duplicate request: %(request)s" % {
+                'request': request.meta.get('dupefilter_msg') or request
+            })
+        elif self.debug:
+            logger.debug("Filtered duplicate request: %(request)s" % {
+                'request': request.meta.get('dupefilter_msg') or request
+            })
+        elif self.logdupes:
+            msg = ("Filtered duplicate request: %(request)s"
+                   " - no more duplicates will be shown"
+                   " (see DUPEFILTER_DEBUG to show all duplicates)")
+            logger.debug(msg % {'request': request.meta.get('dupefilter_msg') or request})
+            self.logdupes = False
+
+        spider.crawler.stats.inc_value('dupefilter/filtered', spider=spider)

{aio-scrapy-2.0.8 → aio-scrapy-2.0.9}/aioscrapy/dupefilters/disk.py

@@ -1,20 +1,19 @@
 import os
 from typing import Optional, Set
 
-from aioscrapy import Request, Spider
+from aioscrapy import Request
 from aioscrapy.dupefilters import DupeFilterBase
-from aioscrapy.utils.log import logger
-from aioscrapy.utils.request import referer_str
 
 
 class DiskRFPDupeFilter(DupeFilterBase):
     """Request Fingerprint duplicates filter built with Disk storage"""
 
-    def __init__(self, path: Optional[str] = None, debug: bool = False):
+    def __init__(self, path: Optional[str] = None, debug: bool = False, info: bool = False):
         self.file: Optional["File object"] = None
         self.debug = debug
         self.fingerprints: Set = set()
         self.logdupes: bool = True
+        self.info: bool = info
         if path:
             self.file = open(os.path.join(path, 'requests.seen'), 'a+')
             self.file.seek(0)
@@ -23,10 +22,11 @@ class DiskRFPDupeFilter(DupeFilterBase):
     @classmethod
     def from_crawler(cls, crawler: "aioscrapy.crawler.Crawler"):
         debug = crawler.settings.getbool('DUPEFILTER_DEBUG')
+        info = crawler.settings.getbool('DUPEFILTER_INFO')
         path = crawler.settings.get('JOBDIR', './job_dir')
         if path and not os.path.exists(path):
             os.makedirs(path)
-        return cls(path, debug)
+        return cls(path, debug, info)
 
     async def request_seen(self, request: Request) -> bool:
         if request.fingerprint in self.fingerprints:
@@ -40,19 +40,5 @@ class DiskRFPDupeFilter(DupeFilterBase):
         if self.file:
             self.file.close()
 
-    def log(self, request: Request, spider: Spider):
-        if self.debug:
-            logger.debug("Filtered duplicate request: %(request)s (referer: %(referer)s)" % {
-                'request': request, 'referer': referer_str(request)
-            })
-        elif self.logdupes:
-            msg = ("Filtered duplicate request: %(request)s"
-                   " - no more duplicates will be shown"
-                   " (see DUPEFILTER_DEBUG to show all duplicates)")
-            logger.debug(msg % {'request': request})
-            self.logdupes = False
-
-        spider.crawler.stats.inc_value('dupefilter/filtered', spider=spider)
-
 
 RFPDupeFilter = DiskRFPDupeFilter

{aio-scrapy-2.0.8 → aio-scrapy-2.0.9}/aioscrapy/dupefilters/redis.py

@@ -2,8 +2,6 @@ from aioscrapy import Request
 from aioscrapy.db import db_manager
 from aioscrapy.dupefilters import DupeFilterBase
 
-from aioscrapy.utils.log import logger
-
 
 class RedisRFPDupeFilter(DupeFilterBase):
     """Request Fingerprint duplicates filter built with Set of Redis"""
@@ -13,13 +11,15 @@ class RedisRFPDupeFilter(DupeFilterBase):
             server: "redis.asyncio.Redis",
             key: str,
             debug: bool = False,
-            keep_on_close: bool = True
+            keep_on_close: bool = True,
+            info: bool = False,
     ):
         self.server = server
         self.key = key
         self.debug = debug
         self.keep_on_close = keep_on_close
         self.logdupes: bool = True
+        self.info: bool = info
 
     @classmethod
     def from_crawler(cls, crawler: "aioscrapy.crawler.Crawler"):
@@ -28,7 +28,8 @@ class RedisRFPDupeFilter(DupeFilterBase):
         keep_on_close = crawler.settings.getbool("KEEP_DUPEFILTER_DATA_ON_CLOSE", True)
         key = dupefilter_key % {'spider': crawler.spider.name}
         debug = crawler.settings.getbool('DUPEFILTER_DEBUG', False)
-        instance = cls(server, key=key, debug=debug, keep_on_close=keep_on_close)
+        info = crawler.settings.getbool('DUPEFILTER_DEBUG', False)
+        instance = cls(server, key=key, debug=debug, keep_on_close=keep_on_close, info=info)
         return instance
 
     async def request_seen(self, request: Request):
@@ -41,17 +42,6 @@ class RedisRFPDupeFilter(DupeFilterBase):
     async def clear(self):
         await self.server.delete(self.key)
 
-    def log(self, request, spider):
-        if self.debug:
-            logger.debug("Filtered duplicate request: %(request)s" % {'request': request})
-        elif self.logdupes:
-            msg = ("Filtered duplicate request %(request)s"
-                   " - no more duplicates will be shown"
-                   " (see DUPEFILTER_DEBUG to show all duplicates)")
-            logger.debug(msg % {'request': request})
-            self.logdupes = False
-        spider.crawler.stats.inc_value('dupefilter/filtered', spider=spider)
-
 
 class HashMap(object):
     def __init__(self, m, seed):

{aio-scrapy-2.0.8 → aio-scrapy-2.0.9}/aioscrapy/libs/downloader/retry.py

@@ -26,6 +26,13 @@ try:
 except ImportError:
     pass
 
+try:
+    from anyio import EndOfStream
+
+    NEED_RETRY_ERROR += (EndOfStream,)
+except ImportError:
+    pass
+
 try:
     from httpx import HTTPError as HttpxError
 

{aio-scrapy-2.0.8 → aio-scrapy-2.0.9}/aioscrapy/utils/log.py

@@ -7,7 +7,9 @@ from loguru import logger as _logger
 
 from aioscrapy.settings import Settings
 
-_logger.remove(0)
+for _handler in _logger._core.handlers.values():
+    if _handler._name == '<stderr>':
+        _logger.remove(_handler._id)
 
 
 def configure_logging(spider: Type["Spider"], settings: Settings):

aio-scrapy-2.0.8/aioscrapy/VERSION

@@ -1 +0,0 @@
-2.0.8

aio-scrapy-2.0.8/aioscrapy/dupefilters/__init__.py

@@ -1,24 +0,0 @@
-from abc import ABCMeta, abstractmethod
-
-from aioscrapy import Request, Spider
-
-
-class DupeFilterBase(metaclass=ABCMeta):
-    """Request Fingerprint duplicates filter"""
-
-    @classmethod
-    @abstractmethod
-    def from_crawler(cls, crawler: "aioscrapy.crawler.Crawler"):
-        """ Get Instance of RFPDupeFilter from crawler """
-
-    @abstractmethod
-    async def request_seen(self, request: Request) -> bool:
-        """ Check whether fingerprint of request exists """
-
-    @abstractmethod
-    async def close(self, reason: str = '') -> None:
-        """ Delete data on close """
-
-    @abstractmethod
-    def log(self, request: Request, spider: Spider) -> None:
-        """ Logs given request """