aio-scrapy 2.0.9__tar.gz → 2.0.10__tar.gz

{aio-scrapy-2.0.9/aio_scrapy.egg-info → aio-scrapy-2.0.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: aio-scrapy
-Version: 2.0.9
+Version: 2.0.10
 Summary: A high-level Web Crawling and Web Scraping framework based on Asyncio
 Home-page: https://github.com/conlin-huang/aio-scrapy.git
 Author: conlin

{aio-scrapy-2.0.9 → aio-scrapy-2.0.10/aio_scrapy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: aio-scrapy
-Version: 2.0.9
+Version: 2.0.10
 Summary: A high-level Web Crawling and Web Scraping framework based on Asyncio
 Home-page: https://github.com/conlin-huang/aio-scrapy.git
 Author: conlin

aio-scrapy-2.0.10/aioscrapy/VERSION

@@ -0,0 +1 @@
+2.0.10

{aio-scrapy-2.0.9 → aio-scrapy-2.0.10}/aioscrapy/core/downloader/__init__.py

@@ -126,6 +126,7 @@ class Downloader(BaseDownloader):
         self.dupefilter = dupefilter
 
         self.total_concurrency: int = self.settings.getint('CONCURRENT_REQUESTS')
+        self.get_requests_count: int = self.settings.getint('GET_REQUESTS_COUNT') or self.total_concurrency
         self.domain_concurrency: int = self.settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
         self.ip_concurrency: int = self.settings.getint('CONCURRENT_REQUESTS_PER_IP')
         self.randomize_delay: bool = self.settings.getbool('RANDOMIZE_DOWNLOAD_DELAY')

{aio-scrapy-2.0.9 → aio-scrapy-2.0.10}/aioscrapy/core/engine.py

@@ -138,7 +138,7 @@ class ExecutionEngine(object):
         while self.unlock and not self._needs_backout() and self.unlock:
             self.unlock = False
             try:
-                async for request in self.scheduler.next_request(self.downloader.total_concurrency):
+                async for request in self.scheduler.next_request(self.downloader.get_requests_count):
                     if request:
                         self.slot.add_request(request)
                         await self.downloader.fetch(request)

{aio-scrapy-2.0.9 → aio-scrapy-2.0.10}/aioscrapy/dupefilters/redis.py

@@ -28,7 +28,7 @@ class RedisRFPDupeFilter(DupeFilterBase):
         keep_on_close = crawler.settings.getbool("KEEP_DUPEFILTER_DATA_ON_CLOSE", True)
         key = dupefilter_key % {'spider': crawler.spider.name}
         debug = crawler.settings.getbool('DUPEFILTER_DEBUG', False)
-        info = crawler.settings.getbool('DUPEFILTER_DEBUG', False)
+        info = crawler.settings.getbool('DUPEFILTER_INFO', False)
         instance = cls(server, key=key, debug=debug, keep_on_close=keep_on_close, info=info)
         return instance
 
@@ -102,8 +102,8 @@ class BloomFilter(object):
 class RedisBloomDupeFilter(RedisRFPDupeFilter):
     """Bloom filter built with the bitis bitmap of redis"""
 
-    def __init__(self, server, key, debug, bit, hash_number, keep_on_close):
-        super().__init__(server, key, debug, keep_on_close)
+    def __init__(self, server, key, debug, bit, hash_number, keep_on_close, info):
+        super().__init__(server, key, debug, keep_on_close, info)
         self.bit = bit
         self.hash_number = hash_number
         self.bf = BloomFilter(server, self.key, bit, hash_number)
@@ -115,9 +115,10 @@ class RedisBloomDupeFilter(RedisRFPDupeFilter):
         keep_on_close = crawler.settings.getbool("KEEP_DUPEFILTER_DATA_ON_CLOSE", True)
         key = dupefilter_key % {'spider': crawler.spider.name}
         debug = crawler.settings.getbool('DUPEFILTER_DEBUG', False)
+        info = crawler.settings.getbool('DUPEFILTER_INFO', False)
         bit = crawler.settings.getint('BLOOMFILTER_BIT', 30)
         hash_number = crawler.settings.getint('BLOOMFILTER_HASH_NUMBER', 6)
-        return cls(server, key=key, debug=debug, bit=bit, hash_number=hash_number, keep_on_close=keep_on_close)
+        return cls(server, key=key, debug=debug, bit=bit, hash_number=hash_number, keep_on_close=keep_on_close, info=info)
 
     async def request_seen(self, request: Request) -> bool:
         fp = await self.bf.exists(request.fingerprint)

aio-scrapy-2.0.9/aioscrapy/VERSION

@@ -1 +0,0 @@
-2.0.9