bhfutils 0.2.4__tar.gz → 0.3.0__tar.gz

{bhfutils-0.2.4 → bhfutils-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bhfutils
-Version: 0.2.4
+Version: 0.3.0
 Summary: Utilities that are used by any spider of Behoof project
 Home-page: https://behoof.app/
 Author: Teplygin Vladimir

{bhfutils-0.2.4 → bhfutils-0.3.0}/bhfutils/crawler/custom_cookies.py

@@ -7,7 +7,7 @@ class CustomCookiesMiddleware(CookiesMiddleware):
     persist between calls
     '''
 
-    def process_request(self, request, spider):
+    def process_request(self, request):
         if 'dont_merge_cookies' in request.meta:
             return
 
@@ -22,4 +22,4 @@ class CustomCookiesMiddleware(CookiesMiddleware):
         request.headers.pop('Cookie', None)
         jar.add_cookie_header(request)
 
-        self._debug_cookie(request, spider)
+        self._debug_cookie(request)

{bhfutils-0.2.4 → bhfutils-0.3.0}/bhfutils/crawler/log_retry_middleware.py

@@ -85,12 +85,12 @@ class LogRetryMiddleware(object):
         crawler.settings.set('SPIDER_NAME', crawler.spider.name)
         return cls.from_settings(crawler.settings)
 
-    def process_exception(self, request, exception, spider):
+    def process_exception(self, request, exception):
         if isinstance(exception, self.EXCEPTIONS_TO_RETRY):
-            self._log_retry(request, exception, spider)
+            self._log_retry(request, exception)
             self._increment_504_stat(request)
 
-    def _log_retry(self, request, exception, spider):
+    def _log_retry(self, request, exception):
         extras = {}
         extras['logger'] = self.logger.name
         extras['error_request'] = request

{bhfutils-0.2.4 → bhfutils-0.3.0}/bhfutils/crawler/meta_passthrough_middleware.py

@@ -9,9 +9,9 @@ class MetaPassthroughMiddleware(object):
         self.setup(settings)
 
     def setup(self, settings):
-        """
+        '''
         Does the actual setup of the middleware
-        """
+        '''
         # set up the default sc logger
         my_level = settings.get('SC_LOG_LEVEL', 'INFO')
         my_name = settings.get('SC_LOGGER_NAME', 'sc-logger')
@@ -45,21 +45,21 @@ class MetaPassthroughMiddleware(object):
                     x.meta[key] = response.meta[key]
         return x
 
-    def process_spider_output(self, response, result, spider):
-        """
+    def process_spider_output(self, response, result):
+        '''
         Ensures the meta data from the response is passed
         through in any Request's generated from the spider (sync path).
-        """
+        '''
         self.logger.debug("processing meta passthrough middleware")
         for x in result:
             yield self._passthrough(response, x)
 
-    async def process_spider_output_async(self, response, result, spider):
-        """
+    async def process_spider_output_async(self, response, result):
+        '''
         Async-generator variant required by Scrapy >= 2.13 when the spider
         output is an async iterable (AsyncCrawlerProcess). Defining both makes
         this a "universal" spider middleware compatible with old + new Scrapy.
-        """
+        '''
         self.logger.debug("processing meta passthrough middleware (async)")
         async for x in result:
-            yield self._passthrough(response, x)
+            yield self._passthrough(response, x)

{bhfutils-0.2.4 → bhfutils-0.3.0}/bhfutils/crawler/redis_stats_middleware.py

@@ -58,7 +58,9 @@ class RedisStatsMiddleware(object):
 
     @classmethod
     def from_crawler(cls, crawler):
-        return cls(crawler.settings)
+        obj = cls(crawler.settings)
+        obj.crawler = crawler
+        return obj
 
     def _get_hostname(self):
         '''
@@ -108,12 +110,13 @@ class RedisStatsMiddleware(object):
                               .format(h=hostname, n=spider_name, s=status_code))
             self.stats_dict[spider_name]['status_codes'][status_code]['lifetime'] = total
 
-    def process_spider_input(self, response, spider):
+    def process_spider_input(self, response):
         '''
         Ensures the meta data from the response is passed
         through in any Request's generated from the spider
         '''
         self.logger.debug("processing redis stats middleware")
+        spider = self.crawler.spider
         if self.settings['STATS_STATUS_CODES']:
             if spider.name not in self.stats_dict:
                 self._setup_stats_status_codes(spider.name)

{bhfutils-0.2.4 → bhfutils-0.3.0}/bhfutils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bhfutils
-Version: 0.2.4
+Version: 0.3.0
 Summary: Utilities that are used by any spider of Behoof project
 Home-page: https://behoof.app/
 Author: Teplygin Vladimir

{bhfutils-0.2.4 → bhfutils-0.3.0}/setup.py

@@ -43,7 +43,7 @@ if 'nosetests' in sys.argv[1:]:
 
 setup(
     name='bhfutils',
-    version='0.2.4',
+    version='0.3.0',
     description='Utilities that are used by any spider of Behoof project',
     long_description=readme(),
     long_description_content_type='text/x-rst',