bhfutils 0.2.2__tar.gz → 0.2.4__tar.gz

{bhfutils-0.2.2 → bhfutils-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bhfutils
-Version: 0.2.2
+Version: 0.2.4
 Summary: Utilities that are used by any spider of Behoof project
 Home-page: https://behoof.app/
 Author: Teplygin Vladimir

{bhfutils-0.2.2 → bhfutils-0.2.4}/bhfutils/crawler/items.py

@@ -51,6 +51,7 @@ class ProductDetailsResponseItem(ProductResponseItem):
     brandName = Field()
     sellerName = Field()
     description = Field()
+    vendorCode = Field()
     details = Field()
     comments = Field()
 

{bhfutils-0.2.2 → bhfutils-0.2.4}/bhfutils/crawler/meta_passthrough_middleware.py

@@ -9,9 +9,9 @@ class MetaPassthroughMiddleware(object):
         self.setup(settings)
 
     def setup(self, settings):
-        '''
+        """
         Does the actual setup of the middleware
-        '''
+        """
         # set up the default sc logger
         my_level = settings.get('SC_LOG_LEVEL', 'INFO')
         my_name = settings.get('SC_LOGGER_NAME', 'sc-logger')
@@ -35,19 +35,31 @@ class MetaPassthroughMiddleware(object):
     def from_crawler(cls, crawler):
         return cls(crawler.settings)
 
+    @staticmethod
+    def _passthrough(response, x):
+        # only operate on requests: pass along all known meta fields, but only
+        # if they were not already set in the spider's new request
+        if isinstance(x, Request):
+            for key in list(response.meta.keys()):
+                if key not in x.meta and key != 'playwright_page_methods':
+                    x.meta[key] = response.meta[key]
+        return x
+
     def process_spider_output(self, response, result, spider):
-        '''
+        """
         Ensures the meta data from the response is passed
-        through in any Request's generated from the spider
-        '''
+        through in any Request's generated from the spider (sync path).
+        """
         self.logger.debug("processing meta passthrough middleware")
         for x in result:
-            # only operate on requests
-            if isinstance(x, Request):
-                self.logger.debug("found request")
-                # pass along all known meta fields, only if
-                # they were not already set in the spider's new request
-                for key in list(response.meta.keys()):
-                    if key not in x.meta and key != 'playwright_page_methods':
-                        x.meta[key] = response.meta[key]
-            yield x
+            yield self._passthrough(response, x)
+
+    async def process_spider_output_async(self, response, result, spider):
+        """
+        Async-generator variant required by Scrapy >= 2.13 when the spider
+        output is an async iterable (AsyncCrawlerProcess). Defining both makes
+        this a "universal" spider middleware compatible with old + new Scrapy.
+        """
+        self.logger.debug("processing meta passthrough middleware (async)")
+        async for x in result:
+            yield self._passthrough(response, x)

{bhfutils-0.2.2 → bhfutils-0.2.4}/bhfutils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bhfutils
-Version: 0.2.2
+Version: 0.2.4
 Summary: Utilities that are used by any spider of Behoof project
 Home-page: https://behoof.app/
 Author: Teplygin Vladimir

{bhfutils-0.2.2 → bhfutils-0.2.4}/setup.py

@@ -43,7 +43,7 @@ if 'nosetests' in sys.argv[1:]:
 
 setup(
     name='bhfutils',
-    version='0.2.2',
+    version='0.2.4',
     description='Utilities that are used by any spider of Behoof project',
     long_description=readme(),
     long_description_content_type='text/x-rst',