modis-crawler-utils 0.3.0__tar.gz → 0.3.2__tar.gz

{modis_crawler_utils-0.3.0 → modis_crawler_utils-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: modis-crawler-utils
-Version: 0.3.0
+Version: 0.3.2
 Summary: Scrapy utils for Modis crawlers projects.
 Home-page: https://gitlab.at.ispras.ru/crawlers/crawler-utils
 License: BSD

{modis_crawler_utils-0.3.0 → modis_crawler_utils-0.3.2}/crawler_utils/echo.py

@@ -31,6 +31,7 @@ class EchoRequest(Request):
         kwargs.setdefault('dont_filter', True)
         super().__init__(**kwargs)
         self.response = response
+        self.meta.setdefault('dont_index', True)
 
     def __str__(self):
         return f'<ECHO {super().__str__()}>'

{modis_crawler_utils-0.3.0 → modis_crawler_utils-0.3.2}/crawler_utils/elasticsearch/requests_middleware.py

@@ -28,9 +28,12 @@ class ElasticRequestsDownloaderMiddleware:
         return mid
 
     def process_request(self, request, spider):
-        request.meta['request_timer/start'] = time.time()
+        if not request.meta.get('dont_index'):
+            request.meta['request_timer/start'] = time.time()
 
     def process_response(self, request, response, spider):
+        if request.meta.get('dont_index') or 'request_timer/start' not in request.meta:
+            return response
         item = self._request_item_base(request)
         item['url'] = response.url
         item['http_status'] = response.status
@@ -39,10 +42,10 @@ class ElasticRequestsDownloaderMiddleware:
         return response
 
     def process_exception(self, request, exception, spider):
-        if 'request_timer/start' in request.meta:
-            item = self._request_item_base(request)
-            self.es_storage.index(self.index, item)
-        return
+        if request.meta.get('dont_index') or 'request_timer/start' not in request.meta:
+            return
+        item = self._request_item_base(request)
+        self.es_storage.index(self.index, item)
 
     def _request_item_base(self, request):
         timestamp = time.time()

{modis_crawler_utils-0.3.0 → modis_crawler_utils-0.3.2}/crawler_utils/files.py

@@ -9,7 +9,7 @@ from collections.abc import Iterable
 from contextlib import suppress
 from io import BytesIO
 from os.path import basename, join, splitext
-from typing import BinaryIO, Union
+from typing import BinaryIO, MutableMapping, Optional, Union
 from urllib.parse import quote_plus, unquote, urlsplit
 
 from itemadapter import ItemAdapter
@@ -24,10 +24,31 @@ from scrapy.utils.misc import md5sum
 from twisted.internet.defer import DeferredList, maybeDeferred
 
 from crawler_utils.elasticsearch.request_fingerprint import default_request_fingerprint
+from crawler_utils.social_media.items import AttachmentType
 
 logger = logging.getLogger(__name__)
 
 
+def set_attachment_type(attachment: MutableMapping, attachment_type: Optional[str]):
+    if attachment_type is None:
+        attachment.pop('attachment_type', None)
+        attachment.pop('attachment_type_raw', None)
+        return
+    try:
+        attachment['attachment_type'] = AttachmentType(attachment_type).value
+        attachment.pop('attachment_type_raw', None)
+    except ValueError:
+        attachment['attachment_type_raw'] = attachment_type
+        attachment.pop('attachment_type', None)
+
+
+def get_attachment_type(attachment: MutableMapping) -> Optional[str]:
+    attachment_type = attachment.get('attachment_type')
+    if isinstance(attachment_type, AttachmentType):
+        attachment_type = attachment_type.value
+    return attachment_type or attachment.get('attachment_type_raw')
+
+
 class FilesPipeline(ImagesPipelineBase, FilesPipelineBase):
     """ Pipeline for downloading files.
 
@@ -363,7 +384,7 @@ class FilesPipeline(ImagesPipelineBase, FilesPipelineBase):
                         'attachment': attachment,
                         'resource_tag': resource_tag,
                         'is_image_request': (self._process_images
-                                             and attachment.get('attachment_type') in self._image_attachment_types)
+                                             and get_attachment_type(attachment) in self._image_attachment_types)
                     }
                 )
 
@@ -423,9 +444,10 @@ class FilesPipeline(ImagesPipelineBase, FilesPipelineBase):
 
     def thumb_path(self, request, thumb_id, response=None, info=None, *, item=None):
         attachment = request.meta.get('attachment') or {'url': request.url}
-        type_tag = attachment.get('attachment_type') or self._default_type_tag
+        type_tag = get_attachment_type(attachment) or self._default_type_tag
         thumb_type_tag = f'{type_tag}-{thumb_id}' if type_tag else thumb_id
-        thumb_attachment = {**attachment, 'attachment_type': thumb_type_tag}
+        thumb_attachment = {**attachment}
+        set_attachment_type(thumb_attachment, thumb_type_tag)
         thumb_request = request.replace(meta={**request.meta, 'attachment': thumb_attachment})
         return self.file_path(thumb_request, response, info)
 
@@ -455,7 +477,8 @@ class FilesPipeline(ImagesPipelineBase, FilesPipelineBase):
                 raise ValueError('Either url or store_path must be provided')
             if not filename:
                 filename = self._filename_from_url(url)
-            attachment = {'url': url, 'filename': filename, 'attachment_type': type_tag}
+            attachment = {'url': url, 'filename': filename}
+            set_attachment_type(attachment, type_tag)
             request = Request(url, meta={'attachment': attachment, 'resource_tag': resource_tag})
             store_path = self.file_path(request, info=self.spiderinfo)
 
@@ -559,7 +582,7 @@ class FilesPipeline(ImagesPipelineBase, FilesPipelineBase):
             return False
         if attachment.get('status'):
             return False  # already processed
-        attachment_type = attachment.get('attachment_type')
+        attachment_type = get_attachment_type(attachment)
         if not attachment_type:
             return True
         if self._allow_attachment_types:
@@ -608,7 +631,7 @@ class FilesPipeline(ImagesPipelineBase, FilesPipelineBase):
         path = ''
         if resource_tag := request.meta.get('resource_tag') or self._default_resource_tag:
             path += '/' + resource_tag
-        if type_tag := request.meta.get('attachment', {}).get('attachment_type') or self._default_type_tag:
+        if type_tag := get_attachment_type(request.meta.get('attachment', {})) or self._default_type_tag:
             path += '/' + type_tag
         if self._digest_path_segment_lengths:
             digest = self._digest_func(request.url.encode('utf-8')).digest()

{modis_crawler_utils-0.3.0 → modis_crawler_utils-0.3.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "modis-crawler-utils"
-version = "0.3.0"
+version = "0.3.2"
 description = "Scrapy utils for Modis crawlers projects."
 authors = [
     "Varlamov <varlamov@ispras.ru>",