modis-crawler-utils 0.3.1__tar.gz → 0.3.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/PKG-INFO +1 -1
  2. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/echo.py +1 -0
  3. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/elasticsearch/requests_middleware.py +8 -5
  4. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/pyproject.toml +1 -1
  5. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/README.md +0 -0
  6. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/__init__.py +0 -0
  7. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/cloudflare_captcha_bypass.py +0 -0
  8. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/credentials/__init__.py +0 -0
  9. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/credentials/credential.py +0 -0
  10. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/credentials/manager.py +0 -0
  11. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/credentials/store/__init__.py +0 -0
  12. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/credentials/store/local.py +0 -0
  13. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/credentials/store/talisman.py +0 -0
  14. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/deploy/__init__.py +0 -0
  15. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/deploy/bump_version.py +0 -0
  16. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/deploy/create_egg.py +0 -0
  17. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/deploy/make_arguments_md_file.py +0 -0
  18. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/domains.py +0 -0
  19. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/elasticsearch/__init__.py +0 -0
  20. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/elasticsearch/dupefilter.py +0 -0
  21. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/elasticsearch/dupefilter_strategies/__init__.py +0 -0
  22. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/elasticsearch/dupefilter_strategies/delta_fetch_items.py +0 -0
  23. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/elasticsearch/dupefilter_strategies/depth_based.py +0 -0
  24. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/elasticsearch/dupefilter_strategies/drop_unproductive_requests.py +0 -0
  25. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/elasticsearch/items_pipeline.py +0 -0
  26. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/elasticsearch/request_fingerprint.py +0 -0
  27. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/elasticsearch/storage.py +0 -0
  28. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/files.py +0 -0
  29. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/images.py +0 -0
  30. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/kafka.py +0 -0
  31. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/logstash.py +0 -0
  32. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/misc.py +0 -0
  33. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/mongodb.py +0 -0
  34. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/proxypy.py +0 -0
  35. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/social_media/__init__.py +0 -0
  36. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/social_media/items.py +0 -0
  37. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/socks.py +0 -0
  38. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/splash_proxy.py +0 -0
  39. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/states/__init__.py +0 -0
  40. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/states/local_store.py +0 -0
  41. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/states/talisman_states_api.py +0 -0
  42. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/talisman_controller.py +0 -0
  43. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/talisman_job_env.py +0 -0
  44. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/talisman_proxy.py +0 -0
  45. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/talisman_spider_state.py +0 -0
  46. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/timestamp.py +0 -0
  47. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/trust_level.py +0 -0
  48. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/uuid.py +0 -0
  49. {modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/validate_input.py +0 -0
{modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: modis-crawler-utils
3
- Version: 0.3.1
3
+ Version: 0.3.2
4
4
  Summary: Scrapy utils for Modis crawlers projects.
5
5
  Home-page: https://gitlab.at.ispras.ru/crawlers/crawler-utils
6
6
  License: BSD
{modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/echo.py RENAMED
@@ -31,6 +31,7 @@ class EchoRequest(Request):
31
31
  kwargs.setdefault('dont_filter', True)
32
32
  super().__init__(**kwargs)
33
33
  self.response = response
34
+ self.meta.setdefault('dont_index', True)
34
35
 
35
36
  def __str__(self):
36
37
  return f'<ECHO {super().__str__()}>'
{modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/crawler_utils/elasticsearch/requests_middleware.py RENAMED
@@ -28,9 +28,12 @@ class ElasticRequestsDownloaderMiddleware:
28
28
  return mid
29
29
 
30
30
  def process_request(self, request, spider):
31
- request.meta['request_timer/start'] = time.time()
31
+ if not request.meta.get('dont_index'):
32
+ request.meta['request_timer/start'] = time.time()
32
33
 
33
34
  def process_response(self, request, response, spider):
35
+ if request.meta.get('dont_index') or 'request_timer/start' not in request.meta:
36
+ return response
34
37
  item = self._request_item_base(request)
35
38
  item['url'] = response.url
36
39
  item['http_status'] = response.status
@@ -39,10 +42,10 @@ class ElasticRequestsDownloaderMiddleware:
39
42
  return response
40
43
 
41
44
  def process_exception(self, request, exception, spider):
42
- if 'request_timer/start' in request.meta:
43
- item = self._request_item_base(request)
44
- self.es_storage.index(self.index, item)
45
- return
45
+ if request.meta.get('dont_index') or 'request_timer/start' not in request.meta:
46
+ return
47
+ item = self._request_item_base(request)
48
+ self.es_storage.index(self.index, item)
46
49
 
47
50
  def _request_item_base(self, request):
48
51
  timestamp = time.time()
{modis_crawler_utils-0.3.1 → modis_crawler_utils-0.3.2}/pyproject.toml RENAMED
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "modis-crawler-utils"
3
- version = "0.3.1"
3
+ version = "0.3.2"
4
4
  description = "Scrapy utils for Modis crawlers projects."
5
5
  authors = [
6
6
  "Varlamov <varlamov@ispras.ru>",