modis-crawler-utils 0.2.78__tar.gz → 0.2.79__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/PKG-INFO +1 -1
  2. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/social_media/items.py +12 -2
  3. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/pyproject.toml +1 -1
  4. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/README.md +0 -0
  5. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/__init__.py +0 -0
  6. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/cloudflare_captcha_bypass.py +0 -0
  7. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/credentials/__init__.py +0 -0
  8. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/credentials/credential.py +0 -0
  9. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/credentials/manager.py +0 -0
  10. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/credentials/store/__init__.py +0 -0
  11. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/credentials/store/local.py +0 -0
  12. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/credentials/store/talisman.py +0 -0
  13. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/deploy/__init__.py +0 -0
  14. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/deploy/bump_version.py +0 -0
  15. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/deploy/create_egg.py +0 -0
  16. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/deploy/make_arguments_md_file.py +0 -0
  17. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/domains.py +0 -0
  18. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/echo.py +0 -0
  19. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/elasticsearch/__init__.py +0 -0
  20. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/elasticsearch/dupefilter.py +0 -0
  21. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/elasticsearch/dupefilter_strategies/__init__.py +0 -0
  22. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/elasticsearch/dupefilter_strategies/delta_fetch_items.py +0 -0
  23. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/elasticsearch/dupefilter_strategies/depth_based.py +0 -0
  24. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/elasticsearch/dupefilter_strategies/drop_unproductive_requests.py +0 -0
  25. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/elasticsearch/items_pipeline.py +0 -0
  26. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/elasticsearch/request_fingerprint.py +0 -0
  27. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/elasticsearch/requests_middleware.py +0 -0
  28. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/elasticsearch/storage.py +0 -0
  29. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/files.py +0 -0
  30. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/images.py +0 -0
  31. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/kafka.py +0 -0
  32. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/logstash.py +0 -0
  33. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/misc.py +0 -0
  34. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/mongodb.py +0 -0
  35. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/proxypy.py +0 -0
  36. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/social_media/__init__.py +0 -0
  37. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/socks.py +0 -0
  38. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/splash_proxy.py +0 -0
  39. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/states/__init__.py +0 -0
  40. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/states/local_store.py +0 -0
  41. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/states/talisman_states_api.py +0 -0
  42. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/talisman_controller.py +0 -0
  43. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/talisman_job_env.py +0 -0
  44. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/talisman_proxy.py +0 -0
  45. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/talisman_spider_state.py +0 -0
  46. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/timestamp.py +0 -0
  47. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/trust_level.py +0 -0
  48. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/uuid.py +0 -0
  49. {modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/validate_input.py +0 -0
{modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: modis-crawler-utils
3
- Version: 0.2.78
3
+ Version: 0.2.79
4
4
  Summary: Scrapy utils for Modis crawlers projects.
5
5
  Home-page: https://gitlab.at.ispras.ru/crawlers/crawler-utils
6
6
  License: BSD
{modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/crawler_utils/social_media/items.py RENAMED
@@ -3,7 +3,7 @@ from dataclasses import dataclass, field
3
3
  from datetime import datetime
4
4
  from enum import Enum
5
5
  from typing import List
6
-
6
+ from typing import Union
7
7
  from dateparser import parse
8
8
 
9
9
  UnixTimestamp = int # Имитация типа timestamp
@@ -70,6 +70,15 @@ class AccountStatus(Enum):
70
70
  NOT_SPECIFIED = None
71
71
 
72
72
 
73
+ class AttachmentType(Enum):
74
+ DOCUMENT = "document"
75
+ IMAGE = "image"
76
+ LINK = "link"
77
+ VIDEO = "video"
78
+ AUDIO = "audio"
79
+ NOT_SPECIFIED = None
80
+
81
+
73
82
  class Gender(Enum):
74
83
  MALE = "Male"
75
84
  FEMALE = "Female"
@@ -328,12 +337,13 @@ class Contact:
328
337
  @dataclass
329
338
  class Attachment:
330
339
  url: str
331
- attachment_type: str = None
340
+ attachment_type: Union[str, AttachmentType] = None
332
341
  title: str = None
333
342
  path: str = None
334
343
  filename: str = None
335
344
  checksum: str = None
336
345
  status: str = None
346
+ to_be_downloaded: bool = True
337
347
  # __________Other Fields____________
338
348
  attachment_id: str = None # VK
339
349
  owner_id: str = None # VK
{modis_crawler_utils-0.2.78 → modis_crawler_utils-0.2.79}/pyproject.toml RENAMED
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "modis-crawler-utils"
3
- version = "0.2.78"
3
+ version = "0.2.79"
4
4
  description = "Scrapy utils for Modis crawlers projects."
5
5
  authors = [
6
6
  "Varlamov <varlamov@ispras.ru>",