warp-beacon 2.2.63__py3-none-any.whl → 2.2.64__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (12) hide show
  1. warp_beacon/__version__.py +1 -1
  2. warp_beacon/scraper/__init__.py +4 -2
  3. warp_beacon/scraper/fail_handler.py +4 -10
  4. warp_beacon/storage/__init__.py +5 -11
  5. warp_beacon/storage/mongo.py +22 -0
  6. warp_beacon/telegram/bot.py +7 -2
  7. {warp_beacon-2.2.63.dist-info → warp_beacon-2.2.64.dist-info}/METADATA +1 -1
  8. {warp_beacon-2.2.63.dist-info → warp_beacon-2.2.64.dist-info}/RECORD +12 -11
  9. {warp_beacon-2.2.63.dist-info → warp_beacon-2.2.64.dist-info}/top_level.txt +1 -0
  10. {warp_beacon-2.2.63.dist-info → warp_beacon-2.2.64.dist-info}/LICENSE +0 -0
  11. {warp_beacon-2.2.63.dist-info → warp_beacon-2.2.64.dist-info}/WHEEL +0 -0
  12. {warp_beacon-2.2.63.dist-info → warp_beacon-2.2.64.dist-info}/entry_points.txt +0 -0
warp_beacon/__version__.py CHANGED
@@ -1,2 +1,2 @@
1
- __version__ = "2.2.63"
1
+ __version__ = "2.2.64"
2
2
 
warp_beacon/scraper/__init__.py CHANGED
@@ -16,6 +16,7 @@ from warp_beacon.jobs.download_job import DownloadJob
16
16
  from warp_beacon.jobs.upload_job import UploadJob
17
17
  from warp_beacon.jobs.types import JobType
18
18
  from warp_beacon.scraper.account_selector import AccountSelector
19
+ from warp_beacon.storage.mongo import DBClient
19
20
  from warp_beacon.scraper.fail_handler import FailHandler
20
21
 
21
22
  import logging
@@ -33,9 +34,10 @@ class AsyncDownloader(object):
33
34
  manager = None
34
35
  acc_selector = None
35
36
  scheduler = None
36
- fail_handler = FailHandler()
37
+ fail_handler = None
37
38
 
38
- def __init__(self, uploader: AsyncUploader, workers_count: int) -> None:
39
+ def __init__(self, uploader: AsyncUploader, db_connect: DBClient, workers_count: int) -> None:
40
+ self.fail_handler = FailHandler(db_connect)
39
41
  self.manager = multiprocessing.Manager()
40
42
  self.allow_loop = self.manager.Value('i', 1)
41
43
  self.acc_selector = AccountSelector(self.manager, ACC_FILE)
warp_beacon/scraper/fail_handler.py CHANGED
@@ -1,8 +1,8 @@
1
1
  import os
2
2
 
3
3
  import pickle
4
- from pymongo import MongoClient
5
4
 
5
+ from warp_beacon.storage.mongo import DBClient
6
6
  from warp_beacon.jobs.download_job import DownloadJob
7
7
 
8
8
  import logging
@@ -10,17 +10,11 @@ import logging
10
10
  class FailHandler(object):
11
11
  client = None
12
12
  db = None
13
- def __init__(self) -> None:
14
- self.client = MongoClient(
15
- host=os.environ.get("MONGODB_HOST", default='127.0.0.1'),
16
- port=int(os.environ.get("MONGODB_PORT", default=27017)),
17
- username=os.environ.get("MONGODB_USER", default='root'),
18
- password=os.environ.get("MONGODB_PASSWORD", default="changeme"))
19
- self.db = self.client.media.failed_jobs
13
+ def __init__(self, client: DBClient) -> None:
14
+ self.db = client.client.media.failed_jobs
20
15
 
21
16
  def __del__(self) -> None:
22
- if self.client:
23
- self.client.close()
17
+ self.client.close()
24
18
 
25
19
  def store_failed_job(self, job: DownloadJob) -> int:
26
20
  db_id = -1
warp_beacon/storage/__init__.py CHANGED
@@ -4,7 +4,7 @@ from enum import Enum
4
4
 
5
5
  from urllib.parse import urlparse, parse_qs
6
6
 
7
- from pymongo import MongoClient
7
+ from warp_beacon.storage.mongo import DBClient
8
8
 
9
9
  import logging
10
10
 
@@ -19,20 +19,14 @@ VIDEO_STORAGE_DIR = os.environ.get("VIDEO_STORAGE_DIR", default="/var/warp_beaco
19
19
  class Storage(object):
20
20
  client = None
21
21
  db = None
22
- def __init__(self) -> None:
22
+ def __init__(self, client: DBClient) -> None:
23
23
  if not os.path.isdir(VIDEO_STORAGE_DIR):
24
24
  os.mkdir(VIDEO_STORAGE_DIR)
25
-
26
- self.client = MongoClient(
27
- host=os.environ.get("MONGODB_HOST", default='127.0.0.1'),
28
- port=int(os.environ.get("MONGODB_PORT", default=27017)),
29
- username=os.environ.get("MONGODB_USER", default='root'),
30
- password=os.environ.get("MONGODB_PASSWORD", default="changeme"))
31
- self.db = self.client.media.media
25
+ self.client = client
26
+ self.db = self.client.client.media.media
32
27
 
33
28
  def __del__(self) -> None:
34
- if self.client:
35
- self.client.close()
29
+ self.client.close()
36
30
 
37
31
  @staticmethod
38
32
  def compute_uniq(url: str) -> str:
warp_beacon/storage/mongo.py ADDED
@@ -0,0 +1,22 @@
1
+ import os
2
+
3
+ from pymongo import MongoClient
4
+
5
+ class DBClient(object):
6
+ client = None
7
+
8
+ def __init__(self) -> None:
9
+ self.client = MongoClient(
10
+ host=os.environ.get("MONGODB_HOST", default='127.0.0.1'),
11
+ port=int(os.environ.get("MONGODB_PORT", default=27017)),
12
+ username=os.environ.get("MONGODB_USER", default='root'),
13
+ password=os.environ.get("MONGODB_PASSWORD", default="changeme")
14
+ )
15
+
16
+ def __del__(self) -> None:
17
+ self.close()
18
+
19
+ def close(self) -> None:
20
+ if self.client:
21
+ self.client.close()
22
+ self.client = None
warp_beacon/telegram/bot.py CHANGED
@@ -15,6 +15,7 @@ import warp_beacon
15
15
  from warp_beacon.__version__ import __version__
16
16
  from warp_beacon.telegram.handlers import Handlers
17
17
  from warp_beacon.telegram.placeholder_message import PlaceholderMessage
18
+ from warp_beacon.storage.mongo import DBClient
18
19
  from warp_beacon.storage import Storage
19
20
  from warp_beacon.uploader import AsyncUploader
20
21
  from warp_beacon.jobs.upload_job import UploadJob
@@ -25,7 +26,7 @@ from warp_beacon.scheduler.scheduler import IGScheduler
25
26
  import logging
26
27
 
27
28
  class Bot(object):
28
- storage = Storage()
29
+ storage = None
29
30
  uploader = None
30
31
  downloader = None
31
32
  allow_loop = True
@@ -40,6 +41,9 @@ class Bot(object):
40
41
  format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", level=logging.INFO
41
42
  )
42
43
 
44
+ db_connect = DBClient()
45
+ self.storage = Storage(db_connect)
46
+
43
47
  logging.info("Starting Warp Beacon version '%s' ...", __version__)
44
48
 
45
49
  workers_amount = min(32, os.cpu_count() + 4)
@@ -71,7 +75,8 @@ class Bot(object):
71
75
  )
72
76
  self.downloader = warp_beacon.scraper.AsyncDownloader(
73
77
  workers_count=int(os.environ.get("WORKERS_POOL_SIZE", default=workers_amount)),
74
- uploader=self.uploader
78
+ uploader=self.uploader,
79
+ db_connect=db_connect
75
80
  )
76
81
 
77
82
  self.scheduler = IGScheduler(self.downloader)
{warp_beacon-2.2.63.dist-info → warp_beacon-2.2.64.dist-info}/METADATA RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: warp_beacon
3
- Version: 2.2.63
3
+ Version: 2.2.64
4
4
  Summary: Telegram bot for expanding external media links
5
5
  Home-page: https://github.com/sb0y/warp_beacon
6
6
  Author: Andrey Bagrintsev
{warp_beacon-2.2.63.dist-info → warp_beacon-2.2.64.dist-info}/RECORD RENAMED
@@ -3,7 +3,7 @@ lib/systemd/system/warp_beacon.service,sha256=lPmHqLqcI2eIV7nwHS0qcALQrznixqJuww
3
3
  var/warp_beacon/accounts.json,sha256=2QQMS9N_Z-TnfZq4U1vSUe4IKyZGmTFdgo8BoF4hQ7E,1710
4
4
  var/warp_beacon/placeholder.gif,sha256=cE5CGJVaop4Sx21zx6j4AyoHU0ncmvQuS2o6hJfEH88,6064
5
5
  warp_beacon/__init__.py,sha256=_rThNODmz0nDp_n4mWo_HKaNFE5jk1_7cRhHyYaencI,163
6
- warp_beacon/__version__.py,sha256=EqSmisKQYEBahR8JvUza2yMGiptpIUkXrqj8nNwfm_g,24
6
+ warp_beacon/__version__.py,sha256=CUkyGXkge2AkmlYlXGiP1505b2K2nULur53nVaqxZL8,24
7
7
  warp_beacon/warp_beacon.py,sha256=7KEtZDj-pdhtl6m-zFLsSojs1ZR4o7L0xbqtdmYPvfE,342
8
8
  warp_beacon/compress/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
9
9
  warp_beacon/compress/video.py,sha256=_PDMVYCyzLYxHv1uZmmzGcG_8rjaZr7BTXsXTTy_oS4,2846
@@ -19,11 +19,11 @@ warp_beacon/mediainfo/silencer.py,sha256=MgUc9Ibbhjhg9GbJMNfJqrdDkMsQShZkQ1sCwvW
19
19
  warp_beacon/mediainfo/video.py,sha256=UBZrhTN5IDI-aYu6tsJEILo9nFkjHhkldGVFmvV7tEI,2480
20
20
  warp_beacon/scheduler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
21
21
  warp_beacon/scheduler/scheduler.py,sha256=F9jF6myIHOI0TmC0-rBt2Un8gVvuhBW9zL-nEAEj0Bc,2585
22
- warp_beacon/scraper/__init__.py,sha256=8Qm7H49WYe-_6uhX6fJswP3l-brFDBpzhwP4dR17L10,16431
22
+ warp_beacon/scraper/__init__.py,sha256=U6dtvsbAB-Wocvl72NrDrc_WcK4yqB9p4frLx98i5sA,16537
23
23
  warp_beacon/scraper/abstract.py,sha256=aNZ9ypF9B8BjflcIwi-7wEzIqF-XPeF0xvfX9CP_iIw,2708
24
24
  warp_beacon/scraper/account_selector.py,sha256=YUzLxfGGBvdTR9sJVcx9SDkYK4tWo3iRkGiPIg6IRn4,3649
25
25
  warp_beacon/scraper/exceptions.py,sha256=fHywTm2-y2RkgM6hVz12CkDMBgHcEoN2a1TBLaVM2t4,1300
26
- warp_beacon/scraper/fail_handler.py,sha256=6bgQOi6NmE09S-bAbXbD56x_RW9ZsIUS89JJIJWpNQI,1210
26
+ warp_beacon/scraper/fail_handler.py,sha256=n9Fm8n4KF-_2iozpUt2XfT150OdrnCNhOxgACAH0D88,947
27
27
  warp_beacon/scraper/instagram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
28
28
  warp_beacon/scraper/instagram/instagram.py,sha256=m2bAqLu-_HchlF5CVZDlytB9cFGxrN7EEbQW-G1f5u4,12857
29
29
  warp_beacon/scraper/youtube/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -31,16 +31,17 @@ warp_beacon/scraper/youtube/abstract.py,sha256=LqQRNjWY1S44nGvthuinnaRvyAKHfDwnI
31
31
  warp_beacon/scraper/youtube/music.py,sha256=LzpnRVL0sa1S5KDy0C5iMwNPPXZnXhIrEUxFX2ro06c,1471
32
32
  warp_beacon/scraper/youtube/shorts.py,sha256=WMjStmQz24zJwpzGak4-UuaVLo4-0hPtH_rAHQWgKdU,1322
33
33
  warp_beacon/scraper/youtube/youtube.py,sha256=JvN5pVz0jtxCY9FGMl1dIg5Ccr2Kulaoxtym0Vb1QwQ,2224
34
- warp_beacon/storage/__init__.py,sha256=m86MDXBMtZZsTPLI2cD-pM5Xkh2g2NYVvp_dtW8eH0Q,3548
34
+ warp_beacon/storage/__init__.py,sha256=2uvyIR0APIW6gOxwJRvCji7wS2q6I7dghvLyWsRqRxo,3312
35
+ warp_beacon/storage/mongo.py,sha256=2ReAOskgeAlWsMYukH1_-PCjy8yN8eaasgBZ2FqQfu4,528
35
36
  warp_beacon/telegram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
36
- warp_beacon/telegram/bot.py,sha256=MONqsxLMJk3jnLFpn1Nfj5w-FSxPNS2Fx9QfaK2S5lw,15446
37
+ warp_beacon/telegram/bot.py,sha256=XCgZmC_EEjgdogt-X3lGC2LRh2uMb14EaM23EoQIiME,15578
37
38
  warp_beacon/telegram/handlers.py,sha256=171BUNjcaqTM1QnJ9KTBzuJOX1tiDys0hnyuAkA1NiI,7262
38
39
  warp_beacon/telegram/placeholder_message.py,sha256=N1fSL1xbFLF5alhnEvk-xbf3v2A_nLWg6xHtVZTaoiA,6396
39
40
  warp_beacon/telegram/utils.py,sha256=9rNC1BqMZm-kKbzVonBOV8MYfMbcO68Bavq-K3Yj2KA,3115
40
41
  warp_beacon/uploader/__init__.py,sha256=rbfsm_eYcfsBRkiaG0glVgBNtQCEe7cVQTWEIAHz5aw,4763
41
- warp_beacon-2.2.63.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
42
- warp_beacon-2.2.63.dist-info/METADATA,sha256=VSO-YcMRLe3O1fpEgrElKgLpBNbAsvbvCL2JWIjtXY8,21303
43
- warp_beacon-2.2.63.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
44
- warp_beacon-2.2.63.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
45
- warp_beacon-2.2.63.dist-info/top_level.txt,sha256=UE5bi4kftU24IJgHm6BHcXiXAd1gUbD20nWmM6DG8yQ,1004
46
- warp_beacon-2.2.63.dist-info/RECORD,,
42
+ warp_beacon-2.2.64.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
43
+ warp_beacon-2.2.64.dist-info/METADATA,sha256=wVlnUMznHhaxp5krX4qDJU27Q053-Rne2KJ5SoC2PBE,21303
44
+ warp_beacon-2.2.64.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
45
+ warp_beacon-2.2.64.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
46
+ warp_beacon-2.2.64.dist-info/top_level.txt,sha256=q_ThNvU42NteyXqWlEBkaIJTSOK7t-eQ1XW-Fc3V0Eg,1030
47
+ warp_beacon-2.2.64.dist-info/RECORD,,
{warp_beacon-2.2.63.dist-info → warp_beacon-2.2.64.dist-info}/top_level.txt RENAMED
@@ -27,6 +27,7 @@ warp_beacon/scraper/youtube/music
27
27
  warp_beacon/scraper/youtube/shorts
28
28
  warp_beacon/scraper/youtube/youtube
29
29
  warp_beacon/storage
30
+ warp_beacon/storage/mongo
30
31
  warp_beacon/telegram
31
32
  warp_beacon/telegram/bot
32
33
  warp_beacon/telegram/handlers