warp-beacon 2.2.63__py3-none-any.whl → 2.2.65__py3-none-any.whl

warp_beacon/__version__.py

@@ -1,2 +1,2 @@
-__version__ = "2.2.63"
+__version__ = "2.2.65"
 

warp_beacon/scraper/__init__.py

@@ -16,6 +16,7 @@ from warp_beacon.jobs.download_job import DownloadJob
 from warp_beacon.jobs.upload_job import UploadJob
 from warp_beacon.jobs.types import JobType
 from warp_beacon.scraper.account_selector import AccountSelector
+from warp_beacon.storage.mongo import DBClient
 from warp_beacon.scraper.fail_handler import FailHandler
 
 import logging
@@ -33,7 +34,6 @@ class AsyncDownloader(object):
 	manager = None
 	acc_selector = None
 	scheduler = None
-	fail_handler = FailHandler()
 
 	def __init__(self, uploader: AsyncUploader, workers_count: int) -> None:
 		self.manager = multiprocessing.Manager()
@@ -85,6 +85,8 @@ class AsyncDownloader(object):
 
 	def do_work(self, selector: AccountSelector) -> None:
 		logging.info("download worker started")
+		# pymongo is not fork-safe so new connect to DB required
+		fail_handler = FailHandler(DBClient())
 		while self.allow_loop.value == 1:
 			try:
 				job = None
@@ -232,7 +234,7 @@ class AsyncDownloader(object):
 										#self.try_next_account(selector, job, report_error="captcha")
 										#e.job.job_postponed_until = time.time() + 300
 										#self.job_queue.put(e.job)
-										self.fail_handler.store_failed_job(e.job)
+										fail_handler.store_failed_job(e.job)
 									break
 								except (UnknownError, Exception) as e:
 									logging.warning("UnknownError occurred!")
@@ -269,7 +271,7 @@ class AsyncDownloader(object):
 
 							if items:
 								# success
-								for job in self.fail_handler.get_failed_jobs():
+								for job in fail_handler.get_failed_jobs():
 									self.queue_task(job)
 								for item in items:
 									media_info = {"filesize": 0}

warp_beacon/scraper/fail_handler.py

@@ -1,8 +1,8 @@
 import os
 
 import pickle
-from pymongo import MongoClient
 
+from warp_beacon.storage.mongo import DBClient
 from warp_beacon.jobs.download_job import DownloadJob
 
 import logging
@@ -10,17 +10,11 @@ import logging
 class FailHandler(object):
 	client = None
 	db = None
-	def __init__(self) -> None:
-		self.client = MongoClient(
-			host=os.environ.get("MONGODB_HOST", default='127.0.0.1'), 
-			port=int(os.environ.get("MONGODB_PORT", default=27017)),
-			username=os.environ.get("MONGODB_USER", default='root'),
-			password=os.environ.get("MONGODB_PASSWORD", default="changeme"))
-		self.db = self.client.media.failed_jobs
+	def __init__(self, client: DBClient) -> None:
+		self.db = client.client.media.failed_jobs
 
 	def __del__(self) -> None:
-		if self.client:
-			self.client.close()
+		self.client.close()
 
 	def store_failed_job(self, job: DownloadJob) -> int:
 		db_id = -1

warp_beacon/storage/__init__.py

@@ -4,7 +4,7 @@ from enum import Enum
 
 from urllib.parse import urlparse, parse_qs
 
-from pymongo import MongoClient
+from warp_beacon.storage.mongo import DBClient
 
 import logging
 
@@ -19,20 +19,14 @@ VIDEO_STORAGE_DIR = os.environ.get("VIDEO_STORAGE_DIR", default="/var/warp_beaco
 class Storage(object):
 	client = None
 	db = None
-	def __init__(self) -> None:
+	def __init__(self, client: DBClient) -> None:
 		if not os.path.isdir(VIDEO_STORAGE_DIR):
 			os.mkdir(VIDEO_STORAGE_DIR)
-
-		self.client = MongoClient(
-			host=os.environ.get("MONGODB_HOST", default='127.0.0.1'), 
-			port=int(os.environ.get("MONGODB_PORT", default=27017)),
-			username=os.environ.get("MONGODB_USER", default='root'),
-			password=os.environ.get("MONGODB_PASSWORD", default="changeme"))
-		self.db = self.client.media.media
+		self.client = client
+		self.db = self.client.client.media.media
 
 	def __del__(self) -> None:
-		if self.client:
-			self.client.close()
+		self.client.close()
 
 	@staticmethod
 	def compute_uniq(url: str) -> str:

warp_beacon/storage/mongo.py

@@ -0,0 +1,22 @@
+import os
+
+from pymongo import MongoClient
+
+class DBClient(object):
+	client = None
+
+	def __init__(self) -> None:
+		self.client = MongoClient(
+			host=os.environ.get("MONGODB_HOST", default='127.0.0.1'), 
+			port=int(os.environ.get("MONGODB_PORT", default=27017)),
+			username=os.environ.get("MONGODB_USER", default='root'),
+			password=os.environ.get("MONGODB_PASSWORD", default="changeme")
+		)
+
+	def __del__(self) -> None:
+		self.close()
+
+	def close(self) -> None:
+		if self.client:
+			self.client.close()
+			self.client = None

warp_beacon/telegram/bot.py

@@ -15,6 +15,7 @@ import warp_beacon
 from warp_beacon.__version__ import __version__
 from warp_beacon.telegram.handlers import Handlers
 from warp_beacon.telegram.placeholder_message import PlaceholderMessage
+from warp_beacon.storage.mongo import DBClient
 from warp_beacon.storage import Storage
 from warp_beacon.uploader import AsyncUploader
 from warp_beacon.jobs.upload_job import UploadJob
@@ -25,7 +26,7 @@ from warp_beacon.scheduler.scheduler import IGScheduler
 import logging
 
 class Bot(object):
-	storage = Storage()
+	storage = None
 	uploader = None
 	downloader = None
 	allow_loop = True
@@ -40,6 +41,9 @@ class Bot(object):
 			format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", level=logging.INFO
 		)
 
+		db_connect = DBClient()
+		self.storage = Storage(db_connect)
+
 		logging.info("Starting Warp Beacon version '%s' ...", __version__)
 
 		workers_amount = min(32, os.cpu_count() + 4)

{warp_beacon-2.2.63.dist-info → warp_beacon-2.2.65.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: warp_beacon
-Version: 2.2.63
+Version: 2.2.65
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

{warp_beacon-2.2.63.dist-info → warp_beacon-2.2.65.dist-info}/RECORD

@@ -3,7 +3,7 @@ lib/systemd/system/warp_beacon.service,sha256=lPmHqLqcI2eIV7nwHS0qcALQrznixqJuww
 var/warp_beacon/accounts.json,sha256=2QQMS9N_Z-TnfZq4U1vSUe4IKyZGmTFdgo8BoF4hQ7E,1710
 var/warp_beacon/placeholder.gif,sha256=cE5CGJVaop4Sx21zx6j4AyoHU0ncmvQuS2o6hJfEH88,6064
 warp_beacon/__init__.py,sha256=_rThNODmz0nDp_n4mWo_HKaNFE5jk1_7cRhHyYaencI,163
-warp_beacon/__version__.py,sha256=EqSmisKQYEBahR8JvUza2yMGiptpIUkXrqj8nNwfm_g,24
+warp_beacon/__version__.py,sha256=yqYvzkYPQ6JzDm_sC5FOlSpY_KmZjlp5n52olmpY4KI,24
 warp_beacon/warp_beacon.py,sha256=7KEtZDj-pdhtl6m-zFLsSojs1ZR4o7L0xbqtdmYPvfE,342
 warp_beacon/compress/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/compress/video.py,sha256=_PDMVYCyzLYxHv1uZmmzGcG_8rjaZr7BTXsXTTy_oS4,2846
@@ -19,11 +19,11 @@ warp_beacon/mediainfo/silencer.py,sha256=MgUc9Ibbhjhg9GbJMNfJqrdDkMsQShZkQ1sCwvW
 warp_beacon/mediainfo/video.py,sha256=UBZrhTN5IDI-aYu6tsJEILo9nFkjHhkldGVFmvV7tEI,2480
 warp_beacon/scheduler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scheduler/scheduler.py,sha256=F9jF6myIHOI0TmC0-rBt2Un8gVvuhBW9zL-nEAEj0Bc,2585
-warp_beacon/scraper/__init__.py,sha256=8Qm7H49WYe-_6uhX6fJswP3l-brFDBpzhwP4dR17L10,16431
+warp_beacon/scraper/__init__.py,sha256=i3yda30elPbnG_JuD1DSXZLdDc2C30UqqbUbEzXlhnY,16538
 warp_beacon/scraper/abstract.py,sha256=aNZ9ypF9B8BjflcIwi-7wEzIqF-XPeF0xvfX9CP_iIw,2708
 warp_beacon/scraper/account_selector.py,sha256=YUzLxfGGBvdTR9sJVcx9SDkYK4tWo3iRkGiPIg6IRn4,3649
 warp_beacon/scraper/exceptions.py,sha256=fHywTm2-y2RkgM6hVz12CkDMBgHcEoN2a1TBLaVM2t4,1300
-warp_beacon/scraper/fail_handler.py,sha256=6bgQOi6NmE09S-bAbXbD56x_RW9ZsIUS89JJIJWpNQI,1210
+warp_beacon/scraper/fail_handler.py,sha256=n9Fm8n4KF-_2iozpUt2XfT150OdrnCNhOxgACAH0D88,947
 warp_beacon/scraper/instagram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scraper/instagram/instagram.py,sha256=m2bAqLu-_HchlF5CVZDlytB9cFGxrN7EEbQW-G1f5u4,12857
 warp_beacon/scraper/youtube/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -31,16 +31,17 @@ warp_beacon/scraper/youtube/abstract.py,sha256=LqQRNjWY1S44nGvthuinnaRvyAKHfDwnI
 warp_beacon/scraper/youtube/music.py,sha256=LzpnRVL0sa1S5KDy0C5iMwNPPXZnXhIrEUxFX2ro06c,1471
 warp_beacon/scraper/youtube/shorts.py,sha256=WMjStmQz24zJwpzGak4-UuaVLo4-0hPtH_rAHQWgKdU,1322
 warp_beacon/scraper/youtube/youtube.py,sha256=JvN5pVz0jtxCY9FGMl1dIg5Ccr2Kulaoxtym0Vb1QwQ,2224
-warp_beacon/storage/__init__.py,sha256=m86MDXBMtZZsTPLI2cD-pM5Xkh2g2NYVvp_dtW8eH0Q,3548
+warp_beacon/storage/__init__.py,sha256=2uvyIR0APIW6gOxwJRvCji7wS2q6I7dghvLyWsRqRxo,3312
+warp_beacon/storage/mongo.py,sha256=2ReAOskgeAlWsMYukH1_-PCjy8yN8eaasgBZ2FqQfu4,528
 warp_beacon/telegram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-warp_beacon/telegram/bot.py,sha256=MONqsxLMJk3jnLFpn1Nfj5w-FSxPNS2Fx9QfaK2S5lw,15446
+warp_beacon/telegram/bot.py,sha256=tA9g6UT0JeV4P6XSrfDXvnTvNen3c75GtDuvbufAs54,15552
 warp_beacon/telegram/handlers.py,sha256=171BUNjcaqTM1QnJ9KTBzuJOX1tiDys0hnyuAkA1NiI,7262
 warp_beacon/telegram/placeholder_message.py,sha256=N1fSL1xbFLF5alhnEvk-xbf3v2A_nLWg6xHtVZTaoiA,6396
 warp_beacon/telegram/utils.py,sha256=9rNC1BqMZm-kKbzVonBOV8MYfMbcO68Bavq-K3Yj2KA,3115
 warp_beacon/uploader/__init__.py,sha256=rbfsm_eYcfsBRkiaG0glVgBNtQCEe7cVQTWEIAHz5aw,4763
-warp_beacon-2.2.63.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-warp_beacon-2.2.63.dist-info/METADATA,sha256=VSO-YcMRLe3O1fpEgrElKgLpBNbAsvbvCL2JWIjtXY8,21303
-warp_beacon-2.2.63.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-warp_beacon-2.2.63.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
-warp_beacon-2.2.63.dist-info/top_level.txt,sha256=UE5bi4kftU24IJgHm6BHcXiXAd1gUbD20nWmM6DG8yQ,1004
-warp_beacon-2.2.63.dist-info/RECORD,,
+warp_beacon-2.2.65.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+warp_beacon-2.2.65.dist-info/METADATA,sha256=jEA0IBn_l4Upqq79Vu7HOTCmVJo1wvlPz6A32zH7xM8,21303
+warp_beacon-2.2.65.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+warp_beacon-2.2.65.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
+warp_beacon-2.2.65.dist-info/top_level.txt,sha256=q_ThNvU42NteyXqWlEBkaIJTSOK7t-eQ1XW-Fc3V0Eg,1030
+warp_beacon-2.2.65.dist-info/RECORD,,

{warp_beacon-2.2.63.dist-info → warp_beacon-2.2.65.dist-info}/top_level.txt

@@ -27,6 +27,7 @@ warp_beacon/scraper/youtube/music
 warp_beacon/scraper/youtube/shorts
 warp_beacon/scraper/youtube/youtube
 warp_beacon/storage
+warp_beacon/storage/mongo
 warp_beacon/telegram
 warp_beacon/telegram/bot
 warp_beacon/telegram/handlers