warp-beacon 2.2.62__py3-none-any.whl → 2.2.64__py3-none-any.whl

warp_beacon/__version__.py

@@ -1,2 +1,2 @@
-__version__ = "2.2.62"
+__version__ = "2.2.64"
 

warp_beacon/scraper/__init__.py

@@ -16,6 +16,8 @@ from warp_beacon.jobs.download_job import DownloadJob
 from warp_beacon.jobs.upload_job import UploadJob
 from warp_beacon.jobs.types import JobType
 from warp_beacon.scraper.account_selector import AccountSelector
+from warp_beacon.storage.mongo import DBClient
+from warp_beacon.scraper.fail_handler import FailHandler
 
 import logging
 
@@ -32,8 +34,10 @@ class AsyncDownloader(object):
 	manager = None
 	acc_selector = None
 	scheduler = None
+	fail_handler = None
 
-	def __init__(self, uploader: AsyncUploader, workers_count: int) -> None:
+	def __init__(self, uploader: AsyncUploader, db_connect: DBClient, workers_count: int) -> None:
+		self.fail_handler = FailHandler(db_connect)
 		self.manager = multiprocessing.Manager()
 		self.allow_loop = self.manager.Value('i', 1)
 		self.acc_selector = AccountSelector(self.manager, ACC_FILE)
@@ -98,7 +102,7 @@ class AsyncDownloader(object):
 							continue
 						if not job.in_process:
 							if job.job_postponed_until > 0:
-								if job.job_postponed_until - time.time() > 0:
+								if (job.job_postponed_until - time.time()) > 0:
 									logging.warning("Job '%s' is postponed, rescheduling", job.url)
 									time.sleep(2)
 									self.job_queue.put(job)
@@ -169,7 +173,7 @@ class AsyncDownloader(object):
 									logging.warning("Telegram limits exceeded :(")
 									logging.exception(e)
 									self.send_message_to_admin(
-										f"Task <code>{job.job_id}</code> failed. URL: '{job.url}'. Reason: '<b>FileTooBi</b>'."
+										f"Task <code>{job.job_id}</code> failed. URL: '{job.url}'. Reason: '<b>FileTooBig</b>'."
 									)
 									self.uploader.queue_task(job.to_upload_job(
 										job_failed=True,
@@ -222,9 +226,15 @@ class AsyncDownloader(object):
 										job_failed_msg="All bot accounts failed to download content. Bot administrator noticed about the issue.")
 									)
 									if e.job.job_origin == Origin.INSTAGRAM:
-										self.try_next_account(selector, job, report_error="captcha")
-										e.job.job_timeout = time.time() + 300
-										self.job_queue.put(e.job)
+										logging.info("Handling captcha postpone")
+										self.uploader.queue_task(job.to_upload_job(
+											job_warning=True,
+											job_warning_msg="Bot is experiencing issues, video delivery may be delayed.")
+										)
+										#self.try_next_account(selector, job, report_error="captcha")
+										#e.job.job_postponed_until = time.time() + 300
+										#self.job_queue.put(e.job)
+										self.fail_handler.store_failed_job(e.job)
 									break
 								except (UnknownError, Exception) as e:
 									logging.warning("UnknownError occurred!")
@@ -260,6 +270,9 @@ class AsyncDownloader(object):
 									break
 
 							if items:
+								# success
+								for job in self.fail_handler.get_failed_jobs():
+									self.queue_task(job)
 								for item in items:
 									media_info = {"filesize": 0}
 									if item["media_type"] == JobType.VIDEO:

warp_beacon/scraper/fail_handler.py

@@ -0,0 +1,42 @@
+import os
+
+import pickle
+
+from warp_beacon.storage.mongo import DBClient
+from warp_beacon.jobs.download_job import DownloadJob
+
+import logging
+
+class FailHandler(object):
+	client = None
+	db = None
+	def __init__(self, client: DBClient) -> None:
+		self.db = client.client.media.failed_jobs
+
+	def __del__(self) -> None:
+		self.client.close()
+
+	def store_failed_job(self, job: DownloadJob) -> int:
+		db_id = -1
+		try:
+			job_serilized = pickle.dumps(job)
+			db_id = self.db.insert_one(
+			{
+				"job_data": job_serilized
+			}).inserted_id
+		except Exception as e:
+			logging.error("Failed to store job as failed!")
+			logging.exception(e)
+		return db_id
+	
+	def get_failed_jobs(self) -> list:
+		ret = []
+		try:
+			cursor = self.db.find()
+			for document in cursor:
+				ret.append(pickle.loads(document["job_data"]))
+			self.db.delete_many({})
+		except Exception as e:
+			logging.error("Failed to get failed jobs!")
+			logging.exception(e)
+		return ret

warp_beacon/storage/__init__.py

@@ -4,7 +4,7 @@ from enum import Enum
 
 from urllib.parse import urlparse, parse_qs
 
-from pymongo import MongoClient
+from warp_beacon.storage.mongo import DBClient
 
 import logging
 
@@ -19,20 +19,14 @@ VIDEO_STORAGE_DIR = os.environ.get("VIDEO_STORAGE_DIR", default="/var/warp_beaco
 class Storage(object):
 	client = None
 	db = None
-	def __init__(self) -> None:
+	def __init__(self, client: DBClient) -> None:
 		if not os.path.isdir(VIDEO_STORAGE_DIR):
 			os.mkdir(VIDEO_STORAGE_DIR)
-
-		self.client = MongoClient(
-			host=os.environ.get("MONGODB_HOST", default='127.0.0.1'), 
-			port=int(os.environ.get("MONGODB_PORT", default=27017)),
-			username=os.environ.get("MONGODB_USER", default='root'),
-			password=os.environ.get("MONGODB_PASSWORD", default="changeme"))
-		self.db = self.client.media.media
+		self.client = client
+		self.db = self.client.client.media.media
 
 	def __del__(self) -> None:
-		if self.client:
-			self.client.close()
+		self.client.close()
 
 	@staticmethod
 	def compute_uniq(url: str) -> str:

warp_beacon/storage/mongo.py

@@ -0,0 +1,22 @@
+import os
+
+from pymongo import MongoClient
+
+class DBClient(object):
+	client = None
+
+	def __init__(self) -> None:
+		self.client = MongoClient(
+			host=os.environ.get("MONGODB_HOST", default='127.0.0.1'), 
+			port=int(os.environ.get("MONGODB_PORT", default=27017)),
+			username=os.environ.get("MONGODB_USER", default='root'),
+			password=os.environ.get("MONGODB_PASSWORD", default="changeme")
+		)
+
+	def __del__(self) -> None:
+		self.close()
+
+	def close(self) -> None:
+		if self.client:
+			self.client.close()
+			self.client = None

warp_beacon/telegram/bot.py

@@ -15,6 +15,7 @@ import warp_beacon
 from warp_beacon.__version__ import __version__
 from warp_beacon.telegram.handlers import Handlers
 from warp_beacon.telegram.placeholder_message import PlaceholderMessage
+from warp_beacon.storage.mongo import DBClient
 from warp_beacon.storage import Storage
 from warp_beacon.uploader import AsyncUploader
 from warp_beacon.jobs.upload_job import UploadJob
@@ -25,7 +26,7 @@ from warp_beacon.scheduler.scheduler import IGScheduler
 import logging
 
 class Bot(object):
-	storage = Storage()
+	storage = None
 	uploader = None
 	downloader = None
 	allow_loop = True
@@ -40,6 +41,9 @@ class Bot(object):
 			format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", level=logging.INFO
 		)
 
+		db_connect = DBClient()
+		self.storage = Storage(db_connect)
+
 		logging.info("Starting Warp Beacon version '%s' ...", __version__)
 
 		workers_amount = min(32, os.cpu_count() + 4)
@@ -71,7 +75,8 @@ class Bot(object):
 		)
 		self.downloader = warp_beacon.scraper.AsyncDownloader(
 			workers_count=int(os.environ.get("WORKERS_POOL_SIZE", default=workers_amount)),
-			uploader=self.uploader
+			uploader=self.uploader,
+			db_connect=db_connect
 		)
 
 		self.scheduler = IGScheduler(self.downloader)

{warp_beacon-2.2.62.dist-info → warp_beacon-2.2.64.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: warp_beacon
-Version: 2.2.62
+Version: 2.2.64
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

{warp_beacon-2.2.62.dist-info → warp_beacon-2.2.64.dist-info}/RECORD

@@ -3,7 +3,7 @@ lib/systemd/system/warp_beacon.service,sha256=lPmHqLqcI2eIV7nwHS0qcALQrznixqJuww
 var/warp_beacon/accounts.json,sha256=2QQMS9N_Z-TnfZq4U1vSUe4IKyZGmTFdgo8BoF4hQ7E,1710
 var/warp_beacon/placeholder.gif,sha256=cE5CGJVaop4Sx21zx6j4AyoHU0ncmvQuS2o6hJfEH88,6064
 warp_beacon/__init__.py,sha256=_rThNODmz0nDp_n4mWo_HKaNFE5jk1_7cRhHyYaencI,163
-warp_beacon/__version__.py,sha256=7RR_ANidaW8-uVYjchjCx2y2CmN4GN0DsLSWCsX_jh0,24
+warp_beacon/__version__.py,sha256=CUkyGXkge2AkmlYlXGiP1505b2K2nULur53nVaqxZL8,24
 warp_beacon/warp_beacon.py,sha256=7KEtZDj-pdhtl6m-zFLsSojs1ZR4o7L0xbqtdmYPvfE,342
 warp_beacon/compress/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/compress/video.py,sha256=_PDMVYCyzLYxHv1uZmmzGcG_8rjaZr7BTXsXTTy_oS4,2846
@@ -19,10 +19,11 @@ warp_beacon/mediainfo/silencer.py,sha256=MgUc9Ibbhjhg9GbJMNfJqrdDkMsQShZkQ1sCwvW
 warp_beacon/mediainfo/video.py,sha256=UBZrhTN5IDI-aYu6tsJEILo9nFkjHhkldGVFmvV7tEI,2480
 warp_beacon/scheduler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scheduler/scheduler.py,sha256=F9jF6myIHOI0TmC0-rBt2Un8gVvuhBW9zL-nEAEj0Bc,2585
-warp_beacon/scraper/__init__.py,sha256=Nc-wdqu1KLLQkf3kkhl7jXzDWiv6vzKPudJdwL9sb24,15938
+warp_beacon/scraper/__init__.py,sha256=U6dtvsbAB-Wocvl72NrDrc_WcK4yqB9p4frLx98i5sA,16537
 warp_beacon/scraper/abstract.py,sha256=aNZ9ypF9B8BjflcIwi-7wEzIqF-XPeF0xvfX9CP_iIw,2708
 warp_beacon/scraper/account_selector.py,sha256=YUzLxfGGBvdTR9sJVcx9SDkYK4tWo3iRkGiPIg6IRn4,3649
 warp_beacon/scraper/exceptions.py,sha256=fHywTm2-y2RkgM6hVz12CkDMBgHcEoN2a1TBLaVM2t4,1300
+warp_beacon/scraper/fail_handler.py,sha256=n9Fm8n4KF-_2iozpUt2XfT150OdrnCNhOxgACAH0D88,947
 warp_beacon/scraper/instagram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scraper/instagram/instagram.py,sha256=m2bAqLu-_HchlF5CVZDlytB9cFGxrN7EEbQW-G1f5u4,12857
 warp_beacon/scraper/youtube/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -30,16 +31,17 @@ warp_beacon/scraper/youtube/abstract.py,sha256=LqQRNjWY1S44nGvthuinnaRvyAKHfDwnI
 warp_beacon/scraper/youtube/music.py,sha256=LzpnRVL0sa1S5KDy0C5iMwNPPXZnXhIrEUxFX2ro06c,1471
 warp_beacon/scraper/youtube/shorts.py,sha256=WMjStmQz24zJwpzGak4-UuaVLo4-0hPtH_rAHQWgKdU,1322
 warp_beacon/scraper/youtube/youtube.py,sha256=JvN5pVz0jtxCY9FGMl1dIg5Ccr2Kulaoxtym0Vb1QwQ,2224
-warp_beacon/storage/__init__.py,sha256=m86MDXBMtZZsTPLI2cD-pM5Xkh2g2NYVvp_dtW8eH0Q,3548
+warp_beacon/storage/__init__.py,sha256=2uvyIR0APIW6gOxwJRvCji7wS2q6I7dghvLyWsRqRxo,3312
+warp_beacon/storage/mongo.py,sha256=2ReAOskgeAlWsMYukH1_-PCjy8yN8eaasgBZ2FqQfu4,528
 warp_beacon/telegram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-warp_beacon/telegram/bot.py,sha256=MONqsxLMJk3jnLFpn1Nfj5w-FSxPNS2Fx9QfaK2S5lw,15446
+warp_beacon/telegram/bot.py,sha256=XCgZmC_EEjgdogt-X3lGC2LRh2uMb14EaM23EoQIiME,15578
 warp_beacon/telegram/handlers.py,sha256=171BUNjcaqTM1QnJ9KTBzuJOX1tiDys0hnyuAkA1NiI,7262
 warp_beacon/telegram/placeholder_message.py,sha256=N1fSL1xbFLF5alhnEvk-xbf3v2A_nLWg6xHtVZTaoiA,6396
 warp_beacon/telegram/utils.py,sha256=9rNC1BqMZm-kKbzVonBOV8MYfMbcO68Bavq-K3Yj2KA,3115
 warp_beacon/uploader/__init__.py,sha256=rbfsm_eYcfsBRkiaG0glVgBNtQCEe7cVQTWEIAHz5aw,4763
-warp_beacon-2.2.62.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-warp_beacon-2.2.62.dist-info/METADATA,sha256=VxkAOnspfZmgfoVkEWS8L5D9sYr4RBjjDNGAd0NkLsU,21303
-warp_beacon-2.2.62.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-warp_beacon-2.2.62.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
-warp_beacon-2.2.62.dist-info/top_level.txt,sha256=ALb_Ft_eG-OY4_m0TWUifNUOZsrx483L-zw7G7vqXoc,971
-warp_beacon-2.2.62.dist-info/RECORD,,
+warp_beacon-2.2.64.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+warp_beacon-2.2.64.dist-info/METADATA,sha256=wVlnUMznHhaxp5krX4qDJU27Q053-Rne2KJ5SoC2PBE,21303
+warp_beacon-2.2.64.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+warp_beacon-2.2.64.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
+warp_beacon-2.2.64.dist-info/top_level.txt,sha256=q_ThNvU42NteyXqWlEBkaIJTSOK7t-eQ1XW-Fc3V0Eg,1030
+warp_beacon-2.2.64.dist-info/RECORD,,

{warp_beacon-2.2.62.dist-info → warp_beacon-2.2.64.dist-info}/top_level.txt

@@ -17,6 +17,7 @@ warp_beacon/scraper
 warp_beacon/scraper/abstract
 warp_beacon/scraper/account_selector
 warp_beacon/scraper/exceptions
+warp_beacon/scraper/fail_handler
 warp_beacon/scraper/instagram
 warp_beacon/scraper/instagram/instagram
 warp_beacon/scraper/types
@@ -26,6 +27,7 @@ warp_beacon/scraper/youtube/music
 warp_beacon/scraper/youtube/shorts
 warp_beacon/scraper/youtube/youtube
 warp_beacon/storage
+warp_beacon/storage/mongo
 warp_beacon/telegram
 warp_beacon/telegram/bot
 warp_beacon/telegram/handlers