warp-beacon 2.2.62__py3-none-any.whl → 2.2.63__py3-none-any.whl

warp_beacon/__version__.py

@@ -1,2 +1,2 @@
-__version__ = "2.2.62"
+__version__ = "2.2.63"
 

warp_beacon/scraper/__init__.py

@@ -16,6 +16,7 @@ from warp_beacon.jobs.download_job import DownloadJob
 from warp_beacon.jobs.upload_job import UploadJob
 from warp_beacon.jobs.types import JobType
 from warp_beacon.scraper.account_selector import AccountSelector
+from warp_beacon.scraper.fail_handler import FailHandler
 
 import logging
 
@@ -32,6 +33,7 @@ class AsyncDownloader(object):
 	manager = None
 	acc_selector = None
 	scheduler = None
+	fail_handler = FailHandler()
 
 	def __init__(self, uploader: AsyncUploader, workers_count: int) -> None:
 		self.manager = multiprocessing.Manager()
@@ -98,7 +100,7 @@ class AsyncDownloader(object):
 							continue
 						if not job.in_process:
 							if job.job_postponed_until > 0:
-								if job.job_postponed_until - time.time() > 0:
+								if (job.job_postponed_until - time.time()) > 0:
 									logging.warning("Job '%s' is postponed, rescheduling", job.url)
 									time.sleep(2)
 									self.job_queue.put(job)
@@ -169,7 +171,7 @@ class AsyncDownloader(object):
 									logging.warning("Telegram limits exceeded :(")
 									logging.exception(e)
 									self.send_message_to_admin(
-										f"Task <code>{job.job_id}</code> failed. URL: '{job.url}'. Reason: '<b>FileTooBi</b>'."
+										f"Task <code>{job.job_id}</code> failed. URL: '{job.url}'. Reason: '<b>FileTooBig</b>'."
 									)
 									self.uploader.queue_task(job.to_upload_job(
 										job_failed=True,
@@ -222,9 +224,15 @@ class AsyncDownloader(object):
 										job_failed_msg="All bot accounts failed to download content. Bot administrator noticed about the issue.")
 									)
 									if e.job.job_origin == Origin.INSTAGRAM:
-										self.try_next_account(selector, job, report_error="captcha")
-										e.job.job_timeout = time.time() + 300
-										self.job_queue.put(e.job)
+										logging.info("Handling captcha postpone")
+										self.uploader.queue_task(job.to_upload_job(
+											job_warning=True,
+											job_warning_msg="Bot is experiencing issues, video delivery may be delayed.")
+										)
+										#self.try_next_account(selector, job, report_error="captcha")
+										#e.job.job_postponed_until = time.time() + 300
+										#self.job_queue.put(e.job)
+										self.fail_handler.store_failed_job(e.job)
 									break
 								except (UnknownError, Exception) as e:
 									logging.warning("UnknownError occurred!")
@@ -260,6 +268,9 @@ class AsyncDownloader(object):
 									break
 
 							if items:
+								# success
+								for job in self.fail_handler.get_failed_jobs():
+									self.queue_task(job)
 								for item in items:
 									media_info = {"filesize": 0}
 									if item["media_type"] == JobType.VIDEO:

warp_beacon/scraper/fail_handler.py

@@ -0,0 +1,48 @@
+import os
+
+import pickle
+from pymongo import MongoClient
+
+from warp_beacon.jobs.download_job import DownloadJob
+
+import logging
+
+class FailHandler(object):
+	client = None
+	db = None
+	def __init__(self) -> None:
+		self.client = MongoClient(
+			host=os.environ.get("MONGODB_HOST", default='127.0.0.1'), 
+			port=int(os.environ.get("MONGODB_PORT", default=27017)),
+			username=os.environ.get("MONGODB_USER", default='root'),
+			password=os.environ.get("MONGODB_PASSWORD", default="changeme"))
+		self.db = self.client.media.failed_jobs
+
+	def __del__(self) -> None:
+		if self.client:
+			self.client.close()
+
+	def store_failed_job(self, job: DownloadJob) -> int:
+		db_id = -1
+		try:
+			job_serilized = pickle.dumps(job)
+			db_id = self.db.insert_one(
+			{
+				"job_data": job_serilized
+			}).inserted_id
+		except Exception as e:
+			logging.error("Failed to store job as failed!")
+			logging.exception(e)
+		return db_id
+	
+	def get_failed_jobs(self) -> list:
+		ret = []
+		try:
+			cursor = self.db.find()
+			for document in cursor:
+				ret.append(pickle.loads(document["job_data"]))
+			self.db.delete_many({})
+		except Exception as e:
+			logging.error("Failed to get failed jobs!")
+			logging.exception(e)
+		return ret

{warp_beacon-2.2.62.dist-info → warp_beacon-2.2.63.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: warp_beacon
-Version: 2.2.62
+Version: 2.2.63
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

{warp_beacon-2.2.62.dist-info → warp_beacon-2.2.63.dist-info}/RECORD

@@ -3,7 +3,7 @@ lib/systemd/system/warp_beacon.service,sha256=lPmHqLqcI2eIV7nwHS0qcALQrznixqJuww
 var/warp_beacon/accounts.json,sha256=2QQMS9N_Z-TnfZq4U1vSUe4IKyZGmTFdgo8BoF4hQ7E,1710
 var/warp_beacon/placeholder.gif,sha256=cE5CGJVaop4Sx21zx6j4AyoHU0ncmvQuS2o6hJfEH88,6064
 warp_beacon/__init__.py,sha256=_rThNODmz0nDp_n4mWo_HKaNFE5jk1_7cRhHyYaencI,163
-warp_beacon/__version__.py,sha256=7RR_ANidaW8-uVYjchjCx2y2CmN4GN0DsLSWCsX_jh0,24
+warp_beacon/__version__.py,sha256=EqSmisKQYEBahR8JvUza2yMGiptpIUkXrqj8nNwfm_g,24
 warp_beacon/warp_beacon.py,sha256=7KEtZDj-pdhtl6m-zFLsSojs1ZR4o7L0xbqtdmYPvfE,342
 warp_beacon/compress/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/compress/video.py,sha256=_PDMVYCyzLYxHv1uZmmzGcG_8rjaZr7BTXsXTTy_oS4,2846
@@ -19,10 +19,11 @@ warp_beacon/mediainfo/silencer.py,sha256=MgUc9Ibbhjhg9GbJMNfJqrdDkMsQShZkQ1sCwvW
 warp_beacon/mediainfo/video.py,sha256=UBZrhTN5IDI-aYu6tsJEILo9nFkjHhkldGVFmvV7tEI,2480
 warp_beacon/scheduler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scheduler/scheduler.py,sha256=F9jF6myIHOI0TmC0-rBt2Un8gVvuhBW9zL-nEAEj0Bc,2585
-warp_beacon/scraper/__init__.py,sha256=Nc-wdqu1KLLQkf3kkhl7jXzDWiv6vzKPudJdwL9sb24,15938
+warp_beacon/scraper/__init__.py,sha256=8Qm7H49WYe-_6uhX6fJswP3l-brFDBpzhwP4dR17L10,16431
 warp_beacon/scraper/abstract.py,sha256=aNZ9ypF9B8BjflcIwi-7wEzIqF-XPeF0xvfX9CP_iIw,2708
 warp_beacon/scraper/account_selector.py,sha256=YUzLxfGGBvdTR9sJVcx9SDkYK4tWo3iRkGiPIg6IRn4,3649
 warp_beacon/scraper/exceptions.py,sha256=fHywTm2-y2RkgM6hVz12CkDMBgHcEoN2a1TBLaVM2t4,1300
+warp_beacon/scraper/fail_handler.py,sha256=6bgQOi6NmE09S-bAbXbD56x_RW9ZsIUS89JJIJWpNQI,1210
 warp_beacon/scraper/instagram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scraper/instagram/instagram.py,sha256=m2bAqLu-_HchlF5CVZDlytB9cFGxrN7EEbQW-G1f5u4,12857
 warp_beacon/scraper/youtube/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -37,9 +38,9 @@ warp_beacon/telegram/handlers.py,sha256=171BUNjcaqTM1QnJ9KTBzuJOX1tiDys0hnyuAkA1
 warp_beacon/telegram/placeholder_message.py,sha256=N1fSL1xbFLF5alhnEvk-xbf3v2A_nLWg6xHtVZTaoiA,6396
 warp_beacon/telegram/utils.py,sha256=9rNC1BqMZm-kKbzVonBOV8MYfMbcO68Bavq-K3Yj2KA,3115
 warp_beacon/uploader/__init__.py,sha256=rbfsm_eYcfsBRkiaG0glVgBNtQCEe7cVQTWEIAHz5aw,4763
-warp_beacon-2.2.62.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-warp_beacon-2.2.62.dist-info/METADATA,sha256=VxkAOnspfZmgfoVkEWS8L5D9sYr4RBjjDNGAd0NkLsU,21303
-warp_beacon-2.2.62.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-warp_beacon-2.2.62.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
-warp_beacon-2.2.62.dist-info/top_level.txt,sha256=ALb_Ft_eG-OY4_m0TWUifNUOZsrx483L-zw7G7vqXoc,971
-warp_beacon-2.2.62.dist-info/RECORD,,
+warp_beacon-2.2.63.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+warp_beacon-2.2.63.dist-info/METADATA,sha256=VSO-YcMRLe3O1fpEgrElKgLpBNbAsvbvCL2JWIjtXY8,21303
+warp_beacon-2.2.63.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+warp_beacon-2.2.63.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
+warp_beacon-2.2.63.dist-info/top_level.txt,sha256=UE5bi4kftU24IJgHm6BHcXiXAd1gUbD20nWmM6DG8yQ,1004
+warp_beacon-2.2.63.dist-info/RECORD,,

{warp_beacon-2.2.62.dist-info → warp_beacon-2.2.63.dist-info}/top_level.txt

@@ -17,6 +17,7 @@ warp_beacon/scraper
 warp_beacon/scraper/abstract
 warp_beacon/scraper/account_selector
 warp_beacon/scraper/exceptions
+warp_beacon/scraper/fail_handler
 warp_beacon/scraper/instagram
 warp_beacon/scraper/instagram/instagram
 warp_beacon/scraper/types