warp-beacon 2.8.14__py3-none-any.whl → 2.8.16__py3-none-any.whl

warp_beacon/__version__.py

@@ -1,2 +1,2 @@
-__version__ = "2.8.14"
+__version__ = "2.8.16"
 

warp_beacon/scraper/__init__.py

@@ -318,7 +318,7 @@ class AsyncDownloader(object):
 							if items:
 								# success
 								for job in fail_handler.get_failed_jobs():
-									self.queue_task(job)
+									self.queue_task(job["job"])
 								# media info processing
 								for item in items:
 									media_info = {"filesize": 0}

warp_beacon/scraper/fail_handler.py

@@ -16,12 +16,15 @@ class FailHandler(object):
 		self.client.close()
 
 	def store_failed_job(self, job: DownloadJob) -> int:
-		db_id = -1
+		db_id = ""
 		try:
 			job_serilized = pickle.dumps(job)
 			db_id = self.db.insert_one(
 			{
-				"job_data": job_serilized
+				"job_data": job_serilized,
+				"uniq_id": job.uniq_id,
+				"message_id": job.message_id,
+				"chat_id": job.chat_id
 			}).inserted_id
 		except Exception as e:
 			logging.error("Failed to store job as failed!")
@@ -33,10 +36,26 @@ class FailHandler(object):
 		try:
 			cursor = self.db.find()
 			for document in cursor:
-				ret.append(pickle.loads(document["job_data"]))
+				ret.append({
+					"_id": document["_id"],
+					"job": pickle.loads(document["job_data"]),
+					"uniq_id": document.get("uniq_id"),
+					"message_id": document.get("message_id"),
+					"chat_id": document.get("chat_id")
+				})
 			if clean:
 				self.db.delete_many({})
 		except Exception as e:
 			logging.error("Failed to get failed jobs!")
 			logging.exception(e)
 		return ret
+	
+	def remove_failed_job(self, uniq_id: str) -> bool:
+		try:
+			result = self.db.delete_one({"uniq_id": uniq_id})
+			if result.deleted_count > 0:
+				return True
+		except Exception as e:
+			logging.error("Failed to remove failed job!", exc_info=e)
+
+		return False

warp_beacon/telegram/bot.py

@@ -69,9 +69,11 @@ class Bot(object):
 		)
 
 		self.editor = EditMessage(self.client)
+		self.handlers = Handlers(self)
 
 		self.uploader = AsyncUploader(
 			storage=self.storage,
+			upload_wrapper=self.handlers.upload_wrapper,
 			admin_message_callback=self.send_text_to_admin,
 			request_yt_auth_callback=self.request_yt_auth,
 			pool_size=int(os.environ.get("UPLOAD_POOL_SIZE", default=workers_amount)),
@@ -85,7 +87,6 @@ class Bot(object):
 		)
 
 		self.scheduler = IGScheduler(self.downloader)
-		self.handlers = Handlers(self)
 
 		self.client.add_handler(MessageHandler(self.handlers.start, filters.command("start")))
 		self.client.add_handler(MessageHandler(self.handlers.help, filters.command("help")))

warp_beacon/telegram/handlers.py

@@ -14,8 +14,6 @@ from warp_beacon.jobs.upload_job import UploadJob
 from warp_beacon.jobs import Origin
 from warp_beacon.jobs.types import JobType
 from warp_beacon.scraper.link_resolver import LinkResolver
-from warp_beacon.scraper.fail_handler import FailHandler
-from warp_beacon.storage.mongo import DBClient
 
 class Handlers(object):
 	storage = None
@@ -25,12 +23,6 @@ class Handlers(object):
 	def __init__(self, bot: "Bot") -> None:
 		self.bot = bot
 		self.storage = bot.storage
-		# add uploader callbacks to handle service restart
-		for job in FailHandler(DBClient()).get_failed_jobs(clean=False):
-			self.bot.uploader.add_callback(
-				job.placeholder_message_id,
-				self.upload_wrapper
-			)
 
 	async def help(self, _: Client, message: Message) -> None:
 		"""Send a message when the command /help is issued."""
@@ -147,11 +139,6 @@ class Handlers(object):
 					text="Failed to create message placeholder. Please check your bot Internet connection."
 				)
 
-			self.bot.uploader.add_callback(
-				job.placeholder_message_id,
-				self.upload_wrapper
-			)
-
 			self.bot.downloader.queue_task(job)
 		except Exception as e:
 			logging.error("Failed to schedule download task!")

warp_beacon/uploader/__init__.py

@@ -12,26 +12,17 @@ from warp_beacon.storage import Storage
 
 class AsyncUploader(object):
 	__JOE_BIDEN_WAKEUP = None
-	threads = None
-	allow_loop = True
-	job_queue = None
-	callbacks = None
-	storage = None
-	in_process = None
-	loop = None
-	admin_message_callback = None
-	request_yt_auth_callback = None
 	pool_size = 1
 
 	def __init__(self,
 			loop: asyncio.AbstractEventLoop,
 			storage: Storage,
+			upload_wrapper: Callable,
 			admin_message_callback: Callable,
 			request_yt_auth_callback: Callable,
 			pool_size: int=min(32, os.cpu_count() + 4)
 		) -> None:
 		self.threads = []
-		self.callbacks = {}
 		self.in_process = set()
 		self.storage = storage
 		self.loop = loop
@@ -39,6 +30,7 @@ class AsyncUploader(object):
 		self.admin_message_callback = admin_message_callback
 		self.request_yt_auth_callback = request_yt_auth_callback
 		self.pool_size = pool_size
+		self.upload_wrapper = upload_wrapper
 	
 	def __del__(self) -> None:
 		self.stop_all()
@@ -49,18 +41,6 @@ class AsyncUploader(object):
 			thread.start()
 			self.threads.append(thread)
 
-	def add_callback(self, message_id: int, callback: Callable) -> None:
-		if message_id not in self.callbacks:
-			async def callback_wrap(*args, **kwargs) -> None:
-				await callback(*args, **kwargs)
-				#self.remove_callback(message_id)
-			self.callbacks[message_id] = {"callback": callback_wrap}
-
-	def remove_callback(self, message_id: int) -> None:
-		if message_id in self.callbacks:
-			logging.debug("Removing callback with message id #%d", message_id)
-			del self.callbacks[message_id]
-
 	def stop_all(self) -> None:
 		self.allow_loop = False
 		if self.threads:
@@ -115,7 +95,6 @@ class AsyncUploader(object):
 	
 					in_process = job.in_process
 					uniq_id = job.uniq_id
-					message_id = job.placeholder_message_id
 
 					if not in_process and not job.job_failed and not job.job_warning and not job.replay:
 						if job.media_type == JobType.TEXT:
@@ -123,65 +102,61 @@ class AsyncUploader(object):
 						else:
 							logging.info("Accepted upload job, file(s): '%s'", path)
 
+					async def callback_wrap(*args, **kwargs) -> None:
+						await self.upload_wrapper(*args, **kwargs)
+
 					try:
-						if message_id in self.callbacks:
-							if job.job_failed:
-								logging.info("URL '%s' download failed. Skipping upload job ...", job.url)
-								if job.job_failed_msg: # we want to say something to user
-									self.loop.call_soon_threadsafe(
-										asyncio.create_task,
-										self.callbacks[message_id]["callback"](job)
-									)
-								self.process_done(uniq_id)
-								self.remove_callback(message_id)
-								continue
-							
-							if job.replay:
+						if job.job_failed:
+							logging.info("URL '%s' download failed. Skipping upload job ...", job.url)
+							if job.job_failed_msg: # we want to say something to user
 								self.loop.call_soon_threadsafe(
 									asyncio.create_task,
-									self.callbacks[message_id]["callback"](job)
+									callback_wrap(job)
 								)
-								self.remove_callback(message_id)
-								continue
+							self.process_done(uniq_id)
+							continue
+							
+						if job.replay:
+							self.loop.call_soon_threadsafe(
+								asyncio.create_task,
+								callback_wrap(job)
+							)
+							continue
 
-							if job.job_warning:
-								logging.info("Job warning occurred ...")
-								if job.job_warning_msg:
-									self.loop.call_soon_threadsafe(
-										asyncio.create_task,
-										self.callbacks[message_id]["callback"](job)
-									)
-								continue
-							if in_process:
-								db_list_dicts = self.storage.db_lookup_id(uniq_id)
-								if db_list_dicts:
-									tg_file_ids = [i["tg_file_id"] for i in db_list_dicts]
-									dlds_len = len(db_list_dicts)
-									if dlds_len > 1:
-										job.tg_file_id = ",".join(tg_file_ids)
-										job.media_type = JobType.COLLECTION
-									elif dlds_len:
-										job.tg_file_id = ",".join(tg_file_ids)
-										db_data = db_list_dicts.pop()
-										job.media_type = JobType[db_data["media_type"].upper()]
-										job.canonical_name = db_data.get("canonical_name", "")
-									self.loop.call_soon_threadsafe(
-										asyncio.create_task,
-										self.callbacks[message_id]["callback"](job)
-									)
-									self.process_done(uniq_id)
-									self.remove_callback(message_id)
-								else:
-									self.queue_task(job)
-							else:
+						if job.job_warning:
+							logging.info("Job warning occurred ...")
+							if job.job_warning_msg:
 								self.loop.call_soon_threadsafe(
 									asyncio.create_task,
-									self.callbacks[message_id]["callback"](job)
+									callback_wrap(job)
+								)
+							continue
+						if in_process:
+							db_list_dicts = self.storage.db_lookup_id(uniq_id)
+							if db_list_dicts:
+								tg_file_ids = [i["tg_file_id"] for i in db_list_dicts]
+								dlds_len = len(db_list_dicts)
+								if dlds_len > 1:
+									job.tg_file_id = ",".join(tg_file_ids)
+									job.media_type = JobType.COLLECTION
+								elif dlds_len:
+									job.tg_file_id = ",".join(tg_file_ids)
+									db_data = db_list_dicts.pop()
+									job.media_type = JobType[db_data["media_type"].upper()]
+									job.canonical_name = db_data.get("canonical_name", "")
+								self.loop.call_soon_threadsafe(
+									asyncio.create_task,
+									callback_wrap(job)
 								)
 								self.process_done(uniq_id)
-								self.remove_callback(message_id)
+							else:
+								self.queue_task(job)
 						else:
-							logging.info("No callback no call!!")
+							self.loop.call_soon_threadsafe(
+								asyncio.create_task,
+								callback_wrap(job)
+							)
+							self.process_done(uniq_id)
 					except Exception as e:
 						logging.exception(e)
 				except Empty:
@@ -189,4 +164,4 @@ class AsyncUploader(object):
 			except Exception as e:
 				logging.error("Exception occurred inside upload worker!")
 				logging.exception(e)
-		logging.info("Thread done")
+			logging.info("Thread done")

{warp_beacon-2.8.14.dist-info → warp_beacon-2.8.16.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.8.14
+Version: 2.8.16
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

{warp_beacon-2.8.14.dist-info → warp_beacon-2.8.16.dist-info}/RECORD

@@ -4,7 +4,7 @@ var/warp_beacon/accounts.json,sha256=OsXdncs6h88xrF_AP6_WDCK1waGBn9SR-uYdIeK37GM
 var/warp_beacon/placeholder.gif,sha256=cE5CGJVaop4Sx21zx6j4AyoHU0ncmvQuS2o6hJfEH88,6064
 var/warp_beacon/proxies.json,sha256=VnjlQDXumOEq72ZFjbh6IqHS1TEHqn8HPYAZqWCeSIA,95
 warp_beacon/__init__.py,sha256=_rThNODmz0nDp_n4mWo_HKaNFE5jk1_7cRhHyYaencI,163
-warp_beacon/__version__.py,sha256=TZgBJIjZg_hpyHZh7yBDpvAmjXgQy0i383jyxYXsn9A,24
+warp_beacon/__version__.py,sha256=9j8dYjzjanXTCo3lfOVfAZGCuvknHdNoJReyvG-GYZk,24
 warp_beacon/warp_beacon.py,sha256=ADCR30uGXIsDrt9WoiI9Ghu2QtWs0qZIK6x3pQKM_B4,1109
 warp_beacon/yt_auth.py,sha256=GUTKqYr_tzDC-07Lx_ahWXSag8EyLxXBUnQbDBIkEmk,6022
 warp_beacon/compress/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -22,11 +22,11 @@ warp_beacon/mediainfo/video.py,sha256=UBZrhTN5IDI-aYu6tsJEILo9nFkjHhkldGVFmvV7tE
 warp_beacon/scheduler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scheduler/instagram_human.py,sha256=lOytnonvqtB_8z6TVzrVJ1prqqgE4fktZZl-xpn7SOA,12951
 warp_beacon/scheduler/scheduler.py,sha256=0u9AIr9fTBmjU1GpOwKlPuNLskeJ4A-S2uAUzb-qO6w,4997
-warp_beacon/scraper/__init__.py,sha256=EBS7k8vmn3ya_Ud0tPHtBo7xUMUP9tabmYk5Ase_zyw,20818
+warp_beacon/scraper/__init__.py,sha256=q4Z2UQ84jjnNviALdyqFv33exnrPwwfAeZK1BH85kFk,20825
 warp_beacon/scraper/abstract.py,sha256=pWbaTu-gDZgi-iFjqMR_uGzPl5KLv-4gTdJ9w6cD4sk,3802
 warp_beacon/scraper/account_selector.py,sha256=n-466AiTXZ8o5cgcNkNwNiWLoi-EkLC7bHh6El1eIF8,10274
 warp_beacon/scraper/exceptions.py,sha256=hicAe6_0xN7Ry2gcFX4UvqPWMtF_lX2ihH1njQAaqCA,1496
-warp_beacon/scraper/fail_handler.py,sha256=5ODu4b8ndZWAcHIXrcUufsWFihetzNUoAi8IgAkreyQ,998
+warp_beacon/scraper/fail_handler.py,sha256=hu0mGS6zeyo3VV86XORvCIWbTApcb2bE_EBoxANpFPo,1529
 warp_beacon/scraper/link_resolver.py,sha256=Rc9ZuMyOo3iPywDHwjngy-WRQ2SXhJwxcg-5ripx7tM,2447
 warp_beacon/scraper/utils.py,sha256=AOZmDki2Pbr84IG-j_wN2UghKCiWFVDYdx6HJl0JTBs,1258
 warp_beacon/scraper/X/X.py,sha256=lKxNe70iIasLnyN8QA_1rLa70Bd3Y9fL6J4AEdUgNJs,9200
@@ -45,20 +45,20 @@ warp_beacon/scraper/youtube/youtube.py,sha256=uYR7XpfP6ZnSvw1Gc4qG_M8jkCyv3maEyt
 warp_beacon/storage/__init__.py,sha256=xg3quvc-Lkc-hCZ2lkrTsGqLyMybTN14mwrSuDJMfD4,3403
 warp_beacon/storage/mongo.py,sha256=qC4ZiO8XXvPnP0rJwz4CJx42pqFsyAjCiW10W5QdT6E,527
 warp_beacon/telegram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-warp_beacon/telegram/bot.py,sha256=xeSzUxNZf8zjU5K_UkwtzBzB6nIml1UPI0g1ITQdoEQ,20675
+warp_beacon/telegram/bot.py,sha256=DdVgW1SraCabgruEzOKRzh2-Gbn8JDSzxEqcMKeDMHg,20723
 warp_beacon/telegram/caption_shortener.py,sha256=EnguNCF52ne7y4P-iJAbI6K3sqoJqJbND_dX5Fhwkv0,1549
 warp_beacon/telegram/download_status.py,sha256=N-Qg13LVcPskyQNKG1lw50K1lhFtXu003muCRzZ7wiE,1561
 warp_beacon/telegram/edit_message.py,sha256=vzSrtlt-QxBU-X9uRSKo7gJftOrvB5Qo3iNsVNptCoE,5684
-warp_beacon/telegram/handlers.py,sha256=2XJ3v9sVsWa3V3cji9FegO7l5JNqXqXxQrxIOzI2fyE,10793
+warp_beacon/telegram/handlers.py,sha256=hA5rzfYcCoU6KJE1YMkEkQnv_B06UiSvJDeqr4JbhXw,10376
 warp_beacon/telegram/placeholder_message.py,sha256=wN9-BRiyrtHG-EvXtZkGJHt2CX71munQ57ITttjt0mw,6400
 warp_beacon/telegram/progress_bar.py,sha256=IP4xtvLtdJtqdr2C-0YaU428iQGrKurbP4Npr31iW74,5014
 warp_beacon/telegram/progress_file_reader.py,sha256=e3equyNKlKs764AD-iE9QRsh3YDHTzP78Mx5tdvPPWs,969
 warp_beacon/telegram/types.py,sha256=Kvdng6uCF1HRoqQgGW1ZYYPJoVuYkFb-LDvMBbW5Hjk,89
 warp_beacon/telegram/utils.py,sha256=zTF8VQfAWetBSjAPbmNe_Zi_LN5fAcWptJKjLaFNHaE,5073
-warp_beacon/uploader/__init__.py,sha256=dR0VjIGSr859TTdorA2tKnjH7EpQOXnG71aXhZFaMl0,5863
-warp_beacon-2.8.14.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-warp_beacon-2.8.14.dist-info/METADATA,sha256=8eP6Ho7mF0Uq7jz0-MLzRb8niSFPPh3ewdFrmN3BH0I,23236
-warp_beacon-2.8.14.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-warp_beacon-2.8.14.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
-warp_beacon-2.8.14.dist-info/top_level.txt,sha256=RraB0PWGvRK2zPYkuICKNgStLG1C5s7rPHHJEHJbkgA,1510
-warp_beacon-2.8.14.dist-info/RECORD,,
+warp_beacon/uploader/__init__.py,sha256=zYHDg23RIAaY0HST8BCo8JFyM-oiq31hZ7EDwf9NvFw,4862
+warp_beacon-2.8.16.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+warp_beacon-2.8.16.dist-info/METADATA,sha256=HR43pAdA8_kTn0nhYsM7DtLJp-8bjuqOPGhRJDW3arI,23236
+warp_beacon-2.8.16.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+warp_beacon-2.8.16.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
+warp_beacon-2.8.16.dist-info/top_level.txt,sha256=RraB0PWGvRK2zPYkuICKNgStLG1C5s7rPHHJEHJbkgA,1510
+warp_beacon-2.8.16.dist-info/RECORD,,