warp-beacon 2.8.14__tar.gz → 2.8.16__tar.gz

{warp_beacon-2.8.14/warp_beacon.egg-info → warp_beacon-2.8.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.8.14
+Version: 2.8.16
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

warp_beacon-2.8.16/warp_beacon/__version__.py

@@ -0,0 +1,2 @@
+__version__ = "2.8.16"
+

{warp_beacon-2.8.14 → warp_beacon-2.8.16}/warp_beacon/scraper/__init__.py

@@ -318,7 +318,7 @@ class AsyncDownloader(object):
 							if items:
 								# success
 								for job in fail_handler.get_failed_jobs():
-									self.queue_task(job)
+									self.queue_task(job["job"])
 								# media info processing
 								for item in items:
 									media_info = {"filesize": 0}

{warp_beacon-2.8.14 → warp_beacon-2.8.16}/warp_beacon/scraper/fail_handler.py

@@ -16,12 +16,15 @@ class FailHandler(object):
 		self.client.close()
 
 	def store_failed_job(self, job: DownloadJob) -> int:
-		db_id = -1
+		db_id = ""
 		try:
 			job_serilized = pickle.dumps(job)
 			db_id = self.db.insert_one(
 			{
-				"job_data": job_serilized
+				"job_data": job_serilized,
+				"uniq_id": job.uniq_id,
+				"message_id": job.message_id,
+				"chat_id": job.chat_id
 			}).inserted_id
 		except Exception as e:
 			logging.error("Failed to store job as failed!")
@@ -33,10 +36,26 @@ class FailHandler(object):
 		try:
 			cursor = self.db.find()
 			for document in cursor:
-				ret.append(pickle.loads(document["job_data"]))
+				ret.append({
+					"_id": document["_id"],
+					"job": pickle.loads(document["job_data"]),
+					"uniq_id": document.get("uniq_id"),
+					"message_id": document.get("message_id"),
+					"chat_id": document.get("chat_id")
+				})
 			if clean:
 				self.db.delete_many({})
 		except Exception as e:
 			logging.error("Failed to get failed jobs!")
 			logging.exception(e)
 		return ret
+	
+	def remove_failed_job(self, uniq_id: str) -> bool:
+		try:
+			result = self.db.delete_one({"uniq_id": uniq_id})
+			if result.deleted_count > 0:
+				return True
+		except Exception as e:
+			logging.error("Failed to remove failed job!", exc_info=e)
+
+		return False

{warp_beacon-2.8.14 → warp_beacon-2.8.16}/warp_beacon/telegram/bot.py

@@ -69,9 +69,11 @@ class Bot(object):
 		)
 
 		self.editor = EditMessage(self.client)
+		self.handlers = Handlers(self)
 
 		self.uploader = AsyncUploader(
 			storage=self.storage,
+			upload_wrapper=self.handlers.upload_wrapper,
 			admin_message_callback=self.send_text_to_admin,
 			request_yt_auth_callback=self.request_yt_auth,
 			pool_size=int(os.environ.get("UPLOAD_POOL_SIZE", default=workers_amount)),
@@ -85,7 +87,6 @@ class Bot(object):
 		)
 
 		self.scheduler = IGScheduler(self.downloader)
-		self.handlers = Handlers(self)
 
 		self.client.add_handler(MessageHandler(self.handlers.start, filters.command("start")))
 		self.client.add_handler(MessageHandler(self.handlers.help, filters.command("help")))

{warp_beacon-2.8.14 → warp_beacon-2.8.16}/warp_beacon/telegram/handlers.py

@@ -14,8 +14,6 @@ from warp_beacon.jobs.upload_job import UploadJob
 from warp_beacon.jobs import Origin
 from warp_beacon.jobs.types import JobType
 from warp_beacon.scraper.link_resolver import LinkResolver
-from warp_beacon.scraper.fail_handler import FailHandler
-from warp_beacon.storage.mongo import DBClient
 
 class Handlers(object):
 	storage = None
@@ -25,12 +23,6 @@ class Handlers(object):
 	def __init__(self, bot: "Bot") -> None:
 		self.bot = bot
 		self.storage = bot.storage
-		# add uploader callbacks to handle service restart
-		for job in FailHandler(DBClient()).get_failed_jobs(clean=False):
-			self.bot.uploader.add_callback(
-				job.placeholder_message_id,
-				self.upload_wrapper
-			)
 
 	async def help(self, _: Client, message: Message) -> None:
 		"""Send a message when the command /help is issued."""
@@ -147,11 +139,6 @@ class Handlers(object):
 					text="Failed to create message placeholder. Please check your bot Internet connection."
 				)
 
-			self.bot.uploader.add_callback(
-				job.placeholder_message_id,
-				self.upload_wrapper
-			)
-
 			self.bot.downloader.queue_task(job)
 		except Exception as e:
 			logging.error("Failed to schedule download task!")

{warp_beacon-2.8.14 → warp_beacon-2.8.16}/warp_beacon/uploader/__init__.py

@@ -12,26 +12,17 @@ from warp_beacon.storage import Storage
 
 class AsyncUploader(object):
 	__JOE_BIDEN_WAKEUP = None
-	threads = None
-	allow_loop = True
-	job_queue = None
-	callbacks = None
-	storage = None
-	in_process = None
-	loop = None
-	admin_message_callback = None
-	request_yt_auth_callback = None
 	pool_size = 1
 
 	def __init__(self,
 			loop: asyncio.AbstractEventLoop,
 			storage: Storage,
+			upload_wrapper: Callable,
 			admin_message_callback: Callable,
 			request_yt_auth_callback: Callable,
 			pool_size: int=min(32, os.cpu_count() + 4)
 		) -> None:
 		self.threads = []
-		self.callbacks = {}
 		self.in_process = set()
 		self.storage = storage
 		self.loop = loop
@@ -39,6 +30,7 @@ class AsyncUploader(object):
 		self.admin_message_callback = admin_message_callback
 		self.request_yt_auth_callback = request_yt_auth_callback
 		self.pool_size = pool_size
+		self.upload_wrapper = upload_wrapper
 	
 	def __del__(self) -> None:
 		self.stop_all()
@@ -49,18 +41,6 @@ class AsyncUploader(object):
 			thread.start()
 			self.threads.append(thread)
 
-	def add_callback(self, message_id: int, callback: Callable) -> None:
-		if message_id not in self.callbacks:
-			async def callback_wrap(*args, **kwargs) -> None:
-				await callback(*args, **kwargs)
-				#self.remove_callback(message_id)
-			self.callbacks[message_id] = {"callback": callback_wrap}
-
-	def remove_callback(self, message_id: int) -> None:
-		if message_id in self.callbacks:
-			logging.debug("Removing callback with message id #%d", message_id)
-			del self.callbacks[message_id]
-
 	def stop_all(self) -> None:
 		self.allow_loop = False
 		if self.threads:
@@ -115,7 +95,6 @@ class AsyncUploader(object):
 	
 					in_process = job.in_process
 					uniq_id = job.uniq_id
-					message_id = job.placeholder_message_id
 
 					if not in_process and not job.job_failed and not job.job_warning and not job.replay:
 						if job.media_type == JobType.TEXT:
@@ -123,65 +102,61 @@ class AsyncUploader(object):
 						else:
 							logging.info("Accepted upload job, file(s): '%s'", path)
 
+					async def callback_wrap(*args, **kwargs) -> None:
+						await self.upload_wrapper(*args, **kwargs)
+
 					try:
-						if message_id in self.callbacks:
-							if job.job_failed:
-								logging.info("URL '%s' download failed. Skipping upload job ...", job.url)
-								if job.job_failed_msg: # we want to say something to user
-									self.loop.call_soon_threadsafe(
-										asyncio.create_task,
-										self.callbacks[message_id]["callback"](job)
-									)
-								self.process_done(uniq_id)
-								self.remove_callback(message_id)
-								continue
-							
-							if job.replay:
+						if job.job_failed:
+							logging.info("URL '%s' download failed. Skipping upload job ...", job.url)
+							if job.job_failed_msg: # we want to say something to user
 								self.loop.call_soon_threadsafe(
 									asyncio.create_task,
-									self.callbacks[message_id]["callback"](job)
+									callback_wrap(job)
 								)
-								self.remove_callback(message_id)
-								continue
+							self.process_done(uniq_id)
+							continue
+							
+						if job.replay:
+							self.loop.call_soon_threadsafe(
+								asyncio.create_task,
+								callback_wrap(job)
+							)
+							continue
 
-							if job.job_warning:
-								logging.info("Job warning occurred ...")
-								if job.job_warning_msg:
-									self.loop.call_soon_threadsafe(
-										asyncio.create_task,
-										self.callbacks[message_id]["callback"](job)
-									)
-								continue
-							if in_process:
-								db_list_dicts = self.storage.db_lookup_id(uniq_id)
-								if db_list_dicts:
-									tg_file_ids = [i["tg_file_id"] for i in db_list_dicts]
-									dlds_len = len(db_list_dicts)
-									if dlds_len > 1:
-										job.tg_file_id = ",".join(tg_file_ids)
-										job.media_type = JobType.COLLECTION
-									elif dlds_len:
-										job.tg_file_id = ",".join(tg_file_ids)
-										db_data = db_list_dicts.pop()
-										job.media_type = JobType[db_data["media_type"].upper()]
-										job.canonical_name = db_data.get("canonical_name", "")
-									self.loop.call_soon_threadsafe(
-										asyncio.create_task,
-										self.callbacks[message_id]["callback"](job)
-									)
-									self.process_done(uniq_id)
-									self.remove_callback(message_id)
-								else:
-									self.queue_task(job)
-							else:
+						if job.job_warning:
+							logging.info("Job warning occurred ...")
+							if job.job_warning_msg:
 								self.loop.call_soon_threadsafe(
 									asyncio.create_task,
-									self.callbacks[message_id]["callback"](job)
+									callback_wrap(job)
+								)
+							continue
+						if in_process:
+							db_list_dicts = self.storage.db_lookup_id(uniq_id)
+							if db_list_dicts:
+								tg_file_ids = [i["tg_file_id"] for i in db_list_dicts]
+								dlds_len = len(db_list_dicts)
+								if dlds_len > 1:
+									job.tg_file_id = ",".join(tg_file_ids)
+									job.media_type = JobType.COLLECTION
+								elif dlds_len:
+									job.tg_file_id = ",".join(tg_file_ids)
+									db_data = db_list_dicts.pop()
+									job.media_type = JobType[db_data["media_type"].upper()]
+									job.canonical_name = db_data.get("canonical_name", "")
+								self.loop.call_soon_threadsafe(
+									asyncio.create_task,
+									callback_wrap(job)
 								)
 								self.process_done(uniq_id)
-								self.remove_callback(message_id)
+							else:
+								self.queue_task(job)
 						else:
-							logging.info("No callback no call!!")
+							self.loop.call_soon_threadsafe(
+								asyncio.create_task,
+								callback_wrap(job)
+							)
+							self.process_done(uniq_id)
 					except Exception as e:
 						logging.exception(e)
 				except Empty:
@@ -189,4 +164,4 @@ class AsyncUploader(object):
 			except Exception as e:
 				logging.error("Exception occurred inside upload worker!")
 				logging.exception(e)
-		logging.info("Thread done")
+			logging.info("Thread done")

{warp_beacon-2.8.14 → warp_beacon-2.8.16/warp_beacon.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.8.14
+Version: 2.8.16
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

warp_beacon-2.8.14/warp_beacon/__version__.py

@@ -1,2 +0,0 @@
-__version__ = "2.8.14"
-