warp-beacon 2.8.13__py3-none-any.whl → 2.8.15__py3-none-any.whl

warp_beacon/__version__.py

@@ -1,2 +1,2 @@
-__version__ = "2.8.13"
+__version__ = "2.8.15"
 

warp_beacon/jobs/abstract.py

@@ -109,6 +109,9 @@ class AbstractJob(ABC):
 		if self.media_type == JobType.COLLECTION:
 			if not self.media_collection:
 				return True
+		elif self.media_type == JobType.TEXT:
+			if not self.message_text:
+				return True
 		elif not self.local_media_path:
 			return True
 		return False
@@ -129,9 +132,13 @@ class AbstractJob(ABC):
 				for j in i:
 					if os.path.exists(j.local_media_path):
 						os.unlink(j.local_media_path)
+		elif self.media_type == JobType.TEXT:
+			pass
 		else:
 			if os.path.exists(self.local_media_path):
 				os.unlink(self.local_media_path)
 			if self.local_compressed_media_path:
 				if os.path.exists(self.local_compressed_media_path):
 					os.unlink(self.local_compressed_media_path)
+
+		return True

warp_beacon/jobs/types.py

@@ -6,4 +6,5 @@ class JobType(str, Enum):
 	IMAGE = "image",
 	AUDIO = "audio",
 	COLLECTION = "collection"
-	ANIMATION = "animation"
+	ANIMATION = "animation"
+	TEXT = "text"

warp_beacon/scraper/X/X.py

@@ -1,7 +1,7 @@
 import os
 import time
 import logging
-from mimetypes import guess_extension
+from mimetypes import guess_extension, guess_type
 from urllib.parse import urlparse
 import requests
 import yt_dlp
@@ -9,10 +9,11 @@ from playwright.sync_api import sync_playwright, Page
 
 from warp_beacon.telegram.utils import Utils
 from warp_beacon.scraper.utils import ScraperUtils
-from warp_beacon.scraper.X.types import XMediaType
 from warp_beacon.jobs.types import JobType
 from warp_beacon.scraper.X.abstract import XAbstract
 
+from warp_beacon.scraper.exceptions import Unavailable
+
 class XScraper(XAbstract):
 	DOWNLOAD_DIR = "/tmp"
 
@@ -29,107 +30,134 @@ class XScraper(XAbstract):
 
 		return ret
 
-	def get_media_type(self, media_info: dict) -> XMediaType:
-		media_type = XMediaType.UNKNOWN
-		#logging.info("[X] post info: '%s'", media_info)
-
-		if 'ext' in media_info:
-			logging.info("[X] Format: '%s'", media_info['ext'])
-		if 'formats' in media_info:
-			logging.info("[X] Contains video.")
-			media_type = XMediaType.VIDEO
-		elif 'thumbnails' in media_info:
-			logging.info("[X] contains images.")
-			media_type = XMediaType.IMAGE
-		else:
-			logging.info("[X] No media found.")
-
-		return media_type
+	def generate_result(self, local_files: list, job_type: JobType, canonical_name: str = "", performer: str = "") -> list:
+		res = []
+		if local_files:
+			if job_type == JobType.COLLECTION:
+				chunks = []
+				for media_chunk in Utils.chunker(local_files, 10):
+					chunk = []
+					for media in media_chunk:
+						mime_type, _ = guess_type(media)
+						chunk.append({
+							"local_media_path": self.rename_local_file(media),
+							"canonical_name": canonical_name,
+							"media_type": JobType.VIDEO if "video" in mime_type else JobType.IMAGE,
+							"media_info": {}
+						})
+					chunks.append(chunk)
+					res.append({
+						"media_type": JobType.COLLECTION,
+						"canonical_name": canonical_name,
+						"items": chunks
+					})
+			else:
+				for local_file in local_files:
+					res.append({
+						"local_media_path": self.rename_local_file(local_file),
+						"performer": performer,
+						"canonical_name": canonical_name,
+						"media_type": job_type
+					})
+		logging.debug(res)
+		return res
 
 	def _download(self, url: str, timeout: int = 60) -> list:
 		res = []
-		job_type = JobType.UNKNOWN
+		post_text = ""
+		pw_proxy = None
+		if self.proxy:
+			dsn = self.proxy.get("dsn", "")
+			if dsn:
+				parsed = urlparse(dsn)
+				pw_proxy = {
+					"server": f"{parsed.scheme}://{parsed.hostname}:{parsed.port}",
+					"username": parsed.username,
+					"password": parsed.password
+				}
+				logging.info("[X] build proxy: %s", pw_proxy)
+
+		contains_images, contains_videos = False, False
+		images, videos = [], []
+		with sync_playwright() as p:
+			with p.chromium.launch(headless=True) as browser:
+				with browser.new_context(proxy=pw_proxy, ignore_https_errors=True) as context:
+					page = context.new_page()
+					page.goto(url, wait_until="networkidle", timeout=(timeout*1000))
+					page.wait_for_selector("article[role='article']", timeout=(timeout*1000))
+
+					contains_videos = self.tweet_contains_video(page)
+					contains_images = self.tweet_contains_images(page)
+
+					if contains_images:
+						post_text, images = self.download_images(page, timeout)
+
+					if not contains_images and not contains_videos:
+						post_text = self.extract_post_text(page)
+
+		if contains_videos:
+			media_info, videos = self.download_videos(url, timeout)
+			if media_info:
+				post_text = self.extract_canonical_name(media_info)
+
+		if not images and not videos:
+			if not post_text:
+				raise Unavailable("Content unvailable")
+			logging.info("[X]: Sending text message")
+			res.append({
+				"message_text": post_text,
+				"media_type": JobType.TEXT
+			})
+			return res
+		
+		if len(images) > 1 or len(videos) > 1:
+			logging.info("[X]: uploading collection")
+			content = images + videos
+			res.extend(self.generate_result(content, JobType.COLLECTION, canonical_name=post_text))
+		else:
+			logging.info("[X]: uploading media")
+			for job_type, content in {JobType.IMAGE: images, JobType.VIDEO: videos}.items():
+				if content:
+					res.extend(self.generate_result(content, job_type, canonical_name=post_text))
+
+		return res
+
+	def download_videos(self, url: str, timeout: int = 60) -> tuple[dict, list[str]]:
+		local_files = []
+		media_info = {}
 		time_name = str(time.time()).replace('.', '_')
 		ydl_opts = {
 			'socket_timeout': timeout,
-			'outtmpl': f'{self.DOWNLOAD_DIR}/x_download_{time_name}.%(ext)s',
+			'outtmpl': f'{self.DOWNLOAD_DIR}/x_download_{time_name}_%(id)s.%(ext)s',
 			'quiet': False,
 			'force_generic_extractor': False,
-			'noplaylist': True,
+			#'noplaylist': True,
 			'merge_output_format': 'mp4',
-			'dump_single_json': True,
+			'dump_single_json': False,
 			'nocheckcertificate': True,
 			'progress_hooks': [self.dlp_on_progress],
 		}
-
 		if self.proxy:
 			proxy_dsn = self.proxy.get("dsn", "")
 			logging.info("[X] Using proxy DSN '%s'", proxy_dsn)
 			if proxy_dsn:
 				ydl_opts["proxy"] = proxy_dsn
 
-		local_file, media_info, media_type, post_text = "", {}, XMediaType.UNKNOWN, ""
-		#tweet_contains_video, tweet_contains_images = False, False
-
-		#with sync_playwright() as p:
-		#	with p.chromium.launch(headless=True) as browser:
-		#		with browser.new_context(proxy=proxy, ignore_https_errors=True) as context:
-		#			page = context.new_page()
-		#			page.goto(url, wait_until="networkidle", timeout=(timeout*1000))
-		#			tweet_contains_video = self.tweet_contains_video(page)
-		#			tweet_contains_images = self.tweet_contains_images(page)
-
 		with yt_dlp.YoutubeDL(ydl_opts) as ydl:
-			try:
-				media_info = ydl.extract_info(url, download=False)
-				media_type = self.get_media_type(media_info)
-				if media_type == XMediaType.VIDEO:
-					local_file = self.download_video(url, ydl, media_info)
-					post_text = self.extract_canonical_name(media_info)
-					job_type = JobType.VIDEO
-			except yt_dlp.utils.DownloadError as e:
-				msg = str(e).lower()
-				if "no video could be found in this tweet" in msg:
-					logging.warning("[X] yt_dlp failed to extract info. Falling back to image scraping.")
-					media_type = XMediaType.IMAGE
-				else:
-					raise
-		
-		images = []
-		if media_type == XMediaType.IMAGE:
-			job_type = JobType.IMAGE
-			images, post_text = self.download_images(url, timeout)
-			if images:
-				if len(images) > 1:
-					job_type = JobType.COLLECTION
-				else:
-					local_file = images[0]
-
-		if job_type == JobType.COLLECTION:
-			chunks = []
-			for media_chunk in Utils.chunker(images, 10):
-				chunk = []
-				for media in media_chunk:
-					chunk.append({
-						"local_media_path": self.rename_local_file(media),
-						"canonical_name": post_text,
-						"media_type": JobType.IMAGE
-					})
-				chunks.append(chunk)
-			res.append({
-				"media_type": JobType.COLLECTION,
-				"items": chunks
-			})
-		else:
-			if local_file:
-				res.append({
-					"local_media_path": self.rename_local_file(local_file),
-					"performer": media_info.get("uploader", "Unknown"),
-					"canonical_name": post_text,
-					"media_type": job_type
-				})
+			info = ydl.extract_info(url, download=False)
+			media_info = info
+			entries = info.get("entries", [info])
+
+			for entry in entries:
+				ret = ydl.download([entry['webpage_url']])
+				if ret == 0:
+					file_path = ydl.prepare_filename(entry)
+					if isinstance(file_path, str):
+						local_files.append(file_path)
+					else:
+						local_files.extend(file_path)
 
-		return res
+		return media_info, local_files
 
 	def adaptive_chunk_size(self, content_length: int) -> int:
 		if content_length < 100_000:
@@ -141,20 +169,11 @@ class XScraper(XAbstract):
 		else:
 			return 65536
 
-	def download_video(self, url: str, ydl: yt_dlp.YoutubeDL, media_info: dict) -> str:
-		local_file = ""
-		ydl.download([url])
-		local_file = ydl.prepare_filename(media_info)
-		logging.debug("Temp filename: '%s'", local_file)
-		if local_file:
-			local_file = self.rename_local_file(local_file)
-		return local_file
-
 	def get_extension_from_headers(self, response: requests.Response) -> str:
 		content_type = response.headers.get("Content-Type", "")
 		return guess_extension(content_type) or ".jpg"
 
-	def download_images(self, url: str, timeout: int = 60) -> tuple[list[str], str]:
+	def download_images(self, page: Page, timeout: int) -> tuple[str, list[str]]:
 		downloaded_imgs = []
 		headers = {
 			"User-Agent": ScraperUtils.get_ua(),
@@ -166,7 +185,7 @@ class XScraper(XAbstract):
 		if self.proxy:
 			proxies = {"https": self.proxy.get("dsn", ""), "http": self.proxy.get("dsn", "")}
 
-		image_urls, post_text = self.extract_image_urls_from_x_post(url, timeout=timeout)
+		image_urls, post_text = self.extract_image_urls_from_x_post(page, timeout)
 
 		if not image_urls:
 			logging.error("[X] Content images are not found!")
@@ -210,101 +229,25 @@ class XScraper(XAbstract):
 							)
 				downloaded_imgs.append(filepath)
 
-		return downloaded_imgs, post_text
+		return post_text, downloaded_imgs
 
-	def extract_post_text(self, page: Page) -> str:
-		try:
-			tweet_texts = []
-			# collecting text blocks from post
-			containers = page.query_selector_all('div[data-testid="tweetText"]')
-			for container in containers:
-				try:
-					spans = container.query_selector_all("span")
-					if spans:
-						for span in spans:
-							text = span.inner_text().strip()
-							if text:
-								tweet_texts.append(text)
-					else:
-						# to span's try container itself
-						text = container.inner_text().strip()
-						if text:
-							tweet_texts.append(text)
-				except Exception:
-					continue
-
-			return " ".join(tweet_texts).strip()
-		except Exception as e:
-			logging.warning("Failed to extract tweet text.", exc_info=e)
-
-		return ""
-
-	def extract_image_urls_from_x_post(self, url: str, timeout: int = 60) -> tuple[list[str], str]:
+	def extract_image_urls_from_x_post(self, page: Page, timeout: int) -> tuple[list[str], str]:
 		img_urls, post_text = [], ''
 
-		proxy = None
-		if self.proxy:
-			dsn = self.proxy.get("dsn", "")
-			if dsn:
-				parsed = urlparse(dsn)
-				proxy = {
-					"server": f"{parsed.scheme}://{parsed.hostname}:{parsed.port}",
-					"username": parsed.username,
-					"password": parsed.password
-				}
-				logging.info("[X] build proxy: %s", proxy)
-
-		with sync_playwright() as p:
-			with p.chromium.launch(headless=True) as browser:
-				with browser.new_context(proxy=proxy, ignore_https_errors=True) as context:
-					page = context.new_page()
-					page.goto(url, wait_until="networkidle", timeout=(timeout*1000))
-
-					#page.wait_for_timeout(3000)
-					page.wait_for_selector("img[src*='pbs.twimg.com/media']", timeout=(timeout*1000))
-					post_text = self.extract_post_text(page)
+		page.wait_for_selector("img[src*='pbs.twimg.com/media']", timeout=(timeout*1000))
+		post_text = self.extract_post_text(page)
 
-					image_elements = page.query_selector_all("img")
-					image_urls = []
+		image_elements = page.query_selector_all("img")
+		image_urls = []
 
-					for img in image_elements:
-						src = img.get_attribute("src")
-						if src and "pbs.twimg.com/media" in src:
-							image_urls.append(src)
+		for img in image_elements:
+			src = img.get_attribute("src")
+			if src and "pbs.twimg.com/media" in src:
+				image_urls.append(src)
 					
-					img_urls = list(set(image_urls))
+		img_urls = list(set(image_urls))
 		return img_urls, post_text
 	
-	def get_media_type_from_info_and_dom(self, media_info: dict, page: Page) -> XMediaType:
-		is_video = (
-			media_info.get("vcodec") != "none" or
-			media_info.get("ext") in {"mp4", "mov", "mkv"} or
-			any(
-				f.get("vcodec") not in (None, "none")
-				for f in media_info.get("formats", [])
-			)
-		)
-
-		try:
-			image_elements = page.query_selector_all("img")
-			image_urls = [
-				img.get_attribute("src")
-				for img in image_elements
-				if img.get_attribute("src") and "pbs.twimg.com/media" in img.get_attribute("src")
-			]
-			has_images = bool(image_urls)
-		except Exception:
-			has_images = False
-
-		if is_video and has_images:
-			return XMediaType.MIXED
-		elif is_video:
-			return XMediaType.VIDEO
-		elif has_images:
-			return XMediaType.IMAGE
-		
-		return XMediaType.UNKNOWN
-	
 	def tweet_contains_video(self, page: Page) -> bool:
 		try:
 			return bool(
@@ -327,4 +270,35 @@ class XScraper(XAbstract):
 			return bool(image_urls)
 		except Exception:
 			pass
-		return False
+		return False
+	
+	def extract_post_text(self, page: Page) -> str:
+		try:
+			text_fragments = []
+
+			# find tweetText containers (in main and quoted)
+			containers = page.query_selector_all('div[data-testid="tweetText"]')
+			for container in containers:
+				fragments = []
+
+				# find <span> and <img alt=...> inside text
+				for node in container.query_selector_all("span, img"):
+					tag = node.evaluate("node => node.tagName.toLowerCase()")
+					if tag == "span":
+						value = node.inner_text().strip()
+						if value:
+							fragments.append(value)
+					elif tag == "img":
+						# emoji as image
+						alt = node.get_attribute("alt")
+						if alt:
+							fragments.append(alt)
+
+				if fragments:
+					text_fragments.append("".join(fragments))
+
+			return "\n\n".join(text_fragments).strip()
+
+		except Exception as e:
+			logging.warning("X: [extract_post_text] error", exc_info=e)
+		return ""

warp_beacon/scraper/X/types.py

@@ -1,7 +1,8 @@
-import enum
+from enum import Flag, auto
 
-class XMediaType(enum.Enum):
+class XMediaType(Flag):
 	UNKNOWN = 0
-	VIDEO = 1
-	IMAGE = 2
-	MIXED = 3
+	VIDEO = auto()
+	IMAGE = auto()
+	MIXED = auto()
+	PLAYLIST = auto()

warp_beacon/scraper/__init__.py

@@ -318,7 +318,8 @@ class AsyncDownloader(object):
 							if items:
 								# success
 								for job in fail_handler.get_failed_jobs():
-									self.queue_task(job)
+									self.queue_task(job["job"])
+								# media info processing
 								for item in items:
 									media_info = {"filesize": 0}
 									if item["media_type"] == JobType.VIDEO:
@@ -372,6 +373,8 @@ class AsyncDownloader(object):
 										job_args["media_collection"] = item["items"]
 										if item.get("save_items", None) is not None:
 											job_args["save_items"] = item.get("save_items", False)
+									elif item["media_type"] == JobType.TEXT:
+										job_args["message_text"] = item.get("message_text", "")
 									else:
 										job_args["local_media_path"] = item["local_media_path"]
 										if item.get("local_compressed_media_path", None):

warp_beacon/scraper/fail_handler.py

@@ -16,12 +16,15 @@ class FailHandler(object):
 		self.client.close()
 
 	def store_failed_job(self, job: DownloadJob) -> int:
-		db_id = -1
+		db_id = ""
 		try:
 			job_serilized = pickle.dumps(job)
 			db_id = self.db.insert_one(
 			{
-				"job_data": job_serilized
+				"job_data": job_serilized,
+				"uniq_id": job.uniq_id,
+				"message_id": job.message_id,
+				"chat_id": job.chat_id
 			}).inserted_id
 		except Exception as e:
 			logging.error("Failed to store job as failed!")
@@ -33,10 +36,26 @@ class FailHandler(object):
 		try:
 			cursor = self.db.find()
 			for document in cursor:
-				ret.append(pickle.loads(document["job_data"]))
+				ret.append({
+					"_id": document["_id"],
+					"job": pickle.loads(document["job_data"]),
+					"uniq_id": document.get("uniq_id"),
+					"message_id": document.get("message_id"),
+					"chat_id": document.get("chat_id")
+				})
 			if clean:
 				self.db.delete_many({})
 		except Exception as e:
 			logging.error("Failed to get failed jobs!")
 			logging.exception(e)
 		return ret
+	
+	def remove_failed_job(self, uniq_id: str) -> bool:
+		try:
+			result = self.db.delete_one({"uniq_id": uniq_id})
+			if result.deleted_count > 0:
+				return True
+		except Exception as e:
+			logging.error("Failed to remove failed job!", exc_info=e)
+
+		return False

warp_beacon/storage/__init__.py

@@ -67,7 +67,8 @@ class Storage(object):
 					"uniq_id": document["uniq_id"],
 					"tg_file_id": document["tg_file_id"],
 					"media_type": document["media_type"],
-					"canonical_name": document.get("canonical_name")
+					"canonical_name": document.get("canonical_name"),
+					"message_text": document.get("message_text")
 				})
 		except Exception as e:
 			logging.error("Error occurred while trying to read from the database!")
@@ -82,13 +83,10 @@ class Storage(object):
 	def db_lookup_id(self, uniq_id: str) -> list[dict]:
 		return self.db_find(uniq_id)
 	
-	def add_media(self, tg_file_ids: list[str], media_url: str, media_type: str, origin: str, canonical_name: str = "") -> list[int]:
+	def add_media(self, tg_file_ids: list[str], media_url: str, media_type: str, origin: str, canonical_name: str = "", message_text: str = "") -> list[int]:
 		uniq_id = self.compute_uniq(media_url)
 		media_ids = []
 		for tg_file_id in tg_file_ids:
-			if not tg_file_id:
-				logging.warning("Passed empty `tg_file_id`! Skipping.")
-				continue
 			if self.db_lookup_id(uniq_id):
 				logging.info("Detected existing uniq_id, skipping storage write operation")
 				continue
@@ -98,7 +96,8 @@ class Storage(object):
 				"media_type": media_type,
 				"tg_file_id": tg_file_id,
 				"origin": origin,
-				"canonical_name": canonical_name
+				"canonical_name": canonical_name,
+				"message_text": message_text
 			}).inserted_id)
 
 		return media_ids

warp_beacon/telegram/bot.py

@@ -393,6 +393,9 @@ class Bot(object):
 							tg_chunk.append(anim)
 					mediafs.append(tg_chunk)
 				args["media"] = mediafs
+		elif job.media_type == JobType.TEXT:
+			args["text"] = f"<b>Post text:</b><pre>{job.message_text}</pre>\n\n{self.build_signature_caption(job)}"
+			args["parse_mode"] = ParseMode.HTML
 
 		args["chat_id"] = job.chat_id
 
@@ -412,7 +415,7 @@ class Bot(object):
 			if render_donates:
 				keyboard_buttons[0].append(InlineKeyboardButton("❤ Donate", url=os.environ.get("DONATE_LINK", "https://pay.cryptocloud.plus/pos/W5BMtNQt5bJFoW2E")))
 
-			if keyboard_buttons[0]:  #job.short_text or render_donates:
+			if keyboard_buttons[0]:
 				args["reply_markup"] = InlineKeyboardMarkup(keyboard_buttons)
 
 		return args
@@ -425,9 +428,14 @@ class Bot(object):
 			while not retry_amount >= max_retries:
 				try:
 					reply_message = None
-					if job.media_type in (JobType.VIDEO, JobType.IMAGE, JobType.AUDIO, JobType.ANIMATION):
-						if job.media_type in (JobType.VIDEO, JobType.AUDIO):
-							await Utils.ensure_me_loaded(self.client)
+					if job.media_type in (JobType.VIDEO, JobType.IMAGE, JobType.AUDIO, JobType.ANIMATION, JobType.TEXT):
+						#if job.media_type in (JobType.VIDEO, JobType.AUDIO):
+						#	await Utils.ensure_me_loaded(self.client)
+						if job.media_type == JobType.TEXT:
+							if job.placeholder_message_id:
+								await self.placeholder.remove(job.chat_id, job.placeholder_message_id)
+								job.placeholder_message_id = None
+
 						if job.placeholder_message_id:
 							try:
 								reply_message = await self.editor.edit(**self.build_tg_args(job))
@@ -440,7 +448,8 @@ class Bot(object):
 								JobType.VIDEO: self.client.send_video,
 								JobType.IMAGE: self.client.send_photo,
 								JobType.AUDIO: self.client.send_audio,
-								JobType.ANIMATION: self.client.send_animation
+								JobType.ANIMATION: self.client.send_animation,
+								JobType.TEXT: self.client.send_message
 							}
 							try:
 								while True:
@@ -460,9 +469,10 @@ class Bot(object):
 									job_args[reality.value.lower()] = job_args.pop(expectation.value.lower())
 									reply_message = await send_funcs[reality](**job_args)
 
-						tg_file_id = Utils.extract_file_id(reply_message)
-						tg_file_ids.append(tg_file_id)
-						job.tg_file_id = tg_file_id
+						if reply_message:
+							tg_file_id = Utils.extract_file_id(reply_message)
+							tg_file_ids.append(tg_file_id)
+							job.tg_file_id = tg_file_id
 						logging.info("Uploaded media file with type '%s' tg_file_id is '%s'", job.media_type.value, job.tg_file_id)
 					elif job.media_type == JobType.COLLECTION:
 						col_job_args = self.build_tg_args(job)

warp_beacon/telegram/handlers.py

@@ -14,8 +14,6 @@ from warp_beacon.jobs.upload_job import UploadJob
 from warp_beacon.jobs import Origin
 from warp_beacon.jobs.types import JobType
 from warp_beacon.scraper.link_resolver import LinkResolver
-from warp_beacon.scraper.fail_handler import FailHandler
-from warp_beacon.storage.mongo import DBClient
 
 class Handlers(object):
 	storage = None
@@ -25,12 +23,7 @@ class Handlers(object):
 	def __init__(self, bot: "Bot") -> None:
 		self.bot = bot
 		self.storage = bot.storage
-		# add uploader callbacks to handle service restart
-		for job in FailHandler(DBClient()).get_failed_jobs(clean=False):
-			self.bot.uploader.add_callback(
-				job.placeholder_message_id,
-				self.upload_wrapper
-			)
+		self.bot.uploader.uploader_wrapper = self.upload_wrapper
 
 	async def help(self, _: Client, message: Message) -> None:
 		"""Send a message when the command /help is issued."""
@@ -111,6 +104,14 @@ class Handlers(object):
 						origin=job.job_origin.value,
 						canonical_name=common_canonical_name
 					)
+				elif job.media_type == JobType.TEXT:
+					self.storage.add_media(
+						tg_file_ids=[None],
+						media_url=job.url,
+						media_type=job.media_type.value,
+						origin=job.job_origin.value,
+						message_text=job.message_text
+					)
 				else:
 					self.storage.add_media(
 						tg_file_ids=[','.join(tg_file_ids)],
@@ -139,11 +140,6 @@ class Handlers(object):
 					text="Failed to create message placeholder. Please check your bot Internet connection."
 				)
 
-			self.bot.uploader.add_callback(
-				job.placeholder_message_id,
-				self.upload_wrapper
-			)
-
 			self.bot.downloader.queue_task(job)
 		except Exception as e:
 			logging.error("Failed to schedule download task!")
@@ -215,6 +211,7 @@ class Handlers(object):
 					elif ent_len:
 						media_type = JobType[entities[0]["media_type"].upper()]
 						canonical_name = entities[0]["canonical_name"]
+						message_text = entities[0]["message_text"]
 						await self.bot.upload_job(
 							UploadJob(
 								url=url,
@@ -228,22 +225,23 @@ class Handlers(object):
 								chat_type=message.chat.type,
 								source_username=Utils.extract_message_author(message),
 								canonical_name=canonical_name,
-								message_leftover=msg_leftover
+								message_leftover=msg_leftover,
+								message_text=message_text
 							)
 						)
 				else:
 					if await self.queue_job(DownloadJob.build(
-							url=url,
-							message_id=effective_message_id,
-							chat_id=chat.id,
-							user_id=message.from_user.id,
-							in_process=self.bot.uploader.is_inprocess(uniq_id),
-							uniq_id=uniq_id,
-							job_origin=origin,
-							source_username=Utils.extract_message_author(message),
-							chat_type=chat.type,
-							message_leftover=msg_leftover
-						)):
+						url=url,
+						message_id=effective_message_id,
+						chat_id=chat.id,
+						user_id=message.from_user.id,
+						in_process=self.bot.uploader.is_inprocess(uniq_id),
+						uniq_id=uniq_id,
+						job_origin=origin,
+						source_username=Utils.extract_message_author(message),
+						chat_type=chat.type,
+						message_leftover=msg_leftover
+					)):
 						self.bot.uploader.set_inprocess(uniq_id)
 
 		if chat.type not in (ChatType.GROUP, ChatType.SUPERGROUP) and not urls:

warp_beacon/uploader/__init__.py

@@ -31,7 +31,6 @@ class AsyncUploader(object):
 			pool_size: int=min(32, os.cpu_count() + 4)
 		) -> None:
 		self.threads = []
-		self.callbacks = {}
 		self.in_process = set()
 		self.storage = storage
 		self.loop = loop
@@ -39,6 +38,7 @@ class AsyncUploader(object):
 		self.admin_message_callback = admin_message_callback
 		self.request_yt_auth_callback = request_yt_auth_callback
 		self.pool_size = pool_size
+		self.upload_wrapper: Callable = lambda: None
 	
 	def __del__(self) -> None:
 		self.stop_all()
@@ -49,18 +49,6 @@ class AsyncUploader(object):
 			thread.start()
 			self.threads.append(thread)
 
-	def add_callback(self, message_id: int, callback: Callable) -> None:
-		if message_id not in self.callbacks:
-			async def callback_wrap(*args, **kwargs) -> None:
-				await callback(*args, **kwargs)
-				#self.remove_callback(message_id)
-			self.callbacks[message_id] = {"callback": callback_wrap}
-
-	def remove_callback(self, message_id: int) -> None:
-		if message_id in self.callbacks:
-			logging.debug("Removing callback with message id #%d", message_id)
-			del self.callbacks[message_id]
-
 	def stop_all(self) -> None:
 		self.allow_loop = False
 		if self.threads:
@@ -89,7 +77,7 @@ class AsyncUploader(object):
 		while self.allow_loop:
 			try:
 				try:
-					job = self.job_queue.get()
+					job: UploadJob = self.job_queue.get()
 					if job is self.__JOE_BIDEN_WAKEUP:
 						break
 					if job.is_message_to_admin and job.message_text and self.admin_message_callback:
@@ -115,70 +103,68 @@ class AsyncUploader(object):
 	
 					in_process = job.in_process
 					uniq_id = job.uniq_id
-					message_id = job.placeholder_message_id
 
 					if not in_process and not job.job_failed and not job.job_warning and not job.replay:
-						logging.info("Accepted upload job, file(s): '%s'", path)
+						if job.media_type == JobType.TEXT:
+							logging.info("Uploading job text: '%s'", job.message_text)
+						else:
+							logging.info("Accepted upload job, file(s): '%s'", path)
+
+					async def callback_wrap(*args, **kwargs) -> None:
+						await self.upload_wrapper(*args, **kwargs)
 
 					try:
-						if message_id in self.callbacks:
-							if job.job_failed:
-								logging.info("URL '%s' download failed. Skipping upload job ...", job.url)
-								if job.job_failed_msg: # we want to say something to user
-									self.loop.call_soon_threadsafe(
-										asyncio.create_task,
-										self.callbacks[message_id]["callback"](job)
-									)
-								self.process_done(uniq_id)
-								self.remove_callback(message_id)
-								continue
-							
-							if job.replay:
+						if job.job_failed:
+							logging.info("URL '%s' download failed. Skipping upload job ...", job.url)
+							if job.job_failed_msg: # we want to say something to user
 								self.loop.call_soon_threadsafe(
 									asyncio.create_task,
-									self.callbacks[message_id]["callback"](job)
+									callback_wrap(job)
 								)
-								self.remove_callback(message_id)
-								continue
+							self.process_done(uniq_id)
+							continue
+							
+						if job.replay:
+							self.loop.call_soon_threadsafe(
+								asyncio.create_task,
+								callback_wrap(job)
+							)
+							continue
 
-							if job.job_warning:
-								logging.info("Job warning occurred ...")
-								if job.job_warning_msg:
-									self.loop.call_soon_threadsafe(
-										asyncio.create_task,
-										self.callbacks[message_id]["callback"](job)
-									)
-								continue
-							if in_process:
-								db_list_dicts = self.storage.db_lookup_id(uniq_id)
-								if db_list_dicts:
-									tg_file_ids = [i["tg_file_id"] for i in db_list_dicts]
-									dlds_len = len(db_list_dicts)
-									if dlds_len > 1:
-										job.tg_file_id = ",".join(tg_file_ids)
-										job.media_type = JobType.COLLECTION
-									elif dlds_len:
-										job.tg_file_id = ",".join(tg_file_ids)
-										db_data = db_list_dicts.pop()
-										job.media_type = JobType[db_data["media_type"].upper()]
-										job.canonical_name = db_data.get("canonical_name", "")
-									self.loop.call_soon_threadsafe(
-										asyncio.create_task,
-										self.callbacks[message_id]["callback"](job)
-									)
-									self.process_done(uniq_id)
-									self.remove_callback(message_id)
-								else:
-									self.queue_task(job)
-							else:
+						if job.job_warning:
+							logging.info("Job warning occurred ...")
+							if job.job_warning_msg:
+								self.loop.call_soon_threadsafe(
+									asyncio.create_task,
+									callback_wrap(job)
+								)
+							continue
+						if in_process:
+							db_list_dicts = self.storage.db_lookup_id(uniq_id)
+							if db_list_dicts:
+								tg_file_ids = [i["tg_file_id"] for i in db_list_dicts]
+								dlds_len = len(db_list_dicts)
+								if dlds_len > 1:
+									job.tg_file_id = ",".join(tg_file_ids)
+									job.media_type = JobType.COLLECTION
+								elif dlds_len:
+									job.tg_file_id = ",".join(tg_file_ids)
+									db_data = db_list_dicts.pop()
+									job.media_type = JobType[db_data["media_type"].upper()]
+									job.canonical_name = db_data.get("canonical_name", "")
 								self.loop.call_soon_threadsafe(
 									asyncio.create_task,
-									self.callbacks[message_id]["callback"](job)
+									callback_wrap(job)
 								)
 								self.process_done(uniq_id)
-								self.remove_callback(message_id)
+							else:
+								self.queue_task(job)
 						else:
-							logging.info("No callback no call!!")
+							self.loop.call_soon_threadsafe(
+								asyncio.create_task,
+								callback_wrap(job)
+							)
+							self.process_done(uniq_id)
 					except Exception as e:
 						logging.exception(e)
 				except Empty:
@@ -186,4 +172,4 @@ class AsyncUploader(object):
 			except Exception as e:
 				logging.error("Exception occurred inside upload worker!")
 				logging.exception(e)
-		logging.info("Thread done")
+			logging.info("Thread done")

{warp_beacon-2.8.13.dist-info → warp_beacon-2.8.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.8.13
+Version: 2.8.15
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

{warp_beacon-2.8.13.dist-info → warp_beacon-2.8.15.dist-info}/RECORD

@@ -4,15 +4,15 @@ var/warp_beacon/accounts.json,sha256=OsXdncs6h88xrF_AP6_WDCK1waGBn9SR-uYdIeK37GM
 var/warp_beacon/placeholder.gif,sha256=cE5CGJVaop4Sx21zx6j4AyoHU0ncmvQuS2o6hJfEH88,6064
 var/warp_beacon/proxies.json,sha256=VnjlQDXumOEq72ZFjbh6IqHS1TEHqn8HPYAZqWCeSIA,95
 warp_beacon/__init__.py,sha256=_rThNODmz0nDp_n4mWo_HKaNFE5jk1_7cRhHyYaencI,163
-warp_beacon/__version__.py,sha256=1_KpREE4Ar5XxUAevYjPftsGB_movvTsXltUwhStZ6Y,24
+warp_beacon/__version__.py,sha256=sZeqGczIyRlbc0NkgjPVyhnq-P8plt-ZEzfbVLMPUZY,24
 warp_beacon/warp_beacon.py,sha256=ADCR30uGXIsDrt9WoiI9Ghu2QtWs0qZIK6x3pQKM_B4,1109
 warp_beacon/yt_auth.py,sha256=GUTKqYr_tzDC-07Lx_ahWXSag8EyLxXBUnQbDBIkEmk,6022
 warp_beacon/compress/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/compress/video.py,sha256=_PDMVYCyzLYxHv1uZmmzGcG_8rjaZr7BTXsXTTy_oS4,2846
 warp_beacon/jobs/__init__.py,sha256=vW5T4jJUla97TNRapX_Y6eJCiPMEbySSlD0SJQKfAXs,189
-warp_beacon/jobs/abstract.py,sha256=x8shgG1So1g-Yqu_uzij7yuqherJTIhVhdslOrq69Z4,3263
+warp_beacon/jobs/abstract.py,sha256=dDGWFJL474_u_Musk-nZ6NfKH6CYdAEQlpX8thl-hPg,3411
 warp_beacon/jobs/download_job.py,sha256=pfSEZpWVzya0hddU5794p2uQYfm4lHrtM1Ck0T-UrLk,844
-warp_beacon/jobs/types.py,sha256=Ae8zINgbs7cOcYkYoOCOACA7duyhnIGMQAJ_SJB1QRQ,176
+warp_beacon/jobs/types.py,sha256=bb73jHm12ahq3BPDwdsWzHB69KmElSpgbj-A0lA3VNk,191
 warp_beacon/jobs/upload_job.py,sha256=_ul4psPej1jLEs-BMcMR80GbXDSmm38jE9yoZtecclY,741
 warp_beacon/mediainfo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/mediainfo/abstract.py,sha256=ZR2JMuRpoh7nDNov9a8YkAfr6BI2HXnXzQtVrLgDxjs,1185
@@ -22,17 +22,17 @@ warp_beacon/mediainfo/video.py,sha256=UBZrhTN5IDI-aYu6tsJEILo9nFkjHhkldGVFmvV7tE
 warp_beacon/scheduler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scheduler/instagram_human.py,sha256=lOytnonvqtB_8z6TVzrVJ1prqqgE4fktZZl-xpn7SOA,12951
 warp_beacon/scheduler/scheduler.py,sha256=0u9AIr9fTBmjU1GpOwKlPuNLskeJ4A-S2uAUzb-qO6w,4997
-warp_beacon/scraper/__init__.py,sha256=o9-HQEf4yQVNtWuJN4NcLUovejiHhP_KkQ1Xf5EaQvU,20670
+warp_beacon/scraper/__init__.py,sha256=q4Z2UQ84jjnNviALdyqFv33exnrPwwfAeZK1BH85kFk,20825
 warp_beacon/scraper/abstract.py,sha256=pWbaTu-gDZgi-iFjqMR_uGzPl5KLv-4gTdJ9w6cD4sk,3802
 warp_beacon/scraper/account_selector.py,sha256=n-466AiTXZ8o5cgcNkNwNiWLoi-EkLC7bHh6El1eIF8,10274
 warp_beacon/scraper/exceptions.py,sha256=hicAe6_0xN7Ry2gcFX4UvqPWMtF_lX2ihH1njQAaqCA,1496
-warp_beacon/scraper/fail_handler.py,sha256=5ODu4b8ndZWAcHIXrcUufsWFihetzNUoAi8IgAkreyQ,998
+warp_beacon/scraper/fail_handler.py,sha256=hu0mGS6zeyo3VV86XORvCIWbTApcb2bE_EBoxANpFPo,1529
 warp_beacon/scraper/link_resolver.py,sha256=Rc9ZuMyOo3iPywDHwjngy-WRQ2SXhJwxcg-5ripx7tM,2447
 warp_beacon/scraper/utils.py,sha256=AOZmDki2Pbr84IG-j_wN2UghKCiWFVDYdx6HJl0JTBs,1258
-warp_beacon/scraper/X/X.py,sha256=3-GBXHcSmGIiDLucJWLpiuaA8EDfP0-B7ws4MiCelkE,9883
+warp_beacon/scraper/X/X.py,sha256=lKxNe70iIasLnyN8QA_1rLa70Bd3Y9fL6J4AEdUgNJs,9200
 warp_beacon/scraper/X/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scraper/X/abstract.py,sha256=pCzZPTCtn8pRbBx2SeuBUpMkEHqnOLtwLBAHYceL12Q,5475
-warp_beacon/scraper/X/types.py,sha256=9Y0PJo3vZ1DMQcyfqoE4y2-AQRAetVmIxQwFDZkZy30,87
+warp_beacon/scraper/X/types.py,sha256=RrAyODNA8WA0YzznOSK2wr-hstXf3BnEisy06uL-bdA,132
 warp_beacon/scraper/instagram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scraper/instagram/captcha.py,sha256=9UYziuqB3Tsat_ET6ex-cnZDbi6yCnsXHSpmE8MuUHk,4651
 warp_beacon/scraper/instagram/instagram.py,sha256=uzqUCVniRa3d9uavoMAz9-9MHvYOh7n_G7UyfgzHgAk,19154
@@ -42,23 +42,23 @@ warp_beacon/scraper/youtube/abstract.py,sha256=7CVR2fW6bpWYYKcveRddd6XlgDsfV_Pp3
 warp_beacon/scraper/youtube/music.py,sha256=5AeSBQyUgVCJT2hoBCV2WvlyuV9US09SYJhmBG_P9F8,2755
 warp_beacon/scraper/youtube/shorts.py,sha256=y0591kpWU35rt5OoWamkcHIstNZ98SXUlUKvYmUsyEY,4030
 warp_beacon/scraper/youtube/youtube.py,sha256=uYR7XpfP6ZnSvw1Gc4qG_M8jkCyv3maEytFdNWlYPwU,6732
-warp_beacon/storage/__init__.py,sha256=NaKKPXjwa8LvWsqnVJVz0riXj765lGswG9piKgI2lkY,3389
+warp_beacon/storage/__init__.py,sha256=xg3quvc-Lkc-hCZ2lkrTsGqLyMybTN14mwrSuDJMfD4,3403
 warp_beacon/storage/mongo.py,sha256=qC4ZiO8XXvPnP0rJwz4CJx42pqFsyAjCiW10W5QdT6E,527
 warp_beacon/telegram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-warp_beacon/telegram/bot.py,sha256=OjN1pniEc_y_Eljg_Nuvoof7sN5cheT1NbbibcEce5U,20234
+warp_beacon/telegram/bot.py,sha256=xeSzUxNZf8zjU5K_UkwtzBzB6nIml1UPI0g1ITQdoEQ,20675
 warp_beacon/telegram/caption_shortener.py,sha256=EnguNCF52ne7y4P-iJAbI6K3sqoJqJbND_dX5Fhwkv0,1549
 warp_beacon/telegram/download_status.py,sha256=N-Qg13LVcPskyQNKG1lw50K1lhFtXu003muCRzZ7wiE,1561
 warp_beacon/telegram/edit_message.py,sha256=vzSrtlt-QxBU-X9uRSKo7gJftOrvB5Qo3iNsVNptCoE,5684
-warp_beacon/telegram/handlers.py,sha256=32kJtlfSZyG4JbEj9W8tH6us7sZg2ziMmnxpEX_qGXw,10482
+warp_beacon/telegram/handlers.py,sha256=gNtu5LQ4vHKo9fzdQ_gh88r5885Ctjdlk0Ve6p8fPr8,10435
 warp_beacon/telegram/placeholder_message.py,sha256=wN9-BRiyrtHG-EvXtZkGJHt2CX71munQ57ITttjt0mw,6400
 warp_beacon/telegram/progress_bar.py,sha256=IP4xtvLtdJtqdr2C-0YaU428iQGrKurbP4Npr31iW74,5014
 warp_beacon/telegram/progress_file_reader.py,sha256=e3equyNKlKs764AD-iE9QRsh3YDHTzP78Mx5tdvPPWs,969
 warp_beacon/telegram/types.py,sha256=Kvdng6uCF1HRoqQgGW1ZYYPJoVuYkFb-LDvMBbW5Hjk,89
 warp_beacon/telegram/utils.py,sha256=zTF8VQfAWetBSjAPbmNe_Zi_LN5fAcWptJKjLaFNHaE,5073
-warp_beacon/uploader/__init__.py,sha256=1enK6qMWaTZEaK456JwaKOfvCvznHA8cjgceOsrF6Po,5732
-warp_beacon-2.8.13.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-warp_beacon-2.8.13.dist-info/METADATA,sha256=cxyRksofUivhcTgS7dcqbP2BCRASrMUE76BeiInH9M8,23236
-warp_beacon-2.8.13.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-warp_beacon-2.8.13.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
-warp_beacon-2.8.13.dist-info/top_level.txt,sha256=RraB0PWGvRK2zPYkuICKNgStLG1C5s7rPHHJEHJbkgA,1510
-warp_beacon-2.8.13.dist-info/RECORD,,
+warp_beacon/uploader/__init__.py,sha256=3UklNdhRO4fYrCf5vWQVHdMNvDvxL9ZlqeQRdpr4yMU,5024
+warp_beacon-2.8.15.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+warp_beacon-2.8.15.dist-info/METADATA,sha256=Tz32Ew3qtE8F4mtdvzYKOQU_muLz3HCGn6OiyD4UlHg,23236
+warp_beacon-2.8.15.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+warp_beacon-2.8.15.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
+warp_beacon-2.8.15.dist-info/top_level.txt,sha256=RraB0PWGvRK2zPYkuICKNgStLG1C5s7rPHHJEHJbkgA,1510
+warp_beacon-2.8.15.dist-info/RECORD,,