warp-beacon 2.8.13__tar.gz → 2.8.15__tar.gz

{warp_beacon-2.8.13/warp_beacon.egg-info → warp_beacon-2.8.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.8.13
+Version: 2.8.15
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

warp_beacon-2.8.15/warp_beacon/__version__.py

@@ -0,0 +1,2 @@
+__version__ = "2.8.15"
+

{warp_beacon-2.8.13 → warp_beacon-2.8.15}/warp_beacon/jobs/abstract.py

@@ -109,6 +109,9 @@ class AbstractJob(ABC):
 		if self.media_type == JobType.COLLECTION:
 			if not self.media_collection:
 				return True
+		elif self.media_type == JobType.TEXT:
+			if not self.message_text:
+				return True
 		elif not self.local_media_path:
 			return True
 		return False
@@ -129,9 +132,13 @@ class AbstractJob(ABC):
 				for j in i:
 					if os.path.exists(j.local_media_path):
 						os.unlink(j.local_media_path)
+		elif self.media_type == JobType.TEXT:
+			pass
 		else:
 			if os.path.exists(self.local_media_path):
 				os.unlink(self.local_media_path)
 			if self.local_compressed_media_path:
 				if os.path.exists(self.local_compressed_media_path):
 					os.unlink(self.local_compressed_media_path)
+
+		return True

{warp_beacon-2.8.13 → warp_beacon-2.8.15}/warp_beacon/jobs/types.py

@@ -6,4 +6,5 @@ class JobType(str, Enum):
 	IMAGE = "image",
 	AUDIO = "audio",
 	COLLECTION = "collection"
-	ANIMATION = "animation"
+	ANIMATION = "animation"
+	TEXT = "text"

warp_beacon-2.8.15/warp_beacon/scraper/X/X.py

@@ -0,0 +1,304 @@
+import os
+import time
+import logging
+from mimetypes import guess_extension, guess_type
+from urllib.parse import urlparse
+import requests
+import yt_dlp
+from playwright.sync_api import sync_playwright, Page
+
+from warp_beacon.telegram.utils import Utils
+from warp_beacon.scraper.utils import ScraperUtils
+from warp_beacon.jobs.types import JobType
+from warp_beacon.scraper.X.abstract import XAbstract
+
+from warp_beacon.scraper.exceptions import Unavailable
+
+class XScraper(XAbstract):
+	DOWNLOAD_DIR = "/tmp"
+
+	def extract_canonical_name(self, media: dict) -> str:
+		ret = ""
+		try:
+			if media.get("title", None):
+				ret = media["title"]
+			if media.get("description", ""):
+				ret += "\n" + media["description"]
+		except Exception as e:
+			logging.warning("Failed to extract canonical media name!")
+			logging.exception(e)
+
+		return ret
+
+	def generate_result(self, local_files: list, job_type: JobType, canonical_name: str = "", performer: str = "") -> list:
+		res = []
+		if local_files:
+			if job_type == JobType.COLLECTION:
+				chunks = []
+				for media_chunk in Utils.chunker(local_files, 10):
+					chunk = []
+					for media in media_chunk:
+						mime_type, _ = guess_type(media)
+						chunk.append({
+							"local_media_path": self.rename_local_file(media),
+							"canonical_name": canonical_name,
+							"media_type": JobType.VIDEO if "video" in mime_type else JobType.IMAGE,
+							"media_info": {}
+						})
+					chunks.append(chunk)
+					res.append({
+						"media_type": JobType.COLLECTION,
+						"canonical_name": canonical_name,
+						"items": chunks
+					})
+			else:
+				for local_file in local_files:
+					res.append({
+						"local_media_path": self.rename_local_file(local_file),
+						"performer": performer,
+						"canonical_name": canonical_name,
+						"media_type": job_type
+					})
+		logging.debug(res)
+		return res
+
+	def _download(self, url: str, timeout: int = 60) -> list:
+		res = []
+		post_text = ""
+		pw_proxy = None
+		if self.proxy:
+			dsn = self.proxy.get("dsn", "")
+			if dsn:
+				parsed = urlparse(dsn)
+				pw_proxy = {
+					"server": f"{parsed.scheme}://{parsed.hostname}:{parsed.port}",
+					"username": parsed.username,
+					"password": parsed.password
+				}
+				logging.info("[X] build proxy: %s", pw_proxy)
+
+		contains_images, contains_videos = False, False
+		images, videos = [], []
+		with sync_playwright() as p:
+			with p.chromium.launch(headless=True) as browser:
+				with browser.new_context(proxy=pw_proxy, ignore_https_errors=True) as context:
+					page = context.new_page()
+					page.goto(url, wait_until="networkidle", timeout=(timeout*1000))
+					page.wait_for_selector("article[role='article']", timeout=(timeout*1000))
+
+					contains_videos = self.tweet_contains_video(page)
+					contains_images = self.tweet_contains_images(page)
+
+					if contains_images:
+						post_text, images = self.download_images(page, timeout)
+
+					if not contains_images and not contains_videos:
+						post_text = self.extract_post_text(page)
+
+		if contains_videos:
+			media_info, videos = self.download_videos(url, timeout)
+			if media_info:
+				post_text = self.extract_canonical_name(media_info)
+
+		if not images and not videos:
+			if not post_text:
+				raise Unavailable("Content unvailable")
+			logging.info("[X]: Sending text message")
+			res.append({
+				"message_text": post_text,
+				"media_type": JobType.TEXT
+			})
+			return res
+		
+		if len(images) > 1 or len(videos) > 1:
+			logging.info("[X]: uploading collection")
+			content = images + videos
+			res.extend(self.generate_result(content, JobType.COLLECTION, canonical_name=post_text))
+		else:
+			logging.info("[X]: uploading media")
+			for job_type, content in {JobType.IMAGE: images, JobType.VIDEO: videos}.items():
+				if content:
+					res.extend(self.generate_result(content, job_type, canonical_name=post_text))
+
+		return res
+
+	def download_videos(self, url: str, timeout: int = 60) -> tuple[dict, list[str]]:
+		local_files = []
+		media_info = {}
+		time_name = str(time.time()).replace('.', '_')
+		ydl_opts = {
+			'socket_timeout': timeout,
+			'outtmpl': f'{self.DOWNLOAD_DIR}/x_download_{time_name}_%(id)s.%(ext)s',
+			'quiet': False,
+			'force_generic_extractor': False,
+			#'noplaylist': True,
+			'merge_output_format': 'mp4',
+			'dump_single_json': False,
+			'nocheckcertificate': True,
+			'progress_hooks': [self.dlp_on_progress],
+		}
+		if self.proxy:
+			proxy_dsn = self.proxy.get("dsn", "")
+			logging.info("[X] Using proxy DSN '%s'", proxy_dsn)
+			if proxy_dsn:
+				ydl_opts["proxy"] = proxy_dsn
+
+		with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+			info = ydl.extract_info(url, download=False)
+			media_info = info
+			entries = info.get("entries", [info])
+
+			for entry in entries:
+				ret = ydl.download([entry['webpage_url']])
+				if ret == 0:
+					file_path = ydl.prepare_filename(entry)
+					if isinstance(file_path, str):
+						local_files.append(file_path)
+					else:
+						local_files.extend(file_path)
+
+		return media_info, local_files
+
+	def adaptive_chunk_size(self, content_length: int) -> int:
+		if content_length < 100_000:
+			return 2048
+		elif content_length < 5_000_000:
+			return 8192
+		elif content_length < 100_000_000:
+			return 32768
+		else:
+			return 65536
+
+	def get_extension_from_headers(self, response: requests.Response) -> str:
+		content_type = response.headers.get("Content-Type", "")
+		return guess_extension(content_type) or ".jpg"
+
+	def download_images(self, page: Page, timeout: int) -> tuple[str, list[str]]:
+		downloaded_imgs = []
+		headers = {
+			"User-Agent": ScraperUtils.get_ua(),
+			"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+			"Accept-Language": "en-us,en;q=0.5",
+			"Sec-Fetch-Mode": "navigate"
+		}
+		proxies = None
+		if self.proxy:
+			proxies = {"https": self.proxy.get("dsn", ""), "http": self.proxy.get("dsn", "")}
+
+		image_urls, post_text = self.extract_image_urls_from_x_post(page, timeout)
+
+		if not image_urls:
+			logging.error("[X] Content images are not found!")
+			return downloaded_imgs
+
+		time_name = str(time.time()).replace('.', '_')
+		for i, img_url in enumerate(set(image_urls)):
+			downloaded = 0
+			if "?name=small" in img_url:
+				img_url = img_url.replace("?name=small", "?name=orig")
+			with requests.get(
+				img_url,
+				headers=headers,
+				timeout=timeout,
+				stream=True,
+				verify=False,
+				proxies=proxies) as request:
+				
+				request.raise_for_status()
+
+				parsed = urlparse(img_url)
+				ext = os.path.splitext(parsed.path)[1]
+				if not ext:
+					ext = self.get_extension_from_headers(request)
+				filename = f"x_download_{time_name}_{i}{ext}"
+				filepath = os.path.join(self.DOWNLOAD_DIR, filename)
+
+				content_length = int(request.headers.get("Content-Length", 0))
+
+				with open(filepath, "wb") as f:
+					#request.raw.decode_content = True
+					chunk_size = self.adaptive_chunk_size(content_length)
+					for chunk in request.iter_content(chunk_size=chunk_size):
+						if chunk:
+							f.write(chunk)
+							downloaded += len(chunk)
+							self.download_progress(
+								total=content_length or None,
+								bytes_transferred=downloaded,
+								path=filepath
+							)
+				downloaded_imgs.append(filepath)
+
+		return post_text, downloaded_imgs
+
+	def extract_image_urls_from_x_post(self, page: Page, timeout: int) -> tuple[list[str], str]:
+		img_urls, post_text = [], ''
+
+		page.wait_for_selector("img[src*='pbs.twimg.com/media']", timeout=(timeout*1000))
+		post_text = self.extract_post_text(page)
+
+		image_elements = page.query_selector_all("img")
+		image_urls = []
+
+		for img in image_elements:
+			src = img.get_attribute("src")
+			if src and "pbs.twimg.com/media" in src:
+				image_urls.append(src)
+					
+		img_urls = list(set(image_urls))
+		return img_urls, post_text
+	
+	def tweet_contains_video(self, page: Page) -> bool:
+		try:
+			return bool(
+				page.query_selector("article video") or
+				page.query_selector("div[data-testid='videoPlayer']") or
+				page.query_selector("div[aria-label='Embedded video']")
+			)
+		except Exception:
+			pass
+		return False
+		
+	def tweet_contains_images(self, page: Page) -> bool:
+		try:
+			image_elements = page.query_selector_all("img")
+			image_urls = [
+				img.get_attribute("src")
+				for img in image_elements
+				if img.get_attribute("src") and "pbs.twimg.com/media" in img.get_attribute("src")
+			]
+			return bool(image_urls)
+		except Exception:
+			pass
+		return False
+	
+	def extract_post_text(self, page: Page) -> str:
+		try:
+			text_fragments = []
+
+			# find tweetText containers (in main and quoted)
+			containers = page.query_selector_all('div[data-testid="tweetText"]')
+			for container in containers:
+				fragments = []
+
+				# find <span> and <img alt=...> inside text
+				for node in container.query_selector_all("span, img"):
+					tag = node.evaluate("node => node.tagName.toLowerCase()")
+					if tag == "span":
+						value = node.inner_text().strip()
+						if value:
+							fragments.append(value)
+					elif tag == "img":
+						# emoji as image
+						alt = node.get_attribute("alt")
+						if alt:
+							fragments.append(alt)
+
+				if fragments:
+					text_fragments.append("".join(fragments))
+
+			return "\n\n".join(text_fragments).strip()
+
+		except Exception as e:
+			logging.warning("X: [extract_post_text] error", exc_info=e)
+		return ""

warp_beacon-2.8.15/warp_beacon/scraper/X/types.py

@@ -0,0 +1,8 @@
+from enum import Flag, auto
+
+class XMediaType(Flag):
+	UNKNOWN = 0
+	VIDEO = auto()
+	IMAGE = auto()
+	MIXED = auto()
+	PLAYLIST = auto()

{warp_beacon-2.8.13 → warp_beacon-2.8.15}/warp_beacon/scraper/__init__.py

@@ -318,7 +318,8 @@ class AsyncDownloader(object):
 							if items:
 								# success
 								for job in fail_handler.get_failed_jobs():
-									self.queue_task(job)
+									self.queue_task(job["job"])
+								# media info processing
 								for item in items:
 									media_info = {"filesize": 0}
 									if item["media_type"] == JobType.VIDEO:
@@ -372,6 +373,8 @@ class AsyncDownloader(object):
 										job_args["media_collection"] = item["items"]
 										if item.get("save_items", None) is not None:
 											job_args["save_items"] = item.get("save_items", False)
+									elif item["media_type"] == JobType.TEXT:
+										job_args["message_text"] = item.get("message_text", "")
 									else:
 										job_args["local_media_path"] = item["local_media_path"]
 										if item.get("local_compressed_media_path", None):

{warp_beacon-2.8.13 → warp_beacon-2.8.15}/warp_beacon/scraper/fail_handler.py

@@ -16,12 +16,15 @@ class FailHandler(object):
 		self.client.close()
 
 	def store_failed_job(self, job: DownloadJob) -> int:
-		db_id = -1
+		db_id = ""
 		try:
 			job_serilized = pickle.dumps(job)
 			db_id = self.db.insert_one(
 			{
-				"job_data": job_serilized
+				"job_data": job_serilized,
+				"uniq_id": job.uniq_id,
+				"message_id": job.message_id,
+				"chat_id": job.chat_id
 			}).inserted_id
 		except Exception as e:
 			logging.error("Failed to store job as failed!")
@@ -33,10 +36,26 @@ class FailHandler(object):
 		try:
 			cursor = self.db.find()
 			for document in cursor:
-				ret.append(pickle.loads(document["job_data"]))
+				ret.append({
+					"_id": document["_id"],
+					"job": pickle.loads(document["job_data"]),
+					"uniq_id": document.get("uniq_id"),
+					"message_id": document.get("message_id"),
+					"chat_id": document.get("chat_id")
+				})
 			if clean:
 				self.db.delete_many({})
 		except Exception as e:
 			logging.error("Failed to get failed jobs!")
 			logging.exception(e)
 		return ret
+	
+	def remove_failed_job(self, uniq_id: str) -> bool:
+		try:
+			result = self.db.delete_one({"uniq_id": uniq_id})
+			if result.deleted_count > 0:
+				return True
+		except Exception as e:
+			logging.error("Failed to remove failed job!", exc_info=e)
+
+		return False

{warp_beacon-2.8.13 → warp_beacon-2.8.15}/warp_beacon/storage/__init__.py

@@ -67,7 +67,8 @@ class Storage(object):
 					"uniq_id": document["uniq_id"],
 					"tg_file_id": document["tg_file_id"],
 					"media_type": document["media_type"],
-					"canonical_name": document.get("canonical_name")
+					"canonical_name": document.get("canonical_name"),
+					"message_text": document.get("message_text")
 				})
 		except Exception as e:
 			logging.error("Error occurred while trying to read from the database!")
@@ -82,13 +83,10 @@ class Storage(object):
 	def db_lookup_id(self, uniq_id: str) -> list[dict]:
 		return self.db_find(uniq_id)
 	
-	def add_media(self, tg_file_ids: list[str], media_url: str, media_type: str, origin: str, canonical_name: str = "") -> list[int]:
+	def add_media(self, tg_file_ids: list[str], media_url: str, media_type: str, origin: str, canonical_name: str = "", message_text: str = "") -> list[int]:
 		uniq_id = self.compute_uniq(media_url)
 		media_ids = []
 		for tg_file_id in tg_file_ids:
-			if not tg_file_id:
-				logging.warning("Passed empty `tg_file_id`! Skipping.")
-				continue
 			if self.db_lookup_id(uniq_id):
 				logging.info("Detected existing uniq_id, skipping storage write operation")
 				continue
@@ -98,7 +96,8 @@ class Storage(object):
 				"media_type": media_type,
 				"tg_file_id": tg_file_id,
 				"origin": origin,
-				"canonical_name": canonical_name
+				"canonical_name": canonical_name,
+				"message_text": message_text
 			}).inserted_id)
 
 		return media_ids

{warp_beacon-2.8.13 → warp_beacon-2.8.15}/warp_beacon/telegram/bot.py

@@ -393,6 +393,9 @@ class Bot(object):
 							tg_chunk.append(anim)
 					mediafs.append(tg_chunk)
 				args["media"] = mediafs
+		elif job.media_type == JobType.TEXT:
+			args["text"] = f"<b>Post text:</b><pre>{job.message_text}</pre>\n\n{self.build_signature_caption(job)}"
+			args["parse_mode"] = ParseMode.HTML
 
 		args["chat_id"] = job.chat_id
 
@@ -412,7 +415,7 @@ class Bot(object):
 			if render_donates:
 				keyboard_buttons[0].append(InlineKeyboardButton("❤ Donate", url=os.environ.get("DONATE_LINK", "https://pay.cryptocloud.plus/pos/W5BMtNQt5bJFoW2E")))
 
-			if keyboard_buttons[0]:  #job.short_text or render_donates:
+			if keyboard_buttons[0]:
 				args["reply_markup"] = InlineKeyboardMarkup(keyboard_buttons)
 
 		return args
@@ -425,9 +428,14 @@ class Bot(object):
 			while not retry_amount >= max_retries:
 				try:
 					reply_message = None
-					if job.media_type in (JobType.VIDEO, JobType.IMAGE, JobType.AUDIO, JobType.ANIMATION):
-						if job.media_type in (JobType.VIDEO, JobType.AUDIO):
-							await Utils.ensure_me_loaded(self.client)
+					if job.media_type in (JobType.VIDEO, JobType.IMAGE, JobType.AUDIO, JobType.ANIMATION, JobType.TEXT):
+						#if job.media_type in (JobType.VIDEO, JobType.AUDIO):
+						#	await Utils.ensure_me_loaded(self.client)
+						if job.media_type == JobType.TEXT:
+							if job.placeholder_message_id:
+								await self.placeholder.remove(job.chat_id, job.placeholder_message_id)
+								job.placeholder_message_id = None
+
 						if job.placeholder_message_id:
 							try:
 								reply_message = await self.editor.edit(**self.build_tg_args(job))
@@ -440,7 +448,8 @@ class Bot(object):
 								JobType.VIDEO: self.client.send_video,
 								JobType.IMAGE: self.client.send_photo,
 								JobType.AUDIO: self.client.send_audio,
-								JobType.ANIMATION: self.client.send_animation
+								JobType.ANIMATION: self.client.send_animation,
+								JobType.TEXT: self.client.send_message
 							}
 							try:
 								while True:
@@ -460,9 +469,10 @@ class Bot(object):
 									job_args[reality.value.lower()] = job_args.pop(expectation.value.lower())
 									reply_message = await send_funcs[reality](**job_args)
 
-						tg_file_id = Utils.extract_file_id(reply_message)
-						tg_file_ids.append(tg_file_id)
-						job.tg_file_id = tg_file_id
+						if reply_message:
+							tg_file_id = Utils.extract_file_id(reply_message)
+							tg_file_ids.append(tg_file_id)
+							job.tg_file_id = tg_file_id
 						logging.info("Uploaded media file with type '%s' tg_file_id is '%s'", job.media_type.value, job.tg_file_id)
 					elif job.media_type == JobType.COLLECTION:
 						col_job_args = self.build_tg_args(job)

{warp_beacon-2.8.13 → warp_beacon-2.8.15}/warp_beacon/telegram/handlers.py

@@ -14,8 +14,6 @@ from warp_beacon.jobs.upload_job import UploadJob
 from warp_beacon.jobs import Origin
 from warp_beacon.jobs.types import JobType
 from warp_beacon.scraper.link_resolver import LinkResolver
-from warp_beacon.scraper.fail_handler import FailHandler
-from warp_beacon.storage.mongo import DBClient
 
 class Handlers(object):
 	storage = None
@@ -25,12 +23,7 @@ class Handlers(object):
 	def __init__(self, bot: "Bot") -> None:
 		self.bot = bot
 		self.storage = bot.storage
-		# add uploader callbacks to handle service restart
-		for job in FailHandler(DBClient()).get_failed_jobs(clean=False):
-			self.bot.uploader.add_callback(
-				job.placeholder_message_id,
-				self.upload_wrapper
-			)
+		self.bot.uploader.uploader_wrapper = self.upload_wrapper
 
 	async def help(self, _: Client, message: Message) -> None:
 		"""Send a message when the command /help is issued."""
@@ -111,6 +104,14 @@ class Handlers(object):
 						origin=job.job_origin.value,
 						canonical_name=common_canonical_name
 					)
+				elif job.media_type == JobType.TEXT:
+					self.storage.add_media(
+						tg_file_ids=[None],
+						media_url=job.url,
+						media_type=job.media_type.value,
+						origin=job.job_origin.value,
+						message_text=job.message_text
+					)
 				else:
 					self.storage.add_media(
 						tg_file_ids=[','.join(tg_file_ids)],
@@ -139,11 +140,6 @@ class Handlers(object):
 					text="Failed to create message placeholder. Please check your bot Internet connection."
 				)
 
-			self.bot.uploader.add_callback(
-				job.placeholder_message_id,
-				self.upload_wrapper
-			)
-
 			self.bot.downloader.queue_task(job)
 		except Exception as e:
 			logging.error("Failed to schedule download task!")
@@ -215,6 +211,7 @@ class Handlers(object):
 					elif ent_len:
 						media_type = JobType[entities[0]["media_type"].upper()]
 						canonical_name = entities[0]["canonical_name"]
+						message_text = entities[0]["message_text"]
 						await self.bot.upload_job(
 							UploadJob(
 								url=url,
@@ -228,22 +225,23 @@ class Handlers(object):
 								chat_type=message.chat.type,
 								source_username=Utils.extract_message_author(message),
 								canonical_name=canonical_name,
-								message_leftover=msg_leftover
+								message_leftover=msg_leftover,
+								message_text=message_text
 							)
 						)
 				else:
 					if await self.queue_job(DownloadJob.build(
-							url=url,
-							message_id=effective_message_id,
-							chat_id=chat.id,
-							user_id=message.from_user.id,
-							in_process=self.bot.uploader.is_inprocess(uniq_id),
-							uniq_id=uniq_id,
-							job_origin=origin,
-							source_username=Utils.extract_message_author(message),
-							chat_type=chat.type,
-							message_leftover=msg_leftover
-						)):
+						url=url,
+						message_id=effective_message_id,
+						chat_id=chat.id,
+						user_id=message.from_user.id,
+						in_process=self.bot.uploader.is_inprocess(uniq_id),
+						uniq_id=uniq_id,
+						job_origin=origin,
+						source_username=Utils.extract_message_author(message),
+						chat_type=chat.type,
+						message_leftover=msg_leftover
+					)):
 						self.bot.uploader.set_inprocess(uniq_id)
 
 		if chat.type not in (ChatType.GROUP, ChatType.SUPERGROUP) and not urls: