warp-beacon 2.8.13__tar.gz → 2.8.14__tar.gz

{warp_beacon-2.8.13/warp_beacon.egg-info → warp_beacon-2.8.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.8.13
+Version: 2.8.14
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

warp_beacon-2.8.14/warp_beacon/__version__.py

@@ -0,0 +1,2 @@
+__version__ = "2.8.14"
+

{warp_beacon-2.8.13 → warp_beacon-2.8.14}/warp_beacon/jobs/abstract.py

@@ -109,6 +109,9 @@ class AbstractJob(ABC):
 		if self.media_type == JobType.COLLECTION:
 			if not self.media_collection:
 				return True
+		elif self.media_type == JobType.TEXT:
+			if not self.message_text:
+				return True
 		elif not self.local_media_path:
 			return True
 		return False
@@ -129,9 +132,13 @@ class AbstractJob(ABC):
 				for j in i:
 					if os.path.exists(j.local_media_path):
 						os.unlink(j.local_media_path)
+		elif self.media_type == JobType.TEXT:
+			pass
 		else:
 			if os.path.exists(self.local_media_path):
 				os.unlink(self.local_media_path)
 			if self.local_compressed_media_path:
 				if os.path.exists(self.local_compressed_media_path):
 					os.unlink(self.local_compressed_media_path)
+
+		return True

{warp_beacon-2.8.13 → warp_beacon-2.8.14}/warp_beacon/jobs/types.py

@@ -6,4 +6,5 @@ class JobType(str, Enum):
 	IMAGE = "image",
 	AUDIO = "audio",
 	COLLECTION = "collection"
-	ANIMATION = "animation"
+	ANIMATION = "animation"
+	TEXT = "text"

warp_beacon-2.8.14/warp_beacon/scraper/X/X.py

@@ -0,0 +1,304 @@
+import os
+import time
+import logging
+from mimetypes import guess_extension, guess_type
+from urllib.parse import urlparse
+import requests
+import yt_dlp
+from playwright.sync_api import sync_playwright, Page
+
+from warp_beacon.telegram.utils import Utils
+from warp_beacon.scraper.utils import ScraperUtils
+from warp_beacon.jobs.types import JobType
+from warp_beacon.scraper.X.abstract import XAbstract
+
+from warp_beacon.scraper.exceptions import Unavailable
+
+class XScraper(XAbstract):
+	DOWNLOAD_DIR = "/tmp"
+
+	def extract_canonical_name(self, media: dict) -> str:
+		ret = ""
+		try:
+			if media.get("title", None):
+				ret = media["title"]
+			if media.get("description", ""):
+				ret += "\n" + media["description"]
+		except Exception as e:
+			logging.warning("Failed to extract canonical media name!")
+			logging.exception(e)
+
+		return ret
+
+	def generate_result(self, local_files: list, job_type: JobType, canonical_name: str = "", performer: str = "") -> list:
+		res = []
+		if local_files:
+			if job_type == JobType.COLLECTION:
+				chunks = []
+				for media_chunk in Utils.chunker(local_files, 10):
+					chunk = []
+					for media in media_chunk:
+						mime_type, _ = guess_type(media)
+						chunk.append({
+							"local_media_path": self.rename_local_file(media),
+							"canonical_name": canonical_name,
+							"media_type": JobType.VIDEO if "video" in mime_type else JobType.IMAGE,
+							"media_info": {}
+						})
+					chunks.append(chunk)
+					res.append({
+						"media_type": JobType.COLLECTION,
+						"canonical_name": canonical_name,
+						"items": chunks
+					})
+			else:
+				for local_file in local_files:
+					res.append({
+						"local_media_path": self.rename_local_file(local_file),
+						"performer": performer,
+						"canonical_name": canonical_name,
+						"media_type": job_type
+					})
+		logging.debug(res)
+		return res
+
+	def _download(self, url: str, timeout: int = 60) -> list:
+		res = []
+		post_text = ""
+		pw_proxy = None
+		if self.proxy:
+			dsn = self.proxy.get("dsn", "")
+			if dsn:
+				parsed = urlparse(dsn)
+				pw_proxy = {
+					"server": f"{parsed.scheme}://{parsed.hostname}:{parsed.port}",
+					"username": parsed.username,
+					"password": parsed.password
+				}
+				logging.info("[X] build proxy: %s", pw_proxy)
+
+		contains_images, contains_videos = False, False
+		images, videos = [], []
+		with sync_playwright() as p:
+			with p.chromium.launch(headless=True) as browser:
+				with browser.new_context(proxy=pw_proxy, ignore_https_errors=True) as context:
+					page = context.new_page()
+					page.goto(url, wait_until="networkidle", timeout=(timeout*1000))
+					page.wait_for_selector("article[role='article']", timeout=(timeout*1000))
+
+					contains_videos = self.tweet_contains_video(page)
+					contains_images = self.tweet_contains_images(page)
+
+					if contains_images:
+						post_text, images = self.download_images(page, timeout)
+
+					if not contains_images and not contains_videos:
+						post_text = self.extract_post_text(page)
+
+		if contains_videos:
+			media_info, videos = self.download_videos(url, timeout)
+			if media_info:
+				post_text = self.extract_canonical_name(media_info)
+
+		if not images and not videos:
+			if not post_text:
+				raise Unavailable("Content unvailable")
+			logging.info("[X]: Sending text message")
+			res.append({
+				"message_text": post_text,
+				"media_type": JobType.TEXT
+			})
+			return res
+		
+		if len(images) > 1 or len(videos) > 1:
+			logging.info("[X]: uploading collection")
+			content = images + videos
+			res.extend(self.generate_result(content, JobType.COLLECTION, canonical_name=post_text))
+		else:
+			logging.info("[X]: uploading media")
+			for job_type, content in {JobType.IMAGE: images, JobType.VIDEO: videos}.items():
+				if content:
+					res.extend(self.generate_result(content, job_type, canonical_name=post_text))
+
+		return res
+
+	def download_videos(self, url: str, timeout: int = 60) -> tuple[dict, list[str]]:
+		local_files = []
+		media_info = {}
+		time_name = str(time.time()).replace('.', '_')
+		ydl_opts = {
+			'socket_timeout': timeout,
+			'outtmpl': f'{self.DOWNLOAD_DIR}/x_download_{time_name}_%(id)s.%(ext)s',
+			'quiet': False,
+			'force_generic_extractor': False,
+			#'noplaylist': True,
+			'merge_output_format': 'mp4',
+			'dump_single_json': False,
+			'nocheckcertificate': True,
+			'progress_hooks': [self.dlp_on_progress],
+		}
+		if self.proxy:
+			proxy_dsn = self.proxy.get("dsn", "")
+			logging.info("[X] Using proxy DSN '%s'", proxy_dsn)
+			if proxy_dsn:
+				ydl_opts["proxy"] = proxy_dsn
+
+		with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+			info = ydl.extract_info(url, download=False)
+			media_info = info
+			entries = info.get("entries", [info])
+
+			for entry in entries:
+				ret = ydl.download([entry['webpage_url']])
+				if ret == 0:
+					file_path = ydl.prepare_filename(entry)
+					if isinstance(file_path, str):
+						local_files.append(file_path)
+					else:
+						local_files.extend(file_path)
+
+		return media_info, local_files
+
+	def adaptive_chunk_size(self, content_length: int) -> int:
+		if content_length < 100_000:
+			return 2048
+		elif content_length < 5_000_000:
+			return 8192
+		elif content_length < 100_000_000:
+			return 32768
+		else:
+			return 65536
+
+	def get_extension_from_headers(self, response: requests.Response) -> str:
+		content_type = response.headers.get("Content-Type", "")
+		return guess_extension(content_type) or ".jpg"
+
+	def download_images(self, page: Page, timeout: int) -> tuple[str, list[str]]:
+		downloaded_imgs = []
+		headers = {
+			"User-Agent": ScraperUtils.get_ua(),
+			"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+			"Accept-Language": "en-us,en;q=0.5",
+			"Sec-Fetch-Mode": "navigate"
+		}
+		proxies = None
+		if self.proxy:
+			proxies = {"https": self.proxy.get("dsn", ""), "http": self.proxy.get("dsn", "")}
+
+		image_urls, post_text = self.extract_image_urls_from_x_post(page, timeout)
+
+		if not image_urls:
+			logging.error("[X] Content images are not found!")
+			return downloaded_imgs
+
+		time_name = str(time.time()).replace('.', '_')
+		for i, img_url in enumerate(set(image_urls)):
+			downloaded = 0
+			if "?name=small" in img_url:
+				img_url = img_url.replace("?name=small", "?name=orig")
+			with requests.get(
+				img_url,
+				headers=headers,
+				timeout=timeout,
+				stream=True,
+				verify=False,
+				proxies=proxies) as request:
+				
+				request.raise_for_status()
+
+				parsed = urlparse(img_url)
+				ext = os.path.splitext(parsed.path)[1]
+				if not ext:
+					ext = self.get_extension_from_headers(request)
+				filename = f"x_download_{time_name}_{i}{ext}"
+				filepath = os.path.join(self.DOWNLOAD_DIR, filename)
+
+				content_length = int(request.headers.get("Content-Length", 0))
+
+				with open(filepath, "wb") as f:
+					#request.raw.decode_content = True
+					chunk_size = self.adaptive_chunk_size(content_length)
+					for chunk in request.iter_content(chunk_size=chunk_size):
+						if chunk:
+							f.write(chunk)
+							downloaded += len(chunk)
+							self.download_progress(
+								total=content_length or None,
+								bytes_transferred=downloaded,
+								path=filepath
+							)
+				downloaded_imgs.append(filepath)
+
+		return post_text, downloaded_imgs
+
+	def extract_image_urls_from_x_post(self, page: Page, timeout: int) -> tuple[list[str], str]:
+		img_urls, post_text = [], ''
+
+		page.wait_for_selector("img[src*='pbs.twimg.com/media']", timeout=(timeout*1000))
+		post_text = self.extract_post_text(page)
+
+		image_elements = page.query_selector_all("img")
+		image_urls = []
+
+		for img in image_elements:
+			src = img.get_attribute("src")
+			if src and "pbs.twimg.com/media" in src:
+				image_urls.append(src)
+					
+		img_urls = list(set(image_urls))
+		return img_urls, post_text
+	
+	def tweet_contains_video(self, page: Page) -> bool:
+		try:
+			return bool(
+				page.query_selector("article video") or
+				page.query_selector("div[data-testid='videoPlayer']") or
+				page.query_selector("div[aria-label='Embedded video']")
+			)
+		except Exception:
+			pass
+		return False
+		
+	def tweet_contains_images(self, page: Page) -> bool:
+		try:
+			image_elements = page.query_selector_all("img")
+			image_urls = [
+				img.get_attribute("src")
+				for img in image_elements
+				if img.get_attribute("src") and "pbs.twimg.com/media" in img.get_attribute("src")
+			]
+			return bool(image_urls)
+		except Exception:
+			pass
+		return False
+	
+	def extract_post_text(self, page: Page) -> str:
+		try:
+			text_fragments = []
+
+			# find tweetText containers (in main and quoted)
+			containers = page.query_selector_all('div[data-testid="tweetText"]')
+			for container in containers:
+				fragments = []
+
+				# find <span> and <img alt=...> inside text
+				for node in container.query_selector_all("span, img"):
+					tag = node.evaluate("node => node.tagName.toLowerCase()")
+					if tag == "span":
+						value = node.inner_text().strip()
+						if value:
+							fragments.append(value)
+					elif tag == "img":
+						# emoji as image
+						alt = node.get_attribute("alt")
+						if alt:
+							fragments.append(alt)
+
+				if fragments:
+					text_fragments.append("".join(fragments))
+
+			return "\n\n".join(text_fragments).strip()
+
+		except Exception as e:
+			logging.warning("X: [extract_post_text] error", exc_info=e)
+		return ""

warp_beacon-2.8.14/warp_beacon/scraper/X/types.py

@@ -0,0 +1,8 @@
+from enum import Flag, auto
+
+class XMediaType(Flag):
+	UNKNOWN = 0
+	VIDEO = auto()
+	IMAGE = auto()
+	MIXED = auto()
+	PLAYLIST = auto()

{warp_beacon-2.8.13 → warp_beacon-2.8.14}/warp_beacon/scraper/__init__.py

@@ -319,6 +319,7 @@ class AsyncDownloader(object):
 								# success
 								for job in fail_handler.get_failed_jobs():
 									self.queue_task(job)
+								# media info processing
 								for item in items:
 									media_info = {"filesize": 0}
 									if item["media_type"] == JobType.VIDEO:
@@ -372,6 +373,8 @@ class AsyncDownloader(object):
 										job_args["media_collection"] = item["items"]
 										if item.get("save_items", None) is not None:
 											job_args["save_items"] = item.get("save_items", False)
+									elif item["media_type"] == JobType.TEXT:
+										job_args["message_text"] = item.get("message_text", "")
 									else:
 										job_args["local_media_path"] = item["local_media_path"]
 										if item.get("local_compressed_media_path", None):

{warp_beacon-2.8.13 → warp_beacon-2.8.14}/warp_beacon/storage/__init__.py

@@ -67,7 +67,8 @@ class Storage(object):
 					"uniq_id": document["uniq_id"],
 					"tg_file_id": document["tg_file_id"],
 					"media_type": document["media_type"],
-					"canonical_name": document.get("canonical_name")
+					"canonical_name": document.get("canonical_name"),
+					"message_text": document.get("message_text")
 				})
 		except Exception as e:
 			logging.error("Error occurred while trying to read from the database!")
@@ -82,13 +83,10 @@ class Storage(object):
 	def db_lookup_id(self, uniq_id: str) -> list[dict]:
 		return self.db_find(uniq_id)
 	
-	def add_media(self, tg_file_ids: list[str], media_url: str, media_type: str, origin: str, canonical_name: str = "") -> list[int]:
+	def add_media(self, tg_file_ids: list[str], media_url: str, media_type: str, origin: str, canonical_name: str = "", message_text: str = "") -> list[int]:
 		uniq_id = self.compute_uniq(media_url)
 		media_ids = []
 		for tg_file_id in tg_file_ids:
-			if not tg_file_id:
-				logging.warning("Passed empty `tg_file_id`! Skipping.")
-				continue
 			if self.db_lookup_id(uniq_id):
 				logging.info("Detected existing uniq_id, skipping storage write operation")
 				continue
@@ -98,7 +96,8 @@ class Storage(object):
 				"media_type": media_type,
 				"tg_file_id": tg_file_id,
 				"origin": origin,
-				"canonical_name": canonical_name
+				"canonical_name": canonical_name,
+				"message_text": message_text
 			}).inserted_id)
 
 		return media_ids

{warp_beacon-2.8.13 → warp_beacon-2.8.14}/warp_beacon/telegram/bot.py

@@ -393,6 +393,9 @@ class Bot(object):
 							tg_chunk.append(anim)
 					mediafs.append(tg_chunk)
 				args["media"] = mediafs
+		elif job.media_type == JobType.TEXT:
+			args["text"] = f"<b>Post text:</b><pre>{job.message_text}</pre>\n\n{self.build_signature_caption(job)}"
+			args["parse_mode"] = ParseMode.HTML
 
 		args["chat_id"] = job.chat_id
 
@@ -412,7 +415,7 @@ class Bot(object):
 			if render_donates:
 				keyboard_buttons[0].append(InlineKeyboardButton("❤ Donate", url=os.environ.get("DONATE_LINK", "https://pay.cryptocloud.plus/pos/W5BMtNQt5bJFoW2E")))
 
-			if keyboard_buttons[0]:  #job.short_text or render_donates:
+			if keyboard_buttons[0]:
 				args["reply_markup"] = InlineKeyboardMarkup(keyboard_buttons)
 
 		return args
@@ -425,9 +428,14 @@ class Bot(object):
 			while not retry_amount >= max_retries:
 				try:
 					reply_message = None
-					if job.media_type in (JobType.VIDEO, JobType.IMAGE, JobType.AUDIO, JobType.ANIMATION):
-						if job.media_type in (JobType.VIDEO, JobType.AUDIO):
-							await Utils.ensure_me_loaded(self.client)
+					if job.media_type in (JobType.VIDEO, JobType.IMAGE, JobType.AUDIO, JobType.ANIMATION, JobType.TEXT):
+						#if job.media_type in (JobType.VIDEO, JobType.AUDIO):
+						#	await Utils.ensure_me_loaded(self.client)
+						if job.media_type == JobType.TEXT:
+							if job.placeholder_message_id:
+								await self.placeholder.remove(job.chat_id, job.placeholder_message_id)
+								job.placeholder_message_id = None
+
 						if job.placeholder_message_id:
 							try:
 								reply_message = await self.editor.edit(**self.build_tg_args(job))
@@ -440,7 +448,8 @@ class Bot(object):
 								JobType.VIDEO: self.client.send_video,
 								JobType.IMAGE: self.client.send_photo,
 								JobType.AUDIO: self.client.send_audio,
-								JobType.ANIMATION: self.client.send_animation
+								JobType.ANIMATION: self.client.send_animation,
+								JobType.TEXT: self.client.send_message
 							}
 							try:
 								while True:
@@ -460,9 +469,10 @@ class Bot(object):
 									job_args[reality.value.lower()] = job_args.pop(expectation.value.lower())
 									reply_message = await send_funcs[reality](**job_args)
 
-						tg_file_id = Utils.extract_file_id(reply_message)
-						tg_file_ids.append(tg_file_id)
-						job.tg_file_id = tg_file_id
+						if reply_message:
+							tg_file_id = Utils.extract_file_id(reply_message)
+							tg_file_ids.append(tg_file_id)
+							job.tg_file_id = tg_file_id
 						logging.info("Uploaded media file with type '%s' tg_file_id is '%s'", job.media_type.value, job.tg_file_id)
 					elif job.media_type == JobType.COLLECTION:
 						col_job_args = self.build_tg_args(job)

{warp_beacon-2.8.13 → warp_beacon-2.8.14}/warp_beacon/telegram/handlers.py

@@ -111,6 +111,14 @@ class Handlers(object):
 						origin=job.job_origin.value,
 						canonical_name=common_canonical_name
 					)
+				elif job.media_type == JobType.TEXT:
+					self.storage.add_media(
+						tg_file_ids=[None],
+						media_url=job.url,
+						media_type=job.media_type.value,
+						origin=job.job_origin.value,
+						message_text=job.message_text
+					)
 				else:
 					self.storage.add_media(
 						tg_file_ids=[','.join(tg_file_ids)],
@@ -215,6 +223,7 @@ class Handlers(object):
 					elif ent_len:
 						media_type = JobType[entities[0]["media_type"].upper()]
 						canonical_name = entities[0]["canonical_name"]
+						message_text = entities[0]["message_text"]
 						await self.bot.upload_job(
 							UploadJob(
 								url=url,
@@ -228,22 +237,23 @@ class Handlers(object):
 								chat_type=message.chat.type,
 								source_username=Utils.extract_message_author(message),
 								canonical_name=canonical_name,
-								message_leftover=msg_leftover
+								message_leftover=msg_leftover,
+								message_text=message_text
 							)
 						)
 				else:
 					if await self.queue_job(DownloadJob.build(
-							url=url,
-							message_id=effective_message_id,
-							chat_id=chat.id,
-							user_id=message.from_user.id,
-							in_process=self.bot.uploader.is_inprocess(uniq_id),
-							uniq_id=uniq_id,
-							job_origin=origin,
-							source_username=Utils.extract_message_author(message),
-							chat_type=chat.type,
-							message_leftover=msg_leftover
-						)):
+						url=url,
+						message_id=effective_message_id,
+						chat_id=chat.id,
+						user_id=message.from_user.id,
+						in_process=self.bot.uploader.is_inprocess(uniq_id),
+						uniq_id=uniq_id,
+						job_origin=origin,
+						source_username=Utils.extract_message_author(message),
+						chat_type=chat.type,
+						message_leftover=msg_leftover
+					)):
 						self.bot.uploader.set_inprocess(uniq_id)
 
 		if chat.type not in (ChatType.GROUP, ChatType.SUPERGROUP) and not urls:

{warp_beacon-2.8.13 → warp_beacon-2.8.14}/warp_beacon/uploader/__init__.py

@@ -89,7 +89,7 @@ class AsyncUploader(object):
 		while self.allow_loop:
 			try:
 				try:
-					job = self.job_queue.get()
+					job: UploadJob = self.job_queue.get()
 					if job is self.__JOE_BIDEN_WAKEUP:
 						break
 					if job.is_message_to_admin and job.message_text and self.admin_message_callback:
@@ -118,7 +118,10 @@ class AsyncUploader(object):
 					message_id = job.placeholder_message_id
 
 					if not in_process and not job.job_failed and not job.job_warning and not job.replay:
-						logging.info("Accepted upload job, file(s): '%s'", path)
+						if job.media_type == JobType.TEXT:
+							logging.info("Uploading job text: '%s'", job.message_text)
+						else:
+							logging.info("Accepted upload job, file(s): '%s'", path)
 
 					try:
 						if message_id in self.callbacks:

{warp_beacon-2.8.13 → warp_beacon-2.8.14/warp_beacon.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.8.13
+Version: 2.8.14
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

warp_beacon-2.8.13/warp_beacon/__version__.py

@@ -1,2 +0,0 @@
-__version__ = "2.8.13"
-

warp_beacon-2.8.13/warp_beacon/scraper/X/X.py

@@ -1,330 +0,0 @@
-import os
-import time
-import logging
-from mimetypes import guess_extension
-from urllib.parse import urlparse
-import requests
-import yt_dlp
-from playwright.sync_api import sync_playwright, Page
-
-from warp_beacon.telegram.utils import Utils
-from warp_beacon.scraper.utils import ScraperUtils
-from warp_beacon.scraper.X.types import XMediaType
-from warp_beacon.jobs.types import JobType
-from warp_beacon.scraper.X.abstract import XAbstract
-
-class XScraper(XAbstract):
-	DOWNLOAD_DIR = "/tmp"
-
-	def extract_canonical_name(self, media: dict) -> str:
-		ret = ""
-		try:
-			if media.get("title", None):
-				ret = media["title"]
-			if media.get("description", ""):
-				ret += "\n" + media["description"]
-		except Exception as e:
-			logging.warning("Failed to extract canonical media name!")
-			logging.exception(e)
-
-		return ret
-
-	def get_media_type(self, media_info: dict) -> XMediaType:
-		media_type = XMediaType.UNKNOWN
-		#logging.info("[X] post info: '%s'", media_info)
-
-		if 'ext' in media_info:
-			logging.info("[X] Format: '%s'", media_info['ext'])
-		if 'formats' in media_info:
-			logging.info("[X] Contains video.")
-			media_type = XMediaType.VIDEO
-		elif 'thumbnails' in media_info:
-			logging.info("[X] contains images.")
-			media_type = XMediaType.IMAGE
-		else:
-			logging.info("[X] No media found.")
-
-		return media_type
-
-	def _download(self, url: str, timeout: int = 60) -> list:
-		res = []
-		job_type = JobType.UNKNOWN
-		time_name = str(time.time()).replace('.', '_')
-		ydl_opts = {
-			'socket_timeout': timeout,
-			'outtmpl': f'{self.DOWNLOAD_DIR}/x_download_{time_name}.%(ext)s',
-			'quiet': False,
-			'force_generic_extractor': False,
-			'noplaylist': True,
-			'merge_output_format': 'mp4',
-			'dump_single_json': True,
-			'nocheckcertificate': True,
-			'progress_hooks': [self.dlp_on_progress],
-		}
-
-		if self.proxy:
-			proxy_dsn = self.proxy.get("dsn", "")
-			logging.info("[X] Using proxy DSN '%s'", proxy_dsn)
-			if proxy_dsn:
-				ydl_opts["proxy"] = proxy_dsn
-
-		local_file, media_info, media_type, post_text = "", {}, XMediaType.UNKNOWN, ""
-		#tweet_contains_video, tweet_contains_images = False, False
-
-		#with sync_playwright() as p:
-		#	with p.chromium.launch(headless=True) as browser:
-		#		with browser.new_context(proxy=proxy, ignore_https_errors=True) as context:
-		#			page = context.new_page()
-		#			page.goto(url, wait_until="networkidle", timeout=(timeout*1000))
-		#			tweet_contains_video = self.tweet_contains_video(page)
-		#			tweet_contains_images = self.tweet_contains_images(page)
-
-		with yt_dlp.YoutubeDL(ydl_opts) as ydl:
-			try:
-				media_info = ydl.extract_info(url, download=False)
-				media_type = self.get_media_type(media_info)
-				if media_type == XMediaType.VIDEO:
-					local_file = self.download_video(url, ydl, media_info)
-					post_text = self.extract_canonical_name(media_info)
-					job_type = JobType.VIDEO
-			except yt_dlp.utils.DownloadError as e:
-				msg = str(e).lower()
-				if "no video could be found in this tweet" in msg:
-					logging.warning("[X] yt_dlp failed to extract info. Falling back to image scraping.")
-					media_type = XMediaType.IMAGE
-				else:
-					raise
-		
-		images = []
-		if media_type == XMediaType.IMAGE:
-			job_type = JobType.IMAGE
-			images, post_text = self.download_images(url, timeout)
-			if images:
-				if len(images) > 1:
-					job_type = JobType.COLLECTION
-				else:
-					local_file = images[0]
-
-		if job_type == JobType.COLLECTION:
-			chunks = []
-			for media_chunk in Utils.chunker(images, 10):
-				chunk = []
-				for media in media_chunk:
-					chunk.append({
-						"local_media_path": self.rename_local_file(media),
-						"canonical_name": post_text,
-						"media_type": JobType.IMAGE
-					})
-				chunks.append(chunk)
-			res.append({
-				"media_type": JobType.COLLECTION,
-				"items": chunks
-			})
-		else:
-			if local_file:
-				res.append({
-					"local_media_path": self.rename_local_file(local_file),
-					"performer": media_info.get("uploader", "Unknown"),
-					"canonical_name": post_text,
-					"media_type": job_type
-				})
-
-		return res
-
-	def adaptive_chunk_size(self, content_length: int) -> int:
-		if content_length < 100_000:
-			return 2048
-		elif content_length < 5_000_000:
-			return 8192
-		elif content_length < 100_000_000:
-			return 32768
-		else:
-			return 65536
-
-	def download_video(self, url: str, ydl: yt_dlp.YoutubeDL, media_info: dict) -> str:
-		local_file = ""
-		ydl.download([url])
-		local_file = ydl.prepare_filename(media_info)
-		logging.debug("Temp filename: '%s'", local_file)
-		if local_file:
-			local_file = self.rename_local_file(local_file)
-		return local_file
-
-	def get_extension_from_headers(self, response: requests.Response) -> str:
-		content_type = response.headers.get("Content-Type", "")
-		return guess_extension(content_type) or ".jpg"
-
-	def download_images(self, url: str, timeout: int = 60) -> tuple[list[str], str]:
-		downloaded_imgs = []
-		headers = {
-			"User-Agent": ScraperUtils.get_ua(),
-			"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-			"Accept-Language": "en-us,en;q=0.5",
-			"Sec-Fetch-Mode": "navigate"
-		}
-		proxies = None
-		if self.proxy:
-			proxies = {"https": self.proxy.get("dsn", ""), "http": self.proxy.get("dsn", "")}
-
-		image_urls, post_text = self.extract_image_urls_from_x_post(url, timeout=timeout)
-
-		if not image_urls:
-			logging.error("[X] Content images are not found!")
-			return downloaded_imgs
-
-		time_name = str(time.time()).replace('.', '_')
-		for i, img_url in enumerate(set(image_urls)):
-			downloaded = 0
-			if "?name=small" in img_url:
-				img_url = img_url.replace("?name=small", "?name=orig")
-			with requests.get(
-				img_url,
-				headers=headers,
-				timeout=timeout,
-				stream=True,
-				verify=False,
-				proxies=proxies) as request:
-				
-				request.raise_for_status()
-
-				parsed = urlparse(img_url)
-				ext = os.path.splitext(parsed.path)[1]
-				if not ext:
-					ext = self.get_extension_from_headers(request)
-				filename = f"x_download_{time_name}_{i}{ext}"
-				filepath = os.path.join(self.DOWNLOAD_DIR, filename)
-
-				content_length = int(request.headers.get("Content-Length", 0))
-
-				with open(filepath, "wb") as f:
-					#request.raw.decode_content = True
-					chunk_size = self.adaptive_chunk_size(content_length)
-					for chunk in request.iter_content(chunk_size=chunk_size):
-						if chunk:
-							f.write(chunk)
-							downloaded += len(chunk)
-							self.download_progress(
-								total=content_length or None,
-								bytes_transferred=downloaded,
-								path=filepath
-							)
-				downloaded_imgs.append(filepath)
-
-		return downloaded_imgs, post_text
-
-	def extract_post_text(self, page: Page) -> str:
-		try:
-			tweet_texts = []
-			# collecting text blocks from post
-			containers = page.query_selector_all('div[data-testid="tweetText"]')
-			for container in containers:
-				try:
-					spans = container.query_selector_all("span")
-					if spans:
-						for span in spans:
-							text = span.inner_text().strip()
-							if text:
-								tweet_texts.append(text)
-					else:
-						# to span's try container itself
-						text = container.inner_text().strip()
-						if text:
-							tweet_texts.append(text)
-				except Exception:
-					continue
-
-			return " ".join(tweet_texts).strip()
-		except Exception as e:
-			logging.warning("Failed to extract tweet text.", exc_info=e)
-
-		return ""
-
-	def extract_image_urls_from_x_post(self, url: str, timeout: int = 60) -> tuple[list[str], str]:
-		img_urls, post_text = [], ''
-
-		proxy = None
-		if self.proxy:
-			dsn = self.proxy.get("dsn", "")
-			if dsn:
-				parsed = urlparse(dsn)
-				proxy = {
-					"server": f"{parsed.scheme}://{parsed.hostname}:{parsed.port}",
-					"username": parsed.username,
-					"password": parsed.password
-				}
-				logging.info("[X] build proxy: %s", proxy)
-
-		with sync_playwright() as p:
-			with p.chromium.launch(headless=True) as browser:
-				with browser.new_context(proxy=proxy, ignore_https_errors=True) as context:
-					page = context.new_page()
-					page.goto(url, wait_until="networkidle", timeout=(timeout*1000))
-
-					#page.wait_for_timeout(3000)
-					page.wait_for_selector("img[src*='pbs.twimg.com/media']", timeout=(timeout*1000))
-					post_text = self.extract_post_text(page)
-
-					image_elements = page.query_selector_all("img")
-					image_urls = []
-
-					for img in image_elements:
-						src = img.get_attribute("src")
-						if src and "pbs.twimg.com/media" in src:
-							image_urls.append(src)
-					
-					img_urls = list(set(image_urls))
-		return img_urls, post_text
-	
-	def get_media_type_from_info_and_dom(self, media_info: dict, page: Page) -> XMediaType:
-		is_video = (
-			media_info.get("vcodec") != "none" or
-			media_info.get("ext") in {"mp4", "mov", "mkv"} or
-			any(
-				f.get("vcodec") not in (None, "none")
-				for f in media_info.get("formats", [])
-			)
-		)
-
-		try:
-			image_elements = page.query_selector_all("img")
-			image_urls = [
-				img.get_attribute("src")
-				for img in image_elements
-				if img.get_attribute("src") and "pbs.twimg.com/media" in img.get_attribute("src")
-			]
-			has_images = bool(image_urls)
-		except Exception:
-			has_images = False
-
-		if is_video and has_images:
-			return XMediaType.MIXED
-		elif is_video:
-			return XMediaType.VIDEO
-		elif has_images:
-			return XMediaType.IMAGE
-		
-		return XMediaType.UNKNOWN
-	
-	def tweet_contains_video(self, page: Page) -> bool:
-		try:
-			return bool(
-				page.query_selector("article video") or
-				page.query_selector("div[data-testid='videoPlayer']") or
-				page.query_selector("div[aria-label='Embedded video']")
-			)
-		except Exception:
-			pass
-		return False
-		
-	def tweet_contains_images(self, page: Page) -> bool:
-		try:
-			image_elements = page.query_selector_all("img")
-			image_urls = [
-				img.get_attribute("src")
-				for img in image_elements
-				if img.get_attribute("src") and "pbs.twimg.com/media" in img.get_attribute("src")
-			]
-			return bool(image_urls)
-		except Exception:
-			pass
-		return False

warp_beacon-2.8.13/warp_beacon/scraper/X/types.py

@@ -1,7 +0,0 @@
-import enum
-
-class XMediaType(enum.Enum):
-	UNKNOWN = 0
-	VIDEO = 1
-	IMAGE = 2
-	MIXED = 3