warp-beacon 2.7.31__tar.gz → 2.8.0__tar.gz

{warp_beacon-2.7.31/warp_beacon.egg-info → warp_beacon-2.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.7.31
+Version: 2.8.0
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

{warp_beacon-2.7.31 → warp_beacon-2.8.0}/setup.py

@@ -52,6 +52,7 @@ setup(
 		'warp_beacon/scraper',
 		'warp_beacon/scraper/instagram',
 		'warp_beacon/scraper/youtube',
+		'warp_beacon/scraper/X',
 		'warp_beacon/mediainfo',
 		'warp_beacon/jobs',
 		'warp_beacon/compress',
@@ -91,6 +92,9 @@ setup(
 		"warp_beacon/scraper/youtube/youtube",
 		"warp_beacon/scraper/youtube/shorts",
 		"warp_beacon/scraper/youtube/music",
+		"warp_beacon/scraper/X/abstract",
+		"warp_beacon/scraper/X/X",
+		"warp_beacon/scraper/X/types",
 		"warp_beacon/scraper/fail_handler",
 		"warp_beacon/scraper/link_resolver",
 		"warp_beacon/scraper/utils",

warp_beacon-2.8.0/warp_beacon/__version__.py

@@ -0,0 +1,2 @@
+__version__ = "2.8.0"
+

{warp_beacon-2.7.31 → warp_beacon-2.8.0}/warp_beacon/jobs/__init__.py

@@ -6,4 +6,5 @@ class Origin(Enum):
 	YT_MUSIC = "yt_music"
 	YOUTUBE = "youtube"
 	YOUTU_BE = "youtu_be"
+	X = "X"
 	UNKNOWN = "unknown"

{warp_beacon-2.7.31 → warp_beacon-2.8.0}/warp_beacon/scheduler/instagram_human.py

@@ -5,7 +5,7 @@ from datetime import datetime
 
 import logging
 
-from instagrapi.types import UserShort
+from instagrapi.types import UserShort, Story
 from warp_beacon.scraper.instagram.instagram import InstagramScraper
 
 class InstagramHuman(object):
@@ -69,7 +69,7 @@ class InstagramHuman(object):
 
 	def watch_stories(self) -> None:
 		logging.info("Simulating stories watch ...")
-		stories = []
+		stories: list[Story] = []
 		try:
 			raw_tray = self.reel_tray_feed_if_needed()
 			if not raw_tray:
@@ -77,16 +77,17 @@ class InstagramHuman(object):
 				return
 			#logging.info("raw_tray: %s", str(raw_tray))
 			tray = raw_tray.get("tray", [])
-			if tray:
+			filtered_tray = [item for item in tray if item.get("seen", 0) == 0]
+			if filtered_tray:
 				reels_tray = []
 				target_len = random.randint(1, 5)
-				for el in tray[:target_len]:
+				for el in filtered_tray[:target_len]:
 					user = el["user"]
 					user_id = user["pk"]
-					if el.get("seen", 0) == 0:
-						reels_tray.append({"user_id": user_id})
+					reels_tray.append({"user_id": user_id})
 
 				for el in reels_tray:
+					# amount?
 					_stories = self.scrapler.download_hndlr(self.scrapler.cl.user_stories, el["user_id"])
 					self.operations_count += 1
 					if _stories:
@@ -106,7 +107,16 @@ class InstagramHuman(object):
 						explore_user = m.user
 						self.operations_count += 1
 						break
-					self.random_pause()
+					pause = 0.0
+					if m.media_type == 2:
+						pause = m.video_duration or random.uniform(2, 20)
+					elif m.media_type == 8:
+						pause = random.uniform(2, 31)
+					else: # img, etc.
+						pause = random.uniform(2, 14)
+					logging.info("Pause for '%.2f' sec ...", round(pause, 2))
+					time.sleep(pause)
+					#self.random_pause()
 
 			if seen:
 				self.scrapler.download_hndlr(self.scrapler.cl.media_seen, seen)

warp_beacon-2.8.0/warp_beacon/scraper/X/X.py

@@ -0,0 +1,215 @@
+import os
+import time
+import logging
+from mimetypes import guess_extension
+from urllib.parse import urlparse
+import requests
+import yt_dlp
+from playwright.sync_api import sync_playwright
+
+from warp_beacon.scraper.utils import ScraperUtils
+from warp_beacon.scraper.X.types import XMediaType
+from warp_beacon.jobs.types import JobType
+from warp_beacon.scraper.X.abstract import XAbstract
+
+class XScraper(XAbstract):
+	DOWNLOAD_DIR = "/tmp"
+
+	def extract_canonical_name(self, media: dict) -> str:
+		ret = ""
+		try:
+			if media.get("title", None):
+				ret = media["title"]
+			if media.get("description", ""):
+				ret += "\n" + media["description"]
+		except Exception as e:
+			logging.warning("Failed to extract canonical media name!")
+			logging.exception(e)
+
+		return ret
+
+	def get_media_type(self, media_info: dict) -> XMediaType:
+		media_type = XMediaType.UNKNOWN
+		#logging.info("[X] post info: '%s'", media_info)
+
+		if 'ext' in media_info:
+			logging.info("[X] Format: '%s'", media_info['ext'])
+		if 'formats' in media_info:
+			logging.info("[X] Contains video.")
+			media_type = XMediaType.VIDEO
+		elif 'thumbnails' in media_info:
+			logging.info("[X] contains images.")
+			media_type = XMediaType.IMAGE
+		else:
+			logging.info("[X] No media found.")
+
+		return media_type
+
+	def _download(self, url: str, timeout: int = 60) -> list:
+		res = []
+		job_type = JobType.UNKNOWN
+		time_name = str(time.time()).replace('.', '_')
+		ydl_opts = {
+			'socket_timeout': timeout,
+			'outtmpl': f'{self.DOWNLOAD_DIR}/x_download_{time_name}.%(ext)s',
+			'quiet': False,
+			'force_generic_extractor': False,
+			'noplaylist': True,
+			'merge_output_format': 'mp4',
+			'dump_single_json': True,
+		}
+
+		if self.proxy:
+			proxy_dsn = self.proxy.get("dsn", "")
+			logging.info("[X] Using proxy DSN '%s'", proxy_dsn)
+			if proxy_dsn:
+				ydl_opts["proxy"] = proxy_dsn
+
+		local_file, media_info, media_type, post_text = "", {}, XMediaType.UNKNOWN, ""
+		with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+			try:
+				media_info = ydl.extract_info(url, download=False)
+				media_type = self.get_media_type(media_info)
+				if media_type == XMediaType.VIDEO:
+					local_file = self.download_video(url, ydl, media_info)
+					post_text = self.extract_canonical_name(media_info)
+					job_type = JobType.VIDEO
+			except yt_dlp.utils.DownloadError:
+				logging.warning("[X] yt_dlp failed to extract info. Falling back to image scraping.")
+				media_type = XMediaType.IMAGE
+
+		if media_type == XMediaType.IMAGE:
+			job_type = JobType.IMAGE
+			images, post_text = self.download_images(url, timeout)
+			if images:
+				local_file = images[0]
+
+		if local_file:
+			res.append({
+				"local_media_path": local_file,
+				"performer": media_info.get("uploader", "Unknown"),
+				'progress_hooks': [self.dlp_on_progress],
+				#"thumb": thumbnail,
+				"canonical_name": post_text,
+				"media_type": job_type
+			})
+
+		return res
+
+	def adaptive_chunk_size(self, content_length: int) -> int:
+		if content_length < 100_000:
+			return 2048
+		elif content_length < 5_000_000:
+			return 8192
+		elif content_length < 100_000_000:
+			return 32768
+		else:
+			return 65536
+
+	def download_video(self, url: str, ydl: yt_dlp.YoutubeDL, media_info: dict) -> str:
+		local_file = ""
+		ydl.download([url])
+		local_file = ydl.prepare_filename(media_info)
+		logging.debug("Temp filename: '%s'", local_file)
+		if local_file:
+			local_file = self.rename_local_file(local_file)
+		return local_file
+
+	def get_extension_from_headers(self, response: requests.Response) -> str:
+		content_type = response.headers.get("Content-Type", "")
+		return guess_extension(content_type) or ".jpg"
+
+	def download_images(self, url: str, timeout: int = 60) -> tuple[list[str], str]:
+		downloaded_imgs = []
+		headers = {
+			"User-Agent": ScraperUtils.get_ua(),
+			"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+			"Accept-Language": "en-us,en;q=0.5",
+			"Sec-Fetch-Mode": "navigate"
+		}
+		proxies = None
+		if self.proxy:
+			proxies = {"https": self.proxy.get("dsn", ""), "http": self.proxy.get("dsn", "")}
+
+		image_urls, post_text = self.extract_image_urls_from_x_post(url, timeout=timeout)
+
+		if not image_urls:
+			logging.error("[X] Content images are not found!")
+			return downloaded_imgs
+
+		time_name = str(time.time()).replace('.', '_')
+		for i, img_url in enumerate(set(image_urls)):
+			downloaded = 0
+			if "?name=small" in img_url:
+				img_url = img_url.replace("?name=small", "?name=orig")
+			with requests.get(
+				img_url,
+				headers=headers,
+				timeout=timeout,
+				stream=True,
+				verify=False,
+				proxies=proxies) as request:
+				
+				request.raise_for_status()
+
+				parsed = urlparse(img_url)
+				ext = os.path.splitext(parsed.path)[1]
+				if not ext:
+					ext = self.get_extension_from_headers(request)
+				filename = f"x_download_{time_name}_{i}{ext}"
+				filepath = os.path.join(self.DOWNLOAD_DIR, filename)
+
+				content_length = int(request.headers.get("Content-Length", 0))
+
+				with open(filepath, "wb") as f:
+					#request.raw.decode_content = True
+					chunk_size = self.adaptive_chunk_size(content_length)
+					for chunk in request.iter_content(chunk_size=chunk_size):
+						if chunk:
+							f.write(chunk)
+							downloaded += len(chunk)
+							self.download_progress(
+								total=content_length or None,
+								bytes_transferred=downloaded,
+								path=filepath
+							)
+				downloaded_imgs.append(filepath)
+
+		return downloaded_imgs, post_text
+	
+	def extract_image_urls_from_x_post(self, url: str, timeout: int = 60) -> tuple[list[str], str]:
+		img_urls, post_text = [], ''
+
+		proxy = None
+		if self.proxy:
+			dsn = self.proxy.get("dsn", "")
+			if dsn:
+				parsed = urlparse(dsn)
+				proxy = {
+					"server": f"{parsed.scheme}://{parsed.hostname}:{parsed.port}",
+					"username": parsed.username,
+					"password": parsed.password
+				}
+
+		with sync_playwright() as p:
+			with p.chromium.launch(headless=True) as browser:
+				with browser.new_context(proxy=proxy) as context:
+					page = context.new_page()
+					page.goto(url, wait_until="networkidle", timeout=(timeout*1000))
+
+					#page.wait_for_timeout(3000)
+					page.wait_for_selector("img[src*='pbs.twimg.com/media']", timeout=(timeout*1000))
+					text_element = page.wait_for_selector('[data-testid="tweetText"]', timeout=(timeout*1000))
+					text = text_element.inner_text()
+
+					image_elements = page.query_selector_all("img")
+					image_urls = []
+
+					for img in image_elements:
+						src = img.get_attribute("src")
+						if src and "pbs.twimg.com/media" in src:
+							image_urls.append(src)
+					
+					img_urls = list(set(image_urls))
+					post_text = str(text)
+		return img_urls, post_text

warp_beacon-2.8.0/warp_beacon/scraper/X/abstract.py

@@ -0,0 +1,139 @@
+import http.client
+import io
+import logging
+import os
+import socket
+import ssl
+import time
+import urllib
+from typing import Callable, Optional, Union
+
+import playwright
+import playwright.sync_api
+import requests
+import urllib3
+#from pytubefix.exceptions import VideoUnavailable, VideoPrivate, MaxRetriesExceeded
+import yt_dlp
+
+from warp_beacon.jobs.download_job import DownloadJob
+from warp_beacon.scraper.abstract import ScraperAbstract
+from warp_beacon.scraper.exceptions import (BadProxy, TimeOut, Unavailable,
+                                            extract_exception_message)
+from warp_beacon.telegram.types import ReportType
+
+
+class XAbstract(ScraperAbstract):
+	DOWNLOAD_DIR = "/tmp"
+	X_MAX_RETRIES_DEFAULT = 8
+	X_PAUSE_BEFORE_RETRY_DEFAULT = 3
+	X_TIMEOUT_DEFAULT = 15
+	X_TIMEOUT_INCREMENT_DEFAULT = 20
+
+	def __init__(self, account: tuple, proxy: dict=None) -> None:
+		super().__init__(account, proxy)
+		self._download_progress_threshold = 0
+	
+	def validate_session(self) -> int:
+		return 0
+
+	def download_hndlr(self, func: Callable, *args: tuple[Union[str, int, dict, tuple, bool]], **kwargs: dict[Union[str, int, dict, tuple, bool]]) -> Optional[Union[list, dict, str, io.BytesIO]]:
+		ret_val = None
+		max_retries = int(os.environ.get("X_MAX_RETRIES", default=self.X_MAX_RETRIES_DEFAULT))
+		pause_secs = int(os.environ.get("X_PAUSE_BEFORE_RETRY", default=self.X_PAUSE_BEFORE_RETRY_DEFAULT))
+		timeout = int(os.environ.get("X_TIMEOUT", default=self.X_TIMEOUT_DEFAULT))
+		timeout_increment = int(os.environ.get("X_TIMEOUT_INCREMENT", default=self.X_TIMEOUT_INCREMENT_DEFAULT))
+		retries = 0
+		while max_retries >= retries:
+			try:
+				kwargs["timeout"] = timeout
+				ret_val = func(*args, **kwargs)
+				break
+			except urllib3.exceptions.ProxyError as e:
+				logging.warning("Proxy error!")
+				raise BadProxy(extract_exception_message(e.original_error))
+			except (socket.timeout,
+					ssl.SSLError,
+					http.client.IncompleteRead,
+					http.client.HTTPException,
+					requests.RequestException,
+					urllib.error.URLError,
+					urllib.error.HTTPError,
+					playwright.sync_api.TimeoutError) as e:
+				if hasattr(e, "code") and (int(e.code) == 403 or int(e.code) == 400):
+					raise Unavailable(extract_exception_message(e))
+				if hasattr(e, "reason") and "Remote end closed connection without response" in str(e.reason):
+					raise Unavailable(extract_exception_message(e))
+				logging.warning("X read timeout! Retrying in '%d' seconds ...", pause_secs)
+				logging.info("Your `X_MAX_RETRIES` values is '%d'", max_retries)
+				logging.exception(extract_exception_message(e))
+				if max_retries <= retries:
+					#self.remove_tmp_files()
+					raise TimeOut(extract_exception_message(e))
+				retries += 1
+				timeout += timeout_increment
+				time.sleep(pause_secs)
+			except yt_dlp.utils.DownloadError as e:
+				raise Unavailable(extract_exception_message(e))
+			except yt_dlp.utils.GeoRestrictedError:
+				raise Unavailable(extract_exception_message(e))
+			except yt_dlp.utils.PostProcessingError as e:
+				raise Unavailable(extract_exception_message(e))
+			except yt_dlp.utils.ExtractorError as e:
+				raise Unavailable(extract_exception_message(e))
+			except yt_dlp.utils.MaxDownloadsReached as e:
+				raise Unavailable(extract_exception_message(e))
+			except yt_dlp.utils.UnavailableVideoError as e:
+				raise Unavailable(extract_exception_message(e))
+			except yt_dlp.utils.ThrottledDownload as e:
+				raise Unavailable(extract_exception_message(e))
+
+		return ret_val
+
+	def download_progress(self, total: int | None, bytes_transferred: int, path: str) -> None:
+		if not total:
+			return
+		percentage_of_completion = round(bytes_transferred / (total or 1) * 100)
+		if percentage_of_completion >= self._download_progress_threshold:
+			logging.debug("[Download] X file '%s', %d", path, percentage_of_completion)
+			msg = {
+				"action": "report_download_status",
+				"current": bytes_transferred,
+				"total": total or 0,
+				"message_id": self.job.placeholder_message_id,
+				"chat_id": self.job.chat_id,
+				"completed": percentage_of_completion >= 100,
+				"report_type": ReportType.PROGRESS
+			}
+			self.status_pipe.send(msg)
+			self._download_progress_threshold += 20
+
+	def dlp_on_progress(self, params: dict) -> None:
+		if params.get("status", "") == "downloading":
+			total_size = int(params.get("total_bytes") or params.get("total_bytes_estimate") or 0)
+			if not total_size or total_size < 0:
+				logging.warning("[Download worker][yt_dlp]: total_size is '%d'", total_size)
+				return
+			bytes_downloaded = int(params.get("downloaded_bytes", 0))
+			percentage_of_completion = bytes_downloaded / (total_size or 1) * 100
+			if total_size == 0 or percentage_of_completion >= self._download_progress_threshold:
+				msg = {
+					"action": "report_download_status",
+					"current": bytes_downloaded,
+					"total": total_size,
+					"message_id": self.job.placeholder_message_id,
+					"chat_id": self.job.chat_id,
+					"completed": percentage_of_completion >= 100,
+					"report_type": ReportType.PROGRESS
+				}
+				self.status_pipe.send(msg)
+				logging.debug("[Download worker][yt_dlp] Downloaded %d%%", percentage_of_completion)
+				if total_size > 0:
+					self._download_progress_threshold += 20
+
+	def _download(self, url: str, timeout: int = 60) -> list:
+		raise NotImplementedError("You should to implement _download method")
+
+	def download(self, job: DownloadJob) -> list:
+		self.job = job
+		ret = self.download_hndlr(self._download, job.url)
+		return ret

warp_beacon-2.8.0/warp_beacon/scraper/X/types.py

@@ -0,0 +1,6 @@
+import enum
+
+class XMediaType(enum.Enum):
+	UNKNOWN = 0
+	VIDEO = 1
+	IMAGE = 2

{warp_beacon-2.7.31 → warp_beacon-2.8.0}/warp_beacon/scraper/__init__.py

@@ -148,6 +148,10 @@ class AsyncDownloader(object):
 							elif job.job_origin is Origin.YOUTUBE:
 								from warp_beacon.scraper.youtube.youtube import YoutubeScraper
 								actor = YoutubeScraper(selector.get_current(), proxy)
+							elif job.job_origin is Origin.X:
+								from warp_beacon.scraper.X.X import XScraper
+								actor = XScraper(selector.get_current(), proxy)
+
 							actor.send_message_to_admin_func = self.send_message_to_admin
 							actor.request_yt_auth = self.request_yt_auth
 							actor.auth_event = self.auth_event

{warp_beacon-2.7.31 → warp_beacon-2.8.0}/warp_beacon/scraper/account_selector.py

@@ -111,12 +111,15 @@ class AccountSelector(object):
 	def get_last_proxy(self) -> Optional[dict]:
 		return self.accounts_meta_data.get("last_proxy", None)
 
-	def get_proxy_list(self) -> List[dict]:
+	def get_proxy_list(self, ipv4: bool = False) -> List[dict]:
 		matched_proxy = []
 		try:
 			acc_id, acc_data = self.get_current()
 			current_acc_pid = acc_data.get("proxy_id", "").strip()
 			for proxy in self.proxies:
+				if ipv4:
+					if proxy.get("ip_version", '') not in ("v4", "both"):
+						continue
 				pid = proxy.get("id", "").strip()
 				if pid and current_acc_pid and pid == current_acc_pid:
 					if "override_force_ipv6" in proxy:
@@ -129,10 +132,10 @@ class AccountSelector(object):
 
 		return matched_proxy
 
-	def get_random_account_proxy(self) -> Optional[dict]:
+	def get_random_account_proxy(self, ipv4: bool = False) -> Optional[dict]:
 		if self.proxies:
 			try:
-				matched_proxy = self.get_proxy_list()
+				matched_proxy = self.get_proxy_list(ipv4)
 				if matched_proxy:
 					if len(matched_proxy) > 1:
 						random.seed(random.seed(time.time_ns() ^ int.from_bytes(os.urandom(len(matched_proxy)), "big")))
@@ -204,7 +207,10 @@ class AccountSelector(object):
 			if not self.current.get("enabled", True):
 				logging.info("Account '%d' is disabled. Probing next ...", idx)
 				self.next()
-		self.current_proxy = self.get_random_account_proxy()
+		ipv4 = False
+		if module_origin is Origin.X:
+			ipv4 = True
+		self.current_proxy = self.get_random_account_proxy(ipv4)
 
 	def next(self) -> dict:
 		idx = self.account_index[self.current_module_name].value

{warp_beacon-2.7.31 → warp_beacon-2.8.0}/warp_beacon/scraper/youtube/abstract.py

@@ -381,11 +381,7 @@ class YoutubeAbstract(ScraperAbstract):
 			logging.warning("Download failed, trying to download with yt_dlp")
 			logging.exception(e)
 		
-		try:
-			ret = self.download_hndlr(self._download_yt_dlp, job.url, thumbnail=thumbnail)
-		except NotImplementedError:
-			logging.info("yt_dlp is not supported for this submodule yet")
-			raise Unavailable("Сontent unvailable")
+		ret = self.download_hndlr(self._download_yt_dlp, job.url, thumbnail=thumbnail)
 
 		return ret
 

{warp_beacon-2.7.31 → warp_beacon-2.8.0}/warp_beacon/storage/__init__.py

@@ -45,9 +45,9 @@ class Storage(object):
 			yt_vid_id = yt_vid_id_list.pop() if yt_vid_id_list else ""
 			if yt_vid_id:
 				path = urlparse(url).path.strip('/').replace("watch", ("yt_music" if parse_mode is UrlParseMode.YT_MUSIC else "youtube"))
-				return ("%s/%s" % (path, yt_vid_id)).strip('/')
+				return f"{path}/{yt_vid_id}".strip('/')
 			else:
-				raise ValueError("Failed to generate uniq_id for url '%s'", url)
+				raise ValueError(f"Failed to generate uniq_id for url '{url}'")
 			
 		path = urlparse(url).path.strip('/')
 		return path

{warp_beacon-2.7.31 → warp_beacon-2.8.0}/warp_beacon/telegram/utils.py

@@ -43,6 +43,9 @@ class Utils(object):
 
 		if "youtube.com/" in url:
 			return Origin.YOUTUBE
+		
+		if "x.com/" in url:
+			return Origin.X
 
 		return Origin.UNKNOWN
 

{warp_beacon-2.7.31 → warp_beacon-2.8.0/warp_beacon.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.7.31
+Version: 2.8.0
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

{warp_beacon-2.7.31 → warp_beacon-2.8.0}/warp_beacon.egg-info/SOURCES.txt

@@ -43,6 +43,10 @@ warp_beacon/scraper/exceptions.py
 warp_beacon/scraper/fail_handler.py
 warp_beacon/scraper/link_resolver.py
 warp_beacon/scraper/utils.py
+warp_beacon/scraper/X/X.py
+warp_beacon/scraper/X/__init__.py
+warp_beacon/scraper/X/abstract.py
+warp_beacon/scraper/X/types.py
 warp_beacon/scraper/instagram/__init__.py
 warp_beacon/scraper/instagram/captcha.py
 warp_beacon/scraper/instagram/instagram.py

{warp_beacon-2.7.31 → warp_beacon-2.8.0}/warp_beacon.egg-info/top_level.txt

@@ -15,6 +15,10 @@ warp_beacon/scheduler
 warp_beacon/scheduler/instagram_human
 warp_beacon/scheduler/scheduler
 warp_beacon/scraper
+warp_beacon/scraper/X
+warp_beacon/scraper/X/X
+warp_beacon/scraper/X/abstract
+warp_beacon/scraper/X/types
 warp_beacon/scraper/abstract
 warp_beacon/scraper/account_selector
 warp_beacon/scraper/exceptions

warp_beacon-2.7.31/warp_beacon/__version__.py

@@ -1,2 +0,0 @@
-__version__ = "2.7.31"
-