warp-beacon 2.8.10__tar.gz → 2.8.11__tar.gz

{warp_beacon-2.8.10/warp_beacon.egg-info → warp_beacon-2.8.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.8.10
+Version: 2.8.11
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev
@@ -242,6 +242,7 @@ Requires-Dist: pydub
 Requires-Dist: SpeechRecognition
 Requires-Dist: playwright
 Requires-Dist: fake-useragent
+Requires-Dist: pyotp
 Dynamic: author
 Dynamic: home-page
 Dynamic: license-file

{warp_beacon-2.8.10 → warp_beacon-2.8.11}/pyproject.toml

@@ -30,7 +30,8 @@ dependencies = [
 	"pydub",
 	"SpeechRecognition",
 	"playwright",
-	"fake-useragent"
+	"fake-useragent",
+	"pyotp"
 ]
 keywords = ["bot", "telegram", "instagram", "export"]
 classifiers = [

warp_beacon-2.8.11/warp_beacon/__version__.py

@@ -0,0 +1,2 @@
+__version__ = "2.8.11"
+

{warp_beacon-2.8.10 → warp_beacon-2.8.11}/warp_beacon/scraper/X/X.py

@@ -59,6 +59,7 @@ class XScraper(XAbstract):
 			'merge_output_format': 'mp4',
 			'dump_single_json': True,
 			'nocheckcertificate': True,
+			'progress_hooks': [self.dlp_on_progress],
 		}
 
 		if self.proxy:
@@ -68,6 +69,16 @@ class XScraper(XAbstract):
 				ydl_opts["proxy"] = proxy_dsn
 
 		local_file, media_info, media_type, post_text = "", {}, XMediaType.UNKNOWN, ""
+		#tweet_contains_video, tweet_contains_images = False, False
+
+		#with sync_playwright() as p:
+		#	with p.chromium.launch(headless=True) as browser:
+		#		with browser.new_context(proxy=proxy, ignore_https_errors=True) as context:
+		#			page = context.new_page()
+		#			page.goto(url, wait_until="networkidle", timeout=(timeout*1000))
+		#			tweet_contains_video = self.tweet_contains_video(page)
+		#			tweet_contains_images = self.tweet_contains_images(page)
+
 		with yt_dlp.YoutubeDL(ydl_opts) as ydl:
 			try:
 				media_info = ydl.extract_info(url, download=False)
@@ -262,4 +273,58 @@ class XScraper(XAbstract):
 							image_urls.append(src)
 					
 					img_urls = list(set(image_urls))
-		return img_urls, post_text
+		return img_urls, post_text
+	
+	def get_media_type_from_info_and_dom(self, media_info: dict, page: Page) -> XMediaType:
+		is_video = (
+			media_info.get("vcodec") != "none" or
+			media_info.get("ext") in {"mp4", "mov", "mkv"} or
+			any(
+				f.get("vcodec") not in (None, "none")
+				for f in media_info.get("formats", [])
+			)
+		)
+
+		try:
+			image_elements = page.query_selector_all("img")
+			image_urls = [
+				img.get_attribute("src")
+				for img in image_elements
+				if img.get_attribute("src") and "pbs.twimg.com/media" in img.get_attribute("src")
+			]
+			has_images = bool(image_urls)
+		except Exception:
+			has_images = False
+
+		if is_video and has_images:
+			return XMediaType.MIXED
+		elif is_video:
+			return XMediaType.VIDEO
+		elif has_images:
+			return XMediaType.IMAGE
+		
+		return XMediaType.UNKNOWN
+	
+	def tweet_contains_video(self, page: Page) -> bool:
+		try:
+			return bool(
+				page.query_selector("article video") or
+				page.query_selector("div[data-testid='videoPlayer']") or
+				page.query_selector("div[aria-label='Embedded video']")
+			)
+		except Exception:
+			pass
+		return False
+		
+	def tweet_contains_images(self, page: Page) -> bool:
+		try:
+			image_elements = page.query_selector_all("img")
+			image_urls = [
+				img.get_attribute("src")
+				for img in image_elements
+				if img.get_attribute("src") and "pbs.twimg.com/media" in img.get_attribute("src")
+			]
+			return bool(image_urls)
+		except Exception:
+			pass
+		return False

{warp_beacon-2.8.10 → warp_beacon-2.8.11}/warp_beacon/scraper/X/types.py

@@ -3,4 +3,5 @@ import enum
 class XMediaType(enum.Enum):
 	UNKNOWN = 0
 	VIDEO = 1
-	IMAGE = 2
+	IMAGE = 2
+	MIXED = 3

{warp_beacon-2.8.10 → warp_beacon-2.8.11}/warp_beacon/scraper/exceptions.py

@@ -44,6 +44,9 @@ class AllAccountsFailed(ScraperError):
 class BadProxy(ScraperError):
 	pass
 
+class TOTPNotProvided(ScraperError):
+	pass
+
 class UnknownError(ScraperError):
 	pass
 

{warp_beacon-2.8.10 → warp_beacon-2.8.11}/warp_beacon/scraper/instagram/instagram.py

@@ -16,15 +16,17 @@ from urllib.parse import urljoin, urlparse
 import requests
 import urllib3
 
+import pyotp
+
 from instagrapi import exceptions
-from instagrapi.exceptions import UnknownError as IGUnknownError
+from instagrapi.exceptions import TwoFactorRequired, UnknownError as IGUnknownError
 from instagrapi.mixins.story import Story
 from instagrapi.types import Media
 from instagrapi.mixins.challenge import ChallengeChoice
 #from instagrapi.exceptions import LoginRequired, PleaseWaitFewMinutes, MediaNotFound, ClientNotFoundError, UserNotFound, ChallengeRequired, \
 #	ChallengeSelfieCaptcha, ChallengeUnknownStep, UnknownError as IGUnknownError
 
-from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, IGRateLimitOccurred, CaptchaIssue, BadProxy, extract_exception_message
+from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, IGRateLimitOccurred, CaptchaIssue, BadProxy, TOTPNotProvided, extract_exception_message
 from warp_beacon.scraper.abstract import ScraperAbstract
 from warp_beacon.jobs.types import JobType
 from warp_beacon.jobs.download_job import DownloadJob
@@ -112,9 +114,6 @@ class InstagramScraper(ScraperAbstract):
 			logging.info("Loading existing session file '%s'", self.inst_session_file)
 			with open(self.inst_session_file, 'r', encoding="utf-8") as f:
 				js = json.loads(f.read())
-				if "warp_timeline_cursor" in js:
-					self.timeline_cursor = js.get("warp_timeline_cursor", {})
-					del js["warp_timeline_cursor"]
 				self.cl.set_settings(js)
 		else:
 			self.download_hndlr(self.login)
@@ -122,9 +121,20 @@ class InstagramScraper(ScraperAbstract):
 	def login(self) -> None:
 		username = self.account["login"]
 		password = self.account["password"]
+		totp_secret = self.account.get("totp_secret")
 		if username and password:
-			self.cl.login(username=username, password=password, verification_code="")
-		self.safe_write_session()
+			try:
+				self.cl.login(username=username, password=password)
+			except TwoFactorRequired:
+				logging.warning("Two factor required!")
+				if not totp_secret:
+					logging.critical("Please specify TOTP secret in account config")
+					raise TOTPNotProvided("Please specify TOTP secret in account config")
+				totp = pyotp.TOTP(totp_secret)
+				code = totp.now()
+				logging.info("TOTP code for now: '%s'", code)
+				self.cl.login(username=username, password=password, verification_code=code)
+			self.safe_write_session()
 
 	def validate_session(self) -> int:
 		from warp_beacon.scheduler.instagram_human import InstagramHuman

{warp_beacon-2.8.10 → warp_beacon-2.8.11}/warp_beacon/scraper/instagram/wb_instagrapi.py

@@ -9,14 +9,68 @@ import requests
 from instagrapi import Client
 from instagrapi.types import Media, User, Story
 from instagrapi.exceptions import (
-	#ClientError,
+	ClientError,
 	#ClientLoginRequired,
+	ClientNotFoundError,
 	VideoNotDownload,
+	MediaNotFound,
 	PrivateError
 )
 
+from instagrapi.extractors import (
+	extract_location,
+	#extract_media_v1,
+	extract_user_short,
+	extract_usertag,
+	extract_resource_v1
+)
+
 from warp_beacon.scraper.utils import ScraperUtils
 
+def extract_media_v1(data):
+	"""Extract media from Private API"""
+	media = deepcopy(data)
+	if "video_versions" in media:
+		# Select Best Quality by Resolutiuon
+		media["video_url"] = sorted(
+			media["video_versions"], key=lambda o: o["height"] * o["width"]
+		)[-1]["url"]
+	if media["media_type"] == 2 and not media.get("product_type"):
+		media["product_type"] = "feed"
+	if "image_versions2" in media:
+		media["thumbnail_url"] = sorted(
+			media["image_versions2"]["candidates"],
+			key=lambda o: o["height"] * o["width"],
+		)[-1]["url"]
+	if media["media_type"] == 8:
+		# remove thumbnail_url and video_url for albums
+		# see resources
+		media.pop("thumbnail_url", "")
+		media.pop("video_url", "")
+	location = media.get("location")
+	media["location"] = location and extract_location(location)
+	media["user"] = extract_user_short(media.get("user"))
+	media["usertags"] = sorted(
+		[
+			extract_usertag(usertag)
+			for usertag in media.get("usertags", {}).get("in", [])
+		],
+		key=lambda tag: tag.user.pk,
+	)
+	media["like_count"] = media.get("like_count", 0)
+	media["has_liked"] = media.get("has_liked", False)
+	#media["sponsor_tags"] = [tag["sponsor"] for tag in media.get("sponsor_tags", [])]
+	media["sponsor_tags"] = [tag["sponsor"] for tag in media.get("sponsor_tags") or []]
+	media["play_count"] = media.get("play_count", 0)
+	media["coauthor_producers"] = media.get("coauthor_producers", [])
+	return Media(
+		caption_text=(media.get("caption") or {}).get("text", ""),
+		resources=[
+			extract_resource_v1(edge) for edge in media.get("carousel_media", [])
+		],
+		**media,
+	)
+
 class WBClient(Client):
 	"""
 	patched instagrapi
@@ -194,6 +248,30 @@ class WBClient(Client):
 			self._medias_cache[media_pk]
 		)  # return copy of cache (dict changes protection)
 	
+	def media_info_v1(self, media_pk: str) -> Media:
+		"""
+		Get Media from PK by Private Mobile API
+
+		Parameters
+		----------
+		media_pk: str
+			Unique identifier of the media
+
+		Returns
+		-------
+		Media
+			An object of Media type
+		"""
+		try:
+			result = self.private_request(f"media/{media_pk}/info/")
+		except ClientNotFoundError as e:
+			raise MediaNotFound(e, media_pk=media_pk, **self.last_json)
+		except ClientError as e:
+			if "Media not found" in str(e):
+				raise MediaNotFound(e, media_pk=media_pk, **self.last_json)
+			raise e
+		return extract_media_v1(result["items"].pop())
+	
 	def user_info_by_username(self, username: str, use_cache: bool = True) -> User:
 		"""
 		Get user object from username

{warp_beacon-2.8.10 → warp_beacon-2.8.11/warp_beacon.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.8.10
+Version: 2.8.11
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev
@@ -242,6 +242,7 @@ Requires-Dist: pydub
 Requires-Dist: SpeechRecognition
 Requires-Dist: playwright
 Requires-Dist: fake-useragent
+Requires-Dist: pyotp
 Dynamic: author
 Dynamic: home-page
 Dynamic: license-file

{warp_beacon-2.8.10 → warp_beacon-2.8.11}/warp_beacon.egg-info/requires.txt

@@ -16,3 +16,4 @@ pydub
 SpeechRecognition
 playwright
 fake-useragent
+pyotp

warp_beacon-2.8.10/warp_beacon/__version__.py

@@ -1,2 +0,0 @@
-__version__ = "2.8.10"
-