warp-beacon 2.8.10__py3-none-any.whl → 2.8.12__py3-none-any.whl

warp_beacon/__version__.py

@@ -1,2 +1,2 @@
-__version__ = "2.8.10"
+__version__ = "2.8.12"
 

warp_beacon/scraper/X/X.py

@@ -59,6 +59,7 @@ class XScraper(XAbstract):
 			'merge_output_format': 'mp4',
 			'dump_single_json': True,
 			'nocheckcertificate': True,
+			'progress_hooks': [self.dlp_on_progress],
 		}
 
 		if self.proxy:
@@ -68,6 +69,16 @@ class XScraper(XAbstract):
 				ydl_opts["proxy"] = proxy_dsn
 
 		local_file, media_info, media_type, post_text = "", {}, XMediaType.UNKNOWN, ""
+		#tweet_contains_video, tweet_contains_images = False, False
+
+		#with sync_playwright() as p:
+		#	with p.chromium.launch(headless=True) as browser:
+		#		with browser.new_context(proxy=proxy, ignore_https_errors=True) as context:
+		#			page = context.new_page()
+		#			page.goto(url, wait_until="networkidle", timeout=(timeout*1000))
+		#			tweet_contains_video = self.tweet_contains_video(page)
+		#			tweet_contains_images = self.tweet_contains_images(page)
+
 		with yt_dlp.YoutubeDL(ydl_opts) as ydl:
 			try:
 				media_info = ydl.extract_info(url, download=False)
@@ -262,4 +273,58 @@ class XScraper(XAbstract):
 							image_urls.append(src)
 					
 					img_urls = list(set(image_urls))
-		return img_urls, post_text
+		return img_urls, post_text
+	
+	def get_media_type_from_info_and_dom(self, media_info: dict, page: Page) -> XMediaType:
+		is_video = (
+			media_info.get("vcodec") != "none" or
+			media_info.get("ext") in {"mp4", "mov", "mkv"} or
+			any(
+				f.get("vcodec") not in (None, "none")
+				for f in media_info.get("formats", [])
+			)
+		)
+
+		try:
+			image_elements = page.query_selector_all("img")
+			image_urls = [
+				img.get_attribute("src")
+				for img in image_elements
+				if img.get_attribute("src") and "pbs.twimg.com/media" in img.get_attribute("src")
+			]
+			has_images = bool(image_urls)
+		except Exception:
+			has_images = False
+
+		if is_video and has_images:
+			return XMediaType.MIXED
+		elif is_video:
+			return XMediaType.VIDEO
+		elif has_images:
+			return XMediaType.IMAGE
+		
+		return XMediaType.UNKNOWN
+	
+	def tweet_contains_video(self, page: Page) -> bool:
+		try:
+			return bool(
+				page.query_selector("article video") or
+				page.query_selector("div[data-testid='videoPlayer']") or
+				page.query_selector("div[aria-label='Embedded video']")
+			)
+		except Exception:
+			pass
+		return False
+		
+	def tweet_contains_images(self, page: Page) -> bool:
+		try:
+			image_elements = page.query_selector_all("img")
+			image_urls = [
+				img.get_attribute("src")
+				for img in image_elements
+				if img.get_attribute("src") and "pbs.twimg.com/media" in img.get_attribute("src")
+			]
+			return bool(image_urls)
+		except Exception:
+			pass
+		return False

warp_beacon/scraper/X/types.py

@@ -3,4 +3,5 @@ import enum
 class XMediaType(enum.Enum):
 	UNKNOWN = 0
 	VIDEO = 1
-	IMAGE = 2
+	IMAGE = 2
+	MIXED = 3

warp_beacon/scraper/exceptions.py

@@ -44,6 +44,9 @@ class AllAccountsFailed(ScraperError):
 class BadProxy(ScraperError):
 	pass
 
+class TOTPNotProvided(ScraperError):
+	pass
+
 class UnknownError(ScraperError):
 	pass
 

warp_beacon/scraper/instagram/instagram.py

@@ -16,15 +16,17 @@ from urllib.parse import urljoin, urlparse
 import requests
 import urllib3
 
+import pyotp
+
 from instagrapi import exceptions
-from instagrapi.exceptions import UnknownError as IGUnknownError
+from instagrapi.exceptions import TwoFactorRequired, UnknownError as IGUnknownError
 from instagrapi.mixins.story import Story
 from instagrapi.types import Media
 from instagrapi.mixins.challenge import ChallengeChoice
 #from instagrapi.exceptions import LoginRequired, PleaseWaitFewMinutes, MediaNotFound, ClientNotFoundError, UserNotFound, ChallengeRequired, \
 #	ChallengeSelfieCaptcha, ChallengeUnknownStep, UnknownError as IGUnknownError
 
-from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, IGRateLimitOccurred, CaptchaIssue, BadProxy, extract_exception_message
+from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, IGRateLimitOccurred, CaptchaIssue, BadProxy, TOTPNotProvided, extract_exception_message
 from warp_beacon.scraper.abstract import ScraperAbstract
 from warp_beacon.jobs.types import JobType
 from warp_beacon.jobs.download_job import DownloadJob
@@ -112,9 +114,6 @@ class InstagramScraper(ScraperAbstract):
 			logging.info("Loading existing session file '%s'", self.inst_session_file)
 			with open(self.inst_session_file, 'r', encoding="utf-8") as f:
 				js = json.loads(f.read())
-				if "warp_timeline_cursor" in js:
-					self.timeline_cursor = js.get("warp_timeline_cursor", {})
-					del js["warp_timeline_cursor"]
 				self.cl.set_settings(js)
 		else:
 			self.download_hndlr(self.login)
@@ -122,9 +121,20 @@ class InstagramScraper(ScraperAbstract):
 	def login(self) -> None:
 		username = self.account["login"]
 		password = self.account["password"]
+		totp_secret = self.account.get("totp_secret")
 		if username and password:
-			self.cl.login(username=username, password=password, verification_code="")
-		self.safe_write_session()
+			try:
+				self.cl.login(username=username, password=password)
+			except TwoFactorRequired:
+				logging.warning("Two factor required!")
+				if not totp_secret:
+					logging.critical("Please specify TOTP secret in account config")
+					raise TOTPNotProvided("Please specify TOTP secret in account config")
+				totp = pyotp.TOTP(totp_secret)
+				code = totp.now()
+				logging.info("TOTP code for now: '%s'", code)
+				self.cl.login(username=username, password=password, verification_code=code)
+			self.safe_write_session()
 
 	def validate_session(self) -> int:
 		from warp_beacon.scheduler.instagram_human import InstagramHuman
@@ -393,7 +403,7 @@ class InstagramScraper(ScraperAbstract):
 				if os.path.exists(self.inst_session_file):
 					os.unlink(self.inst_session_file)
 				raise IGRateLimitOccurred("Instagram ratelimit")
-			except (exceptions.MediaNotFound, exceptions.ClientNotFoundError, exceptions.UserNotFound) as e:
+			except (exceptions.MediaNotFound, exceptions.ClientNotFoundError, exceptions.UserNotFound, exceptions.StoryNotFound) as e:
 				raise NotFound(extract_exception_message(e))
 			except IGUnknownError as e:
 				raise UnknownError(extract_exception_message(e))

warp_beacon/scraper/instagram/wb_instagrapi.py

@@ -9,14 +9,68 @@ import requests
 from instagrapi import Client
 from instagrapi.types import Media, User, Story
 from instagrapi.exceptions import (
-	#ClientError,
+	ClientError,
 	#ClientLoginRequired,
+	ClientNotFoundError,
 	VideoNotDownload,
+	MediaNotFound,
 	PrivateError
 )
 
+from instagrapi.extractors import (
+	extract_location,
+	#extract_media_v1,
+	extract_user_short,
+	extract_usertag,
+	extract_resource_v1
+)
+
 from warp_beacon.scraper.utils import ScraperUtils
 
+def extract_media_v1(data):
+	"""Extract media from Private API"""
+	media = deepcopy(data)
+	if "video_versions" in media:
+		# Select Best Quality by Resolutiuon
+		media["video_url"] = sorted(
+			media["video_versions"], key=lambda o: o["height"] * o["width"]
+		)[-1]["url"]
+	if media["media_type"] == 2 and not media.get("product_type"):
+		media["product_type"] = "feed"
+	if "image_versions2" in media:
+		media["thumbnail_url"] = sorted(
+			media["image_versions2"]["candidates"],
+			key=lambda o: o["height"] * o["width"],
+		)[-1]["url"]
+	if media["media_type"] == 8:
+		# remove thumbnail_url and video_url for albums
+		# see resources
+		media.pop("thumbnail_url", "")
+		media.pop("video_url", "")
+	location = media.get("location")
+	media["location"] = location and extract_location(location)
+	media["user"] = extract_user_short(media.get("user"))
+	media["usertags"] = sorted(
+		[
+			extract_usertag(usertag)
+			for usertag in media.get("usertags", {}).get("in", [])
+		],
+		key=lambda tag: tag.user.pk,
+	)
+	media["like_count"] = media.get("like_count", 0)
+	media["has_liked"] = media.get("has_liked", False)
+	#media["sponsor_tags"] = [tag["sponsor"] for tag in media.get("sponsor_tags", [])]
+	media["sponsor_tags"] = [tag["sponsor"] for tag in media.get("sponsor_tags") or []]
+	media["play_count"] = media.get("play_count", 0)
+	media["coauthor_producers"] = media.get("coauthor_producers", [])
+	return Media(
+		caption_text=(media.get("caption") or {}).get("text", ""),
+		resources=[
+			extract_resource_v1(edge) for edge in media.get("carousel_media", [])
+		],
+		**media,
+	)
+
 class WBClient(Client):
 	"""
 	patched instagrapi
@@ -194,6 +248,30 @@ class WBClient(Client):
 			self._medias_cache[media_pk]
 		)  # return copy of cache (dict changes protection)
 	
+	def media_info_v1(self, media_pk: str) -> Media:
+		"""
+		Get Media from PK by Private Mobile API
+
+		Parameters
+		----------
+		media_pk: str
+			Unique identifier of the media
+
+		Returns
+		-------
+		Media
+			An object of Media type
+		"""
+		try:
+			result = self.private_request(f"media/{media_pk}/info/")
+		except ClientNotFoundError as e:
+			raise MediaNotFound(e, media_pk=media_pk, **self.last_json)
+		except ClientError as e:
+			if "Media not found" in str(e):
+				raise MediaNotFound(e, media_pk=media_pk, **self.last_json)
+			raise e
+		return extract_media_v1(result["items"].pop())
+	
 	def user_info_by_username(self, username: str, use_cache: bool = True) -> User:
 		"""
 		Get user object from username

{warp_beacon-2.8.10.dist-info → warp_beacon-2.8.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.8.10
+Version: 2.8.12
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev
@@ -242,6 +242,7 @@ Requires-Dist: pydub
 Requires-Dist: SpeechRecognition
 Requires-Dist: playwright
 Requires-Dist: fake-useragent
+Requires-Dist: pyotp
 Dynamic: author
 Dynamic: home-page
 Dynamic: license-file

{warp_beacon-2.8.10.dist-info → warp_beacon-2.8.12.dist-info}/RECORD

@@ -4,7 +4,7 @@ var/warp_beacon/accounts.json,sha256=OsXdncs6h88xrF_AP6_WDCK1waGBn9SR-uYdIeK37GM
 var/warp_beacon/placeholder.gif,sha256=cE5CGJVaop4Sx21zx6j4AyoHU0ncmvQuS2o6hJfEH88,6064
 var/warp_beacon/proxies.json,sha256=VnjlQDXumOEq72ZFjbh6IqHS1TEHqn8HPYAZqWCeSIA,95
 warp_beacon/__init__.py,sha256=_rThNODmz0nDp_n4mWo_HKaNFE5jk1_7cRhHyYaencI,163
-warp_beacon/__version__.py,sha256=Ou4DLKTMIKPZuLHA1_MlWO0lsUTdjc49PjxWE4oYrZU,24
+warp_beacon/__version__.py,sha256=ioCIkmzTwVXvR1lLWlgUjehGwiXaxB8kTGnfQ-3C1FA,24
 warp_beacon/warp_beacon.py,sha256=ADCR30uGXIsDrt9WoiI9Ghu2QtWs0qZIK6x3pQKM_B4,1109
 warp_beacon/yt_auth.py,sha256=GUTKqYr_tzDC-07Lx_ahWXSag8EyLxXBUnQbDBIkEmk,6022
 warp_beacon/compress/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -25,18 +25,18 @@ warp_beacon/scheduler/scheduler.py,sha256=0u9AIr9fTBmjU1GpOwKlPuNLskeJ4A-S2uAUzb
 warp_beacon/scraper/__init__.py,sha256=o9-HQEf4yQVNtWuJN4NcLUovejiHhP_KkQ1Xf5EaQvU,20670
 warp_beacon/scraper/abstract.py,sha256=pWbaTu-gDZgi-iFjqMR_uGzPl5KLv-4gTdJ9w6cD4sk,3802
 warp_beacon/scraper/account_selector.py,sha256=n-466AiTXZ8o5cgcNkNwNiWLoi-EkLC7bHh6El1eIF8,10274
-warp_beacon/scraper/exceptions.py,sha256=EKwoF0oH2xZWbNU-v8DOaWK5skKwa3s1yTIBdlcfMpc,1452
+warp_beacon/scraper/exceptions.py,sha256=hicAe6_0xN7Ry2gcFX4UvqPWMtF_lX2ihH1njQAaqCA,1496
 warp_beacon/scraper/fail_handler.py,sha256=5ODu4b8ndZWAcHIXrcUufsWFihetzNUoAi8IgAkreyQ,998
 warp_beacon/scraper/link_resolver.py,sha256=Rc9ZuMyOo3iPywDHwjngy-WRQ2SXhJwxcg-5ripx7tM,2447
 warp_beacon/scraper/utils.py,sha256=AOZmDki2Pbr84IG-j_wN2UghKCiWFVDYdx6HJl0JTBs,1258
-warp_beacon/scraper/X/X.py,sha256=KSAXBqIbm5gMbYHYcIXjc7dx3ZW_Qkz2mV-kQ6CrrfM,7953
+warp_beacon/scraper/X/X.py,sha256=3-GBXHcSmGIiDLucJWLpiuaA8EDfP0-B7ws4MiCelkE,9883
 warp_beacon/scraper/X/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scraper/X/abstract.py,sha256=pCzZPTCtn8pRbBx2SeuBUpMkEHqnOLtwLBAHYceL12Q,5475
-warp_beacon/scraper/X/types.py,sha256=i36Nu2cHpHCkvoeobBQC3B13Ke_N40tgCCApcm_FBFY,76
+warp_beacon/scraper/X/types.py,sha256=9Y0PJo3vZ1DMQcyfqoE4y2-AQRAetVmIxQwFDZkZy30,87
 warp_beacon/scraper/instagram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scraper/instagram/captcha.py,sha256=9UYziuqB3Tsat_ET6ex-cnZDbi6yCnsXHSpmE8MuUHk,4651
-warp_beacon/scraper/instagram/instagram.py,sha256=5rD8kPBk6j59a7Wi1TL6W04KHX6zNO1QYMo3H1OsJHQ,18749
-warp_beacon/scraper/instagram/wb_instagrapi.py,sha256=TgpCjFudeEf2VN0qKgaurFcHECsAlqDc1vM4t7tMScs,9202
+warp_beacon/scraper/instagram/instagram.py,sha256=uzqUCVniRa3d9uavoMAz9-9MHvYOh7n_G7UyfgzHgAk,19154
+warp_beacon/scraper/instagram/wb_instagrapi.py,sha256=piPtcN3pB_obsaPX36Sdm2JxEM7d99ZUTwsd4fJY2B4,11544
 warp_beacon/scraper/youtube/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scraper/youtube/abstract.py,sha256=7CVR2fW6bpWYYKcveRddd6XlgDsfV_Pp3dqV2LpefSc,16088
 warp_beacon/scraper/youtube/music.py,sha256=5AeSBQyUgVCJT2hoBCV2WvlyuV9US09SYJhmBG_P9F8,2755
@@ -56,9 +56,9 @@ warp_beacon/telegram/progress_file_reader.py,sha256=e3equyNKlKs764AD-iE9QRsh3YDH
 warp_beacon/telegram/types.py,sha256=Kvdng6uCF1HRoqQgGW1ZYYPJoVuYkFb-LDvMBbW5Hjk,89
 warp_beacon/telegram/utils.py,sha256=zTF8VQfAWetBSjAPbmNe_Zi_LN5fAcWptJKjLaFNHaE,5073
 warp_beacon/uploader/__init__.py,sha256=1enK6qMWaTZEaK456JwaKOfvCvznHA8cjgceOsrF6Po,5732
-warp_beacon-2.8.10.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-warp_beacon-2.8.10.dist-info/METADATA,sha256=TymhuXptNclz7M2y-swiwHmk7pFrGgD83ehUHDP1mHs,23215
-warp_beacon-2.8.10.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-warp_beacon-2.8.10.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
-warp_beacon-2.8.10.dist-info/top_level.txt,sha256=RraB0PWGvRK2zPYkuICKNgStLG1C5s7rPHHJEHJbkgA,1510
-warp_beacon-2.8.10.dist-info/RECORD,,
+warp_beacon-2.8.12.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+warp_beacon-2.8.12.dist-info/METADATA,sha256=W7U_kzv74XYVaeNArKB7Gw7UmwGBe2T4Ow6WYmUSl6Y,23236
+warp_beacon-2.8.12.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+warp_beacon-2.8.12.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
+warp_beacon-2.8.12.dist-info/top_level.txt,sha256=RraB0PWGvRK2zPYkuICKNgStLG1C5s7rPHHJEHJbkgA,1510
+warp_beacon-2.8.12.dist-info/RECORD,,