warp-beacon 2.6.4__tar.gz → 2.6.6__tar.gz

{warp_beacon-2.6.4/warp_beacon.egg-info → warp_beacon-2.6.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.6.4
+Version: 2.6.6
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev
@@ -238,6 +238,9 @@ Requires-Dist: pymongo
 Requires-Dist: instagrapi==2.0.0
 Requires-Dist: bs4
 Requires-Dist: yt_dlp
+Requires-Dist: pydub
+Requires-Dist: SpeechRecognition
+Requires-Dist: playwright
 Dynamic: author
 Dynamic: home-page
 Dynamic: license-file

{warp_beacon-2.6.4 → warp_beacon-2.6.6}/pyproject.toml

@@ -26,7 +26,10 @@ dependencies = [
 	"pymongo",
 	"instagrapi == 2.0.0",
 	"bs4",
-	"yt_dlp"
+	"yt_dlp",
+	"pydub",
+	"SpeechRecognition",
+	"playwright"
 ]
 keywords = ["bot", "telegram", "instagram", "export"]
 classifiers = [

warp_beacon-2.6.6/warp_beacon/__version__.py

@@ -0,0 +1,2 @@
+__version__ = "2.6.6"
+

{warp_beacon-2.6.4 → warp_beacon-2.6.6}/warp_beacon/scheduler/scheduler.py

@@ -22,6 +22,7 @@ class IGScheduler(object):
 	def __init__(self, downloader: warp_beacon.scraper.AsyncDownloader) -> None:
 		self.downloader = downloader
 		self.event = threading.Event()
+		self.handle_time_planning()
 
 	def __del__(self) -> None:
 		self.stop()
@@ -63,8 +64,8 @@ class IGScheduler(object):
 			if os.path.exists(self.state_file):
 				with open(self.state_file, 'r', encoding="utf-8") as f:
 					self.state = json.loads(f.read())
-				if "remaining" in self.state:
-					logging.info("Next scheduler activity in '%d' seconds", int(self.state["remaining"]))
+				self.handle_time_planning()
+				logging.info("Next scheduler activity in '%d' seconds", int(self.state["remaining"]))
 			self.load_yt_sessions()
 		except Exception as e:
 			logging.error("Failed to load Scheduler state!")
@@ -124,6 +125,10 @@ class IGScheduler(object):
 	def yt_nearest_expire(self) -> int:
 		return int(min(self.state["yt_sess_exp"], key=lambda x: x.get("expires", 0)).get("expires", 0))
 
+	def handle_time_planning(self) -> None:
+		if int(self.state.get("remaining", 0)) <= 0:
+			self.state["remaining"] = randrange(9292, 26200)
+
 	def do_work(self) -> None:
 		logging.info("Scheduler thread started ...")
 		self.load_state()
@@ -135,22 +140,24 @@ class IGScheduler(object):
 				#max_val = max(yt_expires, ig_sched)
 				now = datetime.datetime.now()
 				if 3 <= now.hour < 7 and min_val != yt_expires:
-					logging.info("Scheduler is paused due to night hours (4:00 - 7:00)")
-					self.state["remaining"] = 10800
+					logging.info("Scheduler is paused due to night hours (3:00 - 7:00)")
+					self.state["remaining"] = 14400
 					self.save_state()
 
 				if ig_sched <= 0:
-					self.state["remaining"] = randrange(9292, 26200)
+					self.handle_time_planning()
 
 				start_time = time.time()
-				logging.info("Next scheduler activity in '%s' seconds", min_val)
+				logging.info("Next scheduler activity in '%s' seconds", int(min_val))
 				logging.info("IG timeout '%d' secs", int(self.state["remaining"]))
 				self.event.wait(timeout=min_val)
+				self.event.clear()
 				elapsed = time.time() - start_time
 				self.state["remaining"] -= elapsed
 
 				if self.running:
-					self.validate_ig_session()
+					if self.state["remaining"] <= 0:
+						self.validate_ig_session()
 					if yt_expires <= time.time() + 60:
 						self.validate_yt_session()
 				self.save_state()

warp_beacon-2.6.6/warp_beacon/scraper/instagram/captcha.py

@@ -0,0 +1,155 @@
+import os
+import time
+import random
+import logging
+import asyncio
+from types import CoroutineType
+from typing import Any
+from urllib.parse import urlparse
+import requests
+from warp_beacon.scraper.instagram.instagram import InstagramScraper
+from pydub import AudioSegment
+import speech_recognition as sr
+from playwright.async_api import async_playwright, Page
+
+class CaptchaSolver(object):
+	TIMEOUT_STANDARD = 7
+	TIMEOUT_SHORT = 1
+	TIMEOUT_DETECTION = 0.05
+	TEMP_DIR = "/tmp"
+
+	scraper = None
+	proxy_config = None
+
+	def __init__(self, scraper: InstagramScraper) -> None:
+		self.scraper = scraper
+		if self.scraper.proxy:
+			dsn = self.scraper.proxy.get("dsn", "")
+			self.proxy_config = self.parse_proxy_from_dsn(dsn)
+
+	def parse_proxy_from_dsn(self, dsn: str) -> dict:
+		parsed = urlparse(dsn)
+		
+		proxy_config = {
+			"server": f"{parsed.scheme}://{parsed.hostname}:{parsed.port}",
+		}
+		
+		if parsed.username and parsed.password:
+			proxy_config["username"] = parsed.username
+			proxy_config["password"] = parsed.password
+
+		return proxy_config
+
+	async def _patch_page(self, page: CoroutineType[Any, Any, Page]):
+		await page.add_init_script("""() => {
+			Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
+			window.chrome = { runtime: {} };
+			Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
+			Object.defineProperty(navigator, 'plugins', { get: () => [1,2,3,4,5] });
+		}""")
+
+	async def solve_audio_captcha(self, page: CoroutineType[Any, Any, Page]) -> None:
+		logging.info("Processing audio captcha ..")
+		mp3_path = os.path.join(self.TEMP_DIR, f"{random.randrange(1,1000)}.mp3")
+		wav_path = os.path.join(self.TEMP_DIR, f"{random.randrange(1,1000)}.wav")
+		try:
+			await page.click('button[aria-label=\"Get an audio challenge\"]')
+			time.sleep(0.3)
+			await page.wait_for_selector('audio', timeout=10000)
+
+			audio_src = await page.get_attribute('audio > source', 'src')
+			audio_content = requests.get(audio_src, timeout=60).content
+			with open(mp3_path, 'wb') as f:
+				f.write(audio_content)
+
+			if not os.path.exists(mp3_path):
+				logging.error("MP3 file not downloaded!")
+				return
+
+			sound = AudioSegment.from_mp3(mp3_path)
+			sound.export(wav_path, format='wav')
+
+			recognizer = sr.Recognizer()
+			with sr.AudioFile(wav_path) as source:
+				audio = recognizer.record(source)
+
+			try:
+				text = recognizer.recognize_google(audio)
+				logging.info("Detected text '%s'", text)
+			except sr.UnknownValueError:
+				logging.error("Failed to detect text!")
+				text = ''
+
+			if text:
+				await page.fill('input[type=\"text\"]', text)
+				await page.press('input[type=\"text\"]', 'Enter')
+				logging.info("Audio captcha solved!")
+		except Exception as e:
+			logging.error("Exception in captcha audio solve!")
+			logging.exception(e)
+		finally:
+			if os.path.exists(mp3_path):
+				os.unlink(mp3_path)
+			if os.path.exists(wav_path):
+				os.unlink(wav_path)
+
+	async def solve_challenge(self, challenge_url: str) -> None:
+		async with async_playwright() as p:
+			browser = None
+			try:
+				browser = await p.chromium.launch(
+					headless=True,
+					args=[
+						"--no-sandbox",
+						"--disable-blink-features=AutomationControlled",
+						"--disable-infobars",
+						"--disable-dev-shm-usage"
+					],
+					proxy=self.proxy_config
+				)
+				context = await browser.new_context(
+					viewport={"width": 1280, "height": 800},
+					user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+					java_script_enabled=True,
+					locale="en-US"
+				)
+
+				page = await context.new_page()
+				await self._patch_page(page)
+
+				await page.goto(challenge_url)
+
+				# finding iframe with captcha
+				frame_element = await page.wait_for_selector('iframe[src*=\"recaptcha\"]')
+				frame = await frame_element.content_frame()
+				time.sleep(0.1)
+
+				# checkbox click
+				checkbox = await frame.wait_for_selector('#recaptcha-anchor', timeout=10000)
+				await checkbox.click()
+
+				# waiting for frame with task
+				await asyncio.sleep(3)
+
+				# checking if iframe with task exists
+				frames = page.frames
+				challenge_frame = None
+				for f in frames:
+					if '/recaptcha/' in f.url and 'bframe' in f.url:
+						challenge_frame = f
+						break
+
+				if not challenge_frame:
+					logging.info("Captcha solved!")
+				else:
+					await self.solve_audio_captcha(challenge_frame)
+			except Exception as e:
+				logging.error("Exception in solver!")
+				logging.exception(e)
+
+			if browser:
+				await asyncio.sleep(10)
+				await browser.close()
+
+	def run(self, challenge_url: str) -> None:
+		asyncio.run(self.solve_challenge(challenge_url))

{warp_beacon-2.6.4 → warp_beacon-2.6.6}/warp_beacon/scraper/instagram/instagram.py

@@ -5,6 +5,8 @@ import ssl
 import re
 from typing import Callable, Optional, Union
 
+import logging
+
 import random
 import email
 import imaplib
@@ -28,8 +30,6 @@ from warp_beacon.jobs.types import JobType
 from warp_beacon.jobs.download_job import DownloadJob
 from warp_beacon.telegram.utils import Utils
 
-import logging
-
 INST_SESSION_FILE_TPL = "/var/warp_beacon/inst_session_account_%d.json"
 
 class InstagramScraper(ScraperAbstract):

{warp_beacon-2.6.4 → warp_beacon-2.6.6}/warp_beacon/scraper/youtube/abstract.py

@@ -291,15 +291,27 @@ class YoutubeAbstract(ScraperAbstract):
 
 	def download(self, job: DownloadJob) -> list:
 		ret = []
+		thumbnail = None
 		try:
-			ret = self.download_hndlr(self._download, job.url, session=True)
+			video_id = self.get_video_id(job.url)
+			# shorts custom thumb
+			##vinfo = VideoInfo(local_file)
+			#thumbnail = self.download_hndlr(self.download_thumbnail, video_id=yt.video_id, crop_center=vinfo.get_demensions())
+			if video_id:
+				thumbnail = self.download_hndlr(self.download_thumbnail, video_id)
+		except Exception as e:
+			logging.error("Failed to download thumb!")
+			logging.exception(e)
+
+		try:
+			ret = self.download_hndlr(self._download, job.url, session=True, thumbnail=thumbnail)
 			return ret
 		except (Unavailable, TimeOut, KeyError) as e:
 			logging.warning("Download failed, trying to download with yt_dlp")
 			logging.exception(e)
 		
 		try:
-			ret = self.download_hndlr(self._download_yt_dlp, job.url)
+			ret = self.download_hndlr(self._download_yt_dlp, job.url, thumbnail=thumbnail)
 		except NotImplementedError:
 			logging.info("yt_dlp is not supported for this submodule yet")
 			raise Unavailable("Сontent unvailable")

{warp_beacon-2.6.4 → warp_beacon-2.6.6}/warp_beacon/scraper/youtube/music.py

@@ -1,3 +1,6 @@
+import io
+from typing import Optional
+
 import logging
 
 import time
@@ -9,22 +12,15 @@ from warp_beacon.jobs.types import JobType
 from warp_beacon.scraper.youtube.abstract import YoutubeAbstract
 from warp_beacon.scraper.exceptions import NotFound, FileTooBig, Unavailable
 
-
 class YoutubeMusicScraper(YoutubeAbstract):
 	YT_MAX_RETRIES_DEFAULT = 3
 	YT_PAUSE_BEFORE_RETRY_DEFAULT = 3
 	YT_TIMEOUT_DEFAULT = 2
 	YT_TIMEOUT_INCREMENT_DEFAULT = 60
 
-	def _download(self, url: str, session: bool = True, timeout: int = 0) -> list:
+	def _download(self, url: str, session: bool = True, thumbnail: Optional[io.BytesIO] = None, timeout: int = 0) -> list:
 		res = []
 		try:
-			thumbnail = None
-			audio_id = self.get_video_id(url)
-
-			if audio_id:
-				thumbnail = self.download_hndlr(self.download_thumbnail, audio_id)
-
 			yt = self.build_yt(url, session=session)
 			
 			stream = yt.streams.get_audio_only()
@@ -79,12 +75,8 @@ class YoutubeMusicScraper(YoutubeAbstract):
 
 		return yt_dlp.YoutubeDL(ydl_opts)
 
-	def _download_yt_dlp(self, url: str, timeout: int = 60) -> list:
+	def _download_yt_dlp(self, url: str, timeout: int = 60, thumbnail: Optional[io.BytesIO] = None) -> list:
 		res = []
-		thumbnail = None
-		video_id = self.get_video_id(url)
-		if video_id:
-			thumbnail = self.download_hndlr(self.download_thumbnail, video_id)
 		with self.build_yt_dlp(timeout) as ydl:
 			info = ydl.extract_info(url, download=True)
 			local_file = ydl.prepare_filename(info)

{warp_beacon-2.6.4 → warp_beacon-2.6.6}/warp_beacon/scraper/youtube/shorts.py

@@ -1,3 +1,6 @@
+import io
+from typing import Optional
+
 import logging
 
 from warp_beacon.jobs.types import JobType
@@ -12,9 +15,8 @@ class YoutubeShortsScraper(YoutubeAbstract):
 	YT_TIMEOUT_DEFAULT = 2
 	YT_TIMEOUT_INCREMENT_DEFAULT = 60
 
-	def _download(self, url: str, session: bool = True, timeout: int = 0) -> list:
+	def _download(self, url: str, session: bool = True, thumbnail: Optional[io.BytesIO] = None, timeout: int = 0) -> list:
 		res = []
-		thumbnail = None
 		yt = self.build_yt(url, session=session)
 		stream = yt.streams.get_highest_resolution()
 
@@ -30,8 +32,6 @@ class YoutubeShortsScraper(YoutubeAbstract):
 		)
 
 		local_file = self.rename_local_file(local_file)
-		vinfo = VideoInfo(local_file)
-		thumbnail = self.download_hndlr(self.download_thumbnail, video_id=yt.video_id, crop_center=vinfo.get_demensions())
 
 		logging.debug("Temp filename: '%s'", local_file)
 		res.append({
@@ -44,12 +44,8 @@ class YoutubeShortsScraper(YoutubeAbstract):
 
 		return res
 
-	def _download_yt_dlp(self, url: str, timeout: int = 60) -> list:
+	def _download_yt_dlp(self, url: str, thumbnail: Optional[io.BytesIO] = None, timeout: int = 60) -> list:
 		res = []
-		thumbnail = None
-		video_id = self.get_video_id(url)
-		if video_id:
-			thumbnail = self.download_hndlr(self.download_thumbnail, video_id)
 		with self.build_yt_dlp(timeout) as ydl:
 			info = ydl.extract_info(url, download=True)
 			local_file = ydl.prepare_filename(info)

{warp_beacon-2.6.4 → warp_beacon-2.6.6}/warp_beacon/scraper/youtube/youtube.py

@@ -1,5 +1,7 @@
 import time
 import os
+import io
+from typing import Optional
 import logging
 
 import av
@@ -65,14 +67,10 @@ class YoutubeScraper(YoutubeAbstract):
 
 		return output_path
 
-	def _download_pytubefix_max_res(self, url: str, session: bool = True, timeout: int = 60) -> list:
+	def _download_pytubefix_max_res(self, url: str, session: bool = True, thumbnail: Optional[io.BytesIO] = None, timeout: int = 60) -> list:
 		res = []
 		local_video_file, local_audio_file = '', ''
 		try:
-			thumbnail = None
-			video_id = self.get_video_id(url)
-			if video_id:
-				thumbnail = self.download_hndlr(self.download_thumbnail, video_id)
 			yt = self.build_yt(url, session=session)
 
 			if self.is_live(yt.initial_data):
@@ -125,13 +123,9 @@ class YoutubeScraper(YoutubeAbstract):
 		
 		return res
 
-	def _download_pytube_dash(self, url: str, session: bool = True, timeout: int = 60) -> list:
+	def _download_pytube_dash(self, url: str, session: bool = True, thumbnail: Optional[io.BytesIO] = None, timeout: int = 60) -> list:
 		res = []
 		try:
-			thumbnail = None
-			video_id = self.get_video_id(url)
-			if video_id:
-				thumbnail = self.download_hndlr(self.download_thumbnail, video_id)
 			yt = self.build_yt(url, session=session)
 
 			if self.is_live(yt.initial_data):
@@ -164,12 +158,8 @@ class YoutubeScraper(YoutubeAbstract):
 
 		return res
 	
-	def _download_yt_dlp(self, url: str, timeout: int = 60) -> list:
+	def _download_yt_dlp(self, url: str, thumbnail: Optional[io.BytesIO] = None, timeout: int = 60) -> list:
 		res = []
-		thumbnail = None
-		video_id = self.get_video_id(url)
-		if video_id:
-			thumbnail = self.download_hndlr(self.download_thumbnail, video_id)
 		with self.build_yt_dlp(timeout) as ydl:
 			info = ydl.extract_info(url, download=True)
 			local_file = ydl.prepare_filename(info)

{warp_beacon-2.6.4 → warp_beacon-2.6.6/warp_beacon.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.6.4
+Version: 2.6.6
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev
@@ -238,6 +238,9 @@ Requires-Dist: pymongo
 Requires-Dist: instagrapi==2.0.0
 Requires-Dist: bs4
 Requires-Dist: yt_dlp
+Requires-Dist: pydub
+Requires-Dist: SpeechRecognition
+Requires-Dist: playwright
 Dynamic: author
 Dynamic: home-page
 Dynamic: license-file

{warp_beacon-2.6.4 → warp_beacon-2.6.6}/warp_beacon.egg-info/SOURCES.txt

@@ -42,6 +42,7 @@ warp_beacon/scraper/exceptions.py
 warp_beacon/scraper/fail_handler.py
 warp_beacon/scraper/link_resolver.py
 warp_beacon/scraper/instagram/__init__.py
+warp_beacon/scraper/instagram/captcha.py
 warp_beacon/scraper/instagram/instagram.py
 warp_beacon/scraper/youtube/__init__.py
 warp_beacon/scraper/youtube/abstract.py

{warp_beacon-2.6.4 → warp_beacon-2.6.6}/warp_beacon.egg-info/requires.txt

@@ -12,3 +12,6 @@ pymongo
 instagrapi==2.0.0
 bs4
 yt_dlp
+pydub
+SpeechRecognition
+playwright

warp_beacon-2.6.4/warp_beacon/__version__.py

@@ -1,2 +0,0 @@
-__version__ = "2.6.4"
-