warp-beacon 2.3.44__tar.gz → 2.3.45__tar.gz

{warp_beacon-2.3.44/warp_beacon.egg-info → warp_beacon-2.3.45}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.3.44
+Version: 2.3.45
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

warp_beacon-2.3.45/warp_beacon/__version__.py

@@ -0,0 +1,2 @@
+__version__ = "2.3.45"
+

{warp_beacon-2.3.44 → warp_beacon-2.3.45}/warp_beacon/jobs/abstract.py

@@ -38,6 +38,7 @@ class JobSettings(TypedDict):
 	unvailable_error_count: int
 	geoblock_error_count: int
 	account_switches: int
+	bad_proxy_error_count: int
 	yt_auth: bool
 	session_validation: bool
 	chat_type: ChatType
@@ -76,6 +77,7 @@ class AbstractJob(ABC):
 	unvailable_error_count: int = 0
 	geoblock_error_count: int = 0
 	account_switches: int = 0
+	bad_proxy_error_count: int = 0
 	yt_auth: bool = False
 	session_validation: bool = False
 	chat_type: ChatType = None

{warp_beacon-2.3.44 → warp_beacon-2.3.45}/warp_beacon/scraper/__init__.py

@@ -7,7 +7,8 @@ from queue import Empty
 
 import logging
 
-from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, Unavailable, FileTooBig, YoutubeLiveError, YotubeAgeRestrictedError, IGRateLimitOccurred, CaptchaIssue, AllAccountsFailed
+from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, Unavailable, FileTooBig, YoutubeLiveError, \
+	YotubeAgeRestrictedError, IGRateLimitOccurred, CaptchaIssue, AllAccountsFailed, BadProxy
 from warp_beacon.mediainfo.video import VideoInfo
 from warp_beacon.mediainfo.audio import AudioInfo
 from warp_beacon.mediainfo.silencer import Silencer
@@ -116,21 +117,23 @@ class AsyncDownloader(object):
 									continue
 							actor = None
 							self.acc_selector.set_module(job.job_origin)
+							proxy = selector.get_current_proxy()
 							if job.job_origin is Origin.INSTAGRAM:
 								from warp_beacon.scraper.instagram.instagram import InstagramScraper
-								actor = InstagramScraper(selector.get_current(), selector.get_account_proxy())
+								actor = InstagramScraper(selector.get_current(), proxy)
 							elif job.job_origin is Origin.YT_SHORTS:
 								from warp_beacon.scraper.youtube.shorts import YoutubeShortsScraper
-								actor = YoutubeShortsScraper(selector.get_current(), selector.get_account_proxy())
+								actor = YoutubeShortsScraper(selector.get_current(), proxy)
 							elif job.job_origin is Origin.YT_MUSIC:
 								from warp_beacon.scraper.youtube.music import YoutubeMusicScraper
-								actor = YoutubeMusicScraper(selector.get_current(), selector.get_account_proxy())
+								actor = YoutubeMusicScraper(selector.get_current(), proxy)
 							elif job.job_origin is Origin.YOUTUBE:
 								from warp_beacon.scraper.youtube.youtube import YoutubeScraper
-								actor = YoutubeScraper(selector.get_current(), selector.get_account_proxy())
+								actor = YoutubeScraper(selector.get_current(), proxy)
 							actor.send_message_to_admin_func = self.send_message_to_admin
 							actor.auth_event = self.auth_event
-							while True:
+							# job retry loop
+							while self.allow_loop.value == 1:
 								try:
 									if job.session_validation:
 										logging.info("Validating '%s' session ...", job.job_origin.value)
@@ -154,7 +157,7 @@ class AsyncDownloader(object):
 								except Unavailable as e:
 									logging.warning("Not found or unavailable error occurred!")
 									logging.exception(e)
-									if job.unvailable_error_count > self.acc_selector.count_service_accounts(job.job_origin):
+									if job.unvailable_error_count > selector.count_service_accounts(job.job_origin):
 										self.uploader.queue_task(job.to_upload_job(
 											job_failed=True,
 											job_failed_msg="Video is unvailable for all your service accounts.")
@@ -165,17 +168,22 @@ class AsyncDownloader(object):
 									selector.next()
 									self.job_queue.put(job)
 									break
-								except TimeOut as e:
-									logging.warning("Timeout error occurred!")
+								except (TimeOut, BadProxy) as e:
+									logging.warning("Timeout or BadProxy error occurred!")
 									logging.exception(e)
-									self.send_message_to_admin(
-										f"Task <code>{job.job_id}</code> failed. URL: '{job.url}'. Reason: '<b>TimeOut</b>'."
-									)
-									self.uploader.queue_task(job.to_upload_job(
-										job_failed=True,
-										job_failed_msg="Failed to download content due timeout error. Please check you Internet connection, retry amount or request timeout bot configuration settings.")
-									)
-									break
+									if job.bad_proxy_error_count > len(selector.proxies):
+										self.send_message_to_admin(
+											f"Task <code>{job.job_id}</code> failed. URL: '{job.url}'. Reason: '<b>TimeOut</b>'."
+										)
+										self.uploader.queue_task(job.to_upload_job(
+											job_failed=True,
+											job_failed_msg="Failed to download content due timeout error. Please check you Internet connection, retry amount or request timeout bot configuration settings.")
+										)
+										break
+									job.bad_proxy_error_count += 1
+									logging.info("Trying next proxy")
+									selector.next_proxy()
+									self.job_queue.put(job)
 								except FileTooBig as e:
 									logging.warning("Telegram limits exceeded :(")
 									logging.exception(e)

{warp_beacon-2.3.44 → warp_beacon-2.3.45}/warp_beacon/scraper/account_selector.py

@@ -3,8 +3,9 @@ import time
 import random
 import json
 import re
-from typing import Optional
+from typing import Optional, List
 
+from itertools import cycle
 import multiprocessing
 import multiprocessing.managers
 
@@ -21,6 +22,7 @@ class AccountSelector(object):
 	session_dir = "/var/warp_beacon"
 	manager = None
 	account_index = {}
+	current_proxy = None
 
 	def __init__(self, manager: multiprocessing.managers.SyncManager, acc_file_path: str, proxy_file_path: str=None) -> None:
 		self.manager = manager
@@ -36,22 +38,35 @@ class AccountSelector(object):
 			if proxy_file_path:
 				with open(proxy_file_path, 'r', encoding="utf-8") as f:
 					self.proxies = json.loads(f.read())
+				self.current_proxy = self.get_random_account_proxy()
 		else:
 			raise ValueError("Accounts file not found")
 
-	def get_account_proxy(self) -> Optional[dict]:
+	def get_current_proxy(self) -> Optional[dict]:
+		return self.current_proxy
+
+	def get_proxy_list(self) -> List[dict]:
+		matched_proxy = []
+		try:
+			acc_id, acc_data = self.get_current()
+			current_acc_pid = acc_data.get("proxy_id", "").strip()
+			for proxy in self.proxies:
+				pid = proxy.get("id", "").strip()
+				if pid and current_acc_pid and pid == current_acc_pid:
+					if "override_force_ipv6" in proxy:
+						self.accounts[self.current_module_name][acc_id]["force_ipv6"] = proxy.get("override_force_ipv6", False)
+					logging.info("Account proxy matched '%s'", proxy)
+					matched_proxy.append(proxy)
+		except Exception as e:
+			logging.warning("Failed to form proxy list!")
+			logging.exception(e)
+
+		return matched_proxy
+
+	def get_random_account_proxy(self) -> Optional[dict]:
 		if self.proxies:
 			try:
-				acc_id, acc_data = self.get_current()
-				current_acc_pid = acc_data.get("proxy_id", "").strip()
-				matched_proxy = []
-				for proxy in self.proxies:
-					pid = proxy.get("id", "").strip()
-					if pid and current_acc_pid and pid == current_acc_pid:
-						if "override_force_ipv6" in proxy:
-							self.accounts[self.current_module_name][acc_id]["force_ipv6"] = proxy.get("override_force_ipv6", False)
-						logging.info("Account proxy matched '%s'", proxy)
-						matched_proxy.append(proxy)
+				matched_proxy = self.get_proxy_list()
 				if matched_proxy:
 					if len(matched_proxy) > 1:
 						random.seed(random.seed(time.time_ns() ^ int.from_bytes(os.urandom(len(matched_proxy)), "big")))
@@ -62,12 +77,32 @@ class AccountSelector(object):
 					prox_choice = random.choice(matched_proxy)
 					# saving chosen proxy for history
 					self.accounts_meta_data["last_proxy"] = prox_choice
+					self.current_proxy = prox_choice
 					logging.info("Chosen proxy: '%s'", prox_choice)
 					return prox_choice
 			except Exception as e:
 				logging.warning("Error on selecting account proxy!")
 				logging.exception(e)
 		return None
+	
+	def next_proxy(self) -> Optional[dict]:
+		if not self.proxies:
+			return None
+		proxy = None
+		try:
+			matched_proxies = self.get_proxy_list()
+			if matched_proxies:
+				lit = cycle(matched_proxies)
+				proxy = next(lit)
+				if proxy.get("dsn", {}) == self.accounts_meta_data["last_proxy"].get("dsn", {}):
+					proxy = next(lit)
+				self.current_proxy = proxy
+				self.accounts_meta_data["last_proxy"] = proxy
+		except Exception as e:
+			logging.warning("Error on selection next proxy!")
+			logging.exception(e)
+
+		return proxy
 
 	def load_yt_sessions(self) -> None:
 		if "youtube" not in self.accounts:

{warp_beacon-2.3.44 → warp_beacon-2.3.45}/warp_beacon/scraper/exceptions.py

@@ -41,6 +41,9 @@ class CaptchaIssue(ScraperError):
 class AllAccountsFailed(ScraperError):
 	pass
 
+class BadProxy(ScraperError):
+	pass
+
 class UnknownError(ScraperError):
 	pass
 

{warp_beacon-2.3.44 → warp_beacon-2.3.45}/warp_beacon/scraper/youtube/music.py

@@ -7,7 +7,7 @@ import yt_dlp
 
 from warp_beacon.jobs.types import JobType
 from warp_beacon.scraper.youtube.abstract import YoutubeAbstract
-from warp_beacon.scraper.exceptions import NotFound, FileTooBig
+from warp_beacon.scraper.exceptions import NotFound, FileTooBig, Unavailable
 
 
 class YoutubeMusicScraper(YoutubeAbstract):
@@ -18,39 +18,42 @@ class YoutubeMusicScraper(YoutubeAbstract):
 
 	def _download(self, url: str, session: bool = True, timeout: int = 0) -> list:
 		res = []
-		thumbnail = None
-		audio_id = self.get_video_id(url)
+		try:
+			thumbnail = None
+			audio_id = self.get_video_id(url)
 
-		if audio_id:
-			thumbnail = self.download_hndlr(self.download_thumbnail, audio_id)
+			if audio_id:
+				thumbnail = self.download_hndlr(self.download_thumbnail, audio_id)
 
-		yt = self.build_yt(url, session=session)
-		
-		stream = yt.streams.get_audio_only()
-		
-		if not stream:
-			raise NotFound("No suitable audio stream found")
-		
-		logging.info("Announced audio file size: '%d'", stream.filesize)
-		if stream.filesize > 2e+9:
-			logging.warning("Downloading size reported by YouTube is over than 2 GB!")
-			raise FileTooBig("YouTube file is larger than 2 GB")
-		logging.info("Operation timeout is '%d'", timeout)
-		local_file = stream.download(
-			output_path=self.DOWNLOAD_DIR,
-			max_retries=0,
-			timeout=timeout,
-			skip_existing=False,
-			filename_prefix='yt_download_'
-		)
-		logging.debug("Temp filename: '%s'", local_file)
-		res.append({
-			"local_media_path": self.rename_local_file(local_file),
-			"performer": yt.author,
-			"thumb": thumbnail,
-			"canonical_name": stream.title,
-			"media_type": JobType.AUDIO
-		})
+			yt = self.build_yt(url, session=session)
+			
+			stream = yt.streams.get_audio_only()
+			
+			if not stream:
+				raise NotFound("No suitable audio stream found")
+			
+			logging.info("Announced audio file size: '%d'", stream.filesize)
+			if stream.filesize > 2e+9:
+				logging.warning("Downloading size reported by YouTube is over than 2 GB!")
+				raise FileTooBig("YouTube file is larger than 2 GB")
+			logging.info("Operation timeout is '%d'", timeout)
+			local_file = stream.download(
+				output_path=self.DOWNLOAD_DIR,
+				max_retries=0,
+				timeout=timeout,
+				skip_existing=False,
+				filename_prefix='yt_download_'
+			)
+			logging.debug("Temp filename: '%s'", local_file)
+			res.append({
+				"local_media_path": self.rename_local_file(local_file),
+				"performer": yt.author,
+				"thumb": thumbnail,
+				"canonical_name": stream.title,
+				"media_type": JobType.AUDIO
+			})
+		except Exception:
+			raise Unavailable("Content unavailable")
 
 		return res
 

{warp_beacon-2.3.44 → warp_beacon-2.3.45/warp_beacon.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.3.44
+Version: 2.3.45
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

warp_beacon-2.3.44/warp_beacon/__version__.py

@@ -1,2 +0,0 @@
-__version__ = "2.3.44"
-