warp-beacon 2.3.44__py3-none-any.whl → 2.3.46__py3-none-any.whl

warp_beacon/__version__.py

@@ -1,2 +1,2 @@
-__version__ = "2.3.44"
+__version__ = "2.3.46"
 

warp_beacon/jobs/abstract.py

@@ -38,6 +38,7 @@ class JobSettings(TypedDict):
 	unvailable_error_count: int
 	geoblock_error_count: int
 	account_switches: int
+	bad_proxy_error_count: int
 	yt_auth: bool
 	session_validation: bool
 	chat_type: ChatType
@@ -76,6 +77,7 @@ class AbstractJob(ABC):
 	unvailable_error_count: int = 0
 	geoblock_error_count: int = 0
 	account_switches: int = 0
+	bad_proxy_error_count: int = 0
 	yt_auth: bool = False
 	session_validation: bool = False
 	chat_type: ChatType = None

warp_beacon/scraper/__init__.py

@@ -7,7 +7,8 @@ from queue import Empty
 
 import logging
 
-from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, Unavailable, FileTooBig, YoutubeLiveError, YotubeAgeRestrictedError, IGRateLimitOccurred, CaptchaIssue, AllAccountsFailed
+from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, Unavailable, FileTooBig, YoutubeLiveError, \
+	YotubeAgeRestrictedError, IGRateLimitOccurred, CaptchaIssue, AllAccountsFailed, BadProxy
 from warp_beacon.mediainfo.video import VideoInfo
 from warp_beacon.mediainfo.audio import AudioInfo
 from warp_beacon.mediainfo.silencer import Silencer
@@ -116,21 +117,23 @@ class AsyncDownloader(object):
 									continue
 							actor = None
 							self.acc_selector.set_module(job.job_origin)
+							proxy = selector.get_current_proxy()
 							if job.job_origin is Origin.INSTAGRAM:
 								from warp_beacon.scraper.instagram.instagram import InstagramScraper
-								actor = InstagramScraper(selector.get_current(), selector.get_account_proxy())
+								actor = InstagramScraper(selector.get_current(), proxy)
 							elif job.job_origin is Origin.YT_SHORTS:
 								from warp_beacon.scraper.youtube.shorts import YoutubeShortsScraper
-								actor = YoutubeShortsScraper(selector.get_current(), selector.get_account_proxy())
+								actor = YoutubeShortsScraper(selector.get_current(), proxy)
 							elif job.job_origin is Origin.YT_MUSIC:
 								from warp_beacon.scraper.youtube.music import YoutubeMusicScraper
-								actor = YoutubeMusicScraper(selector.get_current(), selector.get_account_proxy())
+								actor = YoutubeMusicScraper(selector.get_current(), proxy)
 							elif job.job_origin is Origin.YOUTUBE:
 								from warp_beacon.scraper.youtube.youtube import YoutubeScraper
-								actor = YoutubeScraper(selector.get_current(), selector.get_account_proxy())
+								actor = YoutubeScraper(selector.get_current(), proxy)
 							actor.send_message_to_admin_func = self.send_message_to_admin
 							actor.auth_event = self.auth_event
-							while True:
+							# job retry loop
+							while self.allow_loop.value == 1:
 								try:
 									if job.session_validation:
 										logging.info("Validating '%s' session ...", job.job_origin.value)
@@ -154,7 +157,7 @@ class AsyncDownloader(object):
 								except Unavailable as e:
 									logging.warning("Not found or unavailable error occurred!")
 									logging.exception(e)
-									if job.unvailable_error_count > self.acc_selector.count_service_accounts(job.job_origin):
+									if job.unvailable_error_count > selector.count_service_accounts(job.job_origin):
 										self.uploader.queue_task(job.to_upload_job(
 											job_failed=True,
 											job_failed_msg="Video is unvailable for all your service accounts.")
@@ -165,17 +168,22 @@ class AsyncDownloader(object):
 									selector.next()
 									self.job_queue.put(job)
 									break
-								except TimeOut as e:
-									logging.warning("Timeout error occurred!")
+								except (TimeOut, BadProxy) as e:
+									logging.warning("Timeout or BadProxy error occurred!")
 									logging.exception(e)
-									self.send_message_to_admin(
-										f"Task <code>{job.job_id}</code> failed. URL: '{job.url}'. Reason: '<b>TimeOut</b>'."
-									)
-									self.uploader.queue_task(job.to_upload_job(
-										job_failed=True,
-										job_failed_msg="Failed to download content due timeout error. Please check you Internet connection, retry amount or request timeout bot configuration settings.")
-									)
-									break
+									if job.bad_proxy_error_count > len(selector.proxies):
+										self.send_message_to_admin(
+											f"Task <code>{job.job_id}</code> failed. URL: '{job.url}'. Reason: '<b>TimeOut</b>'."
+										)
+										self.uploader.queue_task(job.to_upload_job(
+											job_failed=True,
+											job_failed_msg="Failed to download content due timeout error. Please check you Internet connection, retry amount or request timeout bot configuration settings.")
+										)
+										break
+									job.bad_proxy_error_count += 1
+									logging.info("Trying next proxy")
+									selector.next_proxy()
+									self.job_queue.put(job)
 								except FileTooBig as e:
 									logging.warning("Telegram limits exceeded :(")
 									logging.exception(e)

warp_beacon/scraper/account_selector.py

@@ -3,8 +3,9 @@ import time
 import random
 import json
 import re
-from typing import Optional
+from typing import Optional, List
 
+from itertools import cycle
 import multiprocessing
 import multiprocessing.managers
 
@@ -21,6 +22,7 @@ class AccountSelector(object):
 	session_dir = "/var/warp_beacon"
 	manager = None
 	account_index = {}
+	current_proxy = None
 
 	def __init__(self, manager: multiprocessing.managers.SyncManager, acc_file_path: str, proxy_file_path: str=None) -> None:
 		self.manager = manager
@@ -36,22 +38,35 @@ class AccountSelector(object):
 			if proxy_file_path:
 				with open(proxy_file_path, 'r', encoding="utf-8") as f:
 					self.proxies = json.loads(f.read())
+				self.current_proxy = self.get_random_account_proxy()
 		else:
 			raise ValueError("Accounts file not found")
 
-	def get_account_proxy(self) -> Optional[dict]:
+	def get_current_proxy(self) -> Optional[dict]:
+		return self.current_proxy
+
+	def get_proxy_list(self) -> List[dict]:
+		matched_proxy = []
+		try:
+			acc_id, acc_data = self.get_current()
+			current_acc_pid = acc_data.get("proxy_id", "").strip()
+			for proxy in self.proxies:
+				pid = proxy.get("id", "").strip()
+				if pid and current_acc_pid and pid == current_acc_pid:
+					if "override_force_ipv6" in proxy:
+						self.accounts[self.current_module_name][acc_id]["force_ipv6"] = proxy.get("override_force_ipv6", False)
+					logging.info("Account proxy matched '%s'", proxy)
+					matched_proxy.append(proxy)
+		except Exception as e:
+			logging.warning("Failed to form proxy list!")
+			logging.exception(e)
+
+		return matched_proxy
+
+	def get_random_account_proxy(self) -> Optional[dict]:
 		if self.proxies:
 			try:
-				acc_id, acc_data = self.get_current()
-				current_acc_pid = acc_data.get("proxy_id", "").strip()
-				matched_proxy = []
-				for proxy in self.proxies:
-					pid = proxy.get("id", "").strip()
-					if pid and current_acc_pid and pid == current_acc_pid:
-						if "override_force_ipv6" in proxy:
-							self.accounts[self.current_module_name][acc_id]["force_ipv6"] = proxy.get("override_force_ipv6", False)
-						logging.info("Account proxy matched '%s'", proxy)
-						matched_proxy.append(proxy)
+				matched_proxy = self.get_proxy_list()
 				if matched_proxy:
 					if len(matched_proxy) > 1:
 						random.seed(random.seed(time.time_ns() ^ int.from_bytes(os.urandom(len(matched_proxy)), "big")))
@@ -62,12 +77,32 @@ class AccountSelector(object):
 					prox_choice = random.choice(matched_proxy)
 					# saving chosen proxy for history
 					self.accounts_meta_data["last_proxy"] = prox_choice
+					self.current_proxy = prox_choice
 					logging.info("Chosen proxy: '%s'", prox_choice)
 					return prox_choice
 			except Exception as e:
 				logging.warning("Error on selecting account proxy!")
 				logging.exception(e)
 		return None
+	
+	def next_proxy(self) -> Optional[dict]:
+		if not self.proxies:
+			return None
+		proxy = None
+		try:
+			matched_proxies = self.get_proxy_list()
+			if matched_proxies:
+				lit = cycle(matched_proxies)
+				proxy = next(lit)
+				if proxy.get("dsn", {}) == self.accounts_meta_data["last_proxy"].get("dsn", {}):
+					proxy = next(lit)
+				self.current_proxy = proxy
+				self.accounts_meta_data["last_proxy"] = proxy
+		except Exception as e:
+			logging.warning("Error on selection next proxy!")
+			logging.exception(e)
+
+		return proxy
 
 	def load_yt_sessions(self) -> None:
 		if "youtube" not in self.accounts:

warp_beacon/scraper/exceptions.py

@@ -41,6 +41,9 @@ class CaptchaIssue(ScraperError):
 class AllAccountsFailed(ScraperError):
 	pass
 
+class BadProxy(ScraperError):
+	pass
+
 class UnknownError(ScraperError):
 	pass
 

warp_beacon/scraper/youtube/abstract.py

@@ -218,7 +218,8 @@ class YoutubeAbstract(ScraperAbstract):
 					http.client.HTTPException,
 					requests.RequestException,
 					urllib.error.URLError,
-					urllib.error.HTTPError) as e:
+					urllib.error.HTTPError,
+					KeyError) as e:
 				if hasattr(e, "code") and (int(e.code) == 403 or int(e.code) == 400):
 					raise Unavailable(extract_exception_message(e))
 				logging.warning("Youtube read timeout! Retrying in '%d' seconds ...", pause_secs)

warp_beacon/scraper/youtube/music.py

@@ -7,7 +7,7 @@ import yt_dlp
 
 from warp_beacon.jobs.types import JobType
 from warp_beacon.scraper.youtube.abstract import YoutubeAbstract
-from warp_beacon.scraper.exceptions import NotFound, FileTooBig
+from warp_beacon.scraper.exceptions import NotFound, FileTooBig, Unavailable
 
 
 class YoutubeMusicScraper(YoutubeAbstract):
@@ -18,39 +18,42 @@ class YoutubeMusicScraper(YoutubeAbstract):
 
 	def _download(self, url: str, session: bool = True, timeout: int = 0) -> list:
 		res = []
-		thumbnail = None
-		audio_id = self.get_video_id(url)
+		try:
+			thumbnail = None
+			audio_id = self.get_video_id(url)
 
-		if audio_id:
-			thumbnail = self.download_hndlr(self.download_thumbnail, audio_id)
+			if audio_id:
+				thumbnail = self.download_hndlr(self.download_thumbnail, audio_id)
 
-		yt = self.build_yt(url, session=session)
-		
-		stream = yt.streams.get_audio_only()
-		
-		if not stream:
-			raise NotFound("No suitable audio stream found")
-		
-		logging.info("Announced audio file size: '%d'", stream.filesize)
-		if stream.filesize > 2e+9:
-			logging.warning("Downloading size reported by YouTube is over than 2 GB!")
-			raise FileTooBig("YouTube file is larger than 2 GB")
-		logging.info("Operation timeout is '%d'", timeout)
-		local_file = stream.download(
-			output_path=self.DOWNLOAD_DIR,
-			max_retries=0,
-			timeout=timeout,
-			skip_existing=False,
-			filename_prefix='yt_download_'
-		)
-		logging.debug("Temp filename: '%s'", local_file)
-		res.append({
-			"local_media_path": self.rename_local_file(local_file),
-			"performer": yt.author,
-			"thumb": thumbnail,
-			"canonical_name": stream.title,
-			"media_type": JobType.AUDIO
-		})
+			yt = self.build_yt(url, session=session)
+			
+			stream = yt.streams.get_audio_only()
+			
+			if not stream:
+				raise NotFound("No suitable audio stream found")
+			
+			logging.info("Announced audio file size: '%d'", stream.filesize)
+			if stream.filesize > 2e+9:
+				logging.warning("Downloading size reported by YouTube is over than 2 GB!")
+				raise FileTooBig("YouTube file is larger than 2 GB")
+			logging.info("Operation timeout is '%d'", timeout)
+			local_file = stream.download(
+				output_path=self.DOWNLOAD_DIR,
+				max_retries=0,
+				timeout=timeout,
+				skip_existing=False,
+				filename_prefix='yt_download_'
+			)
+			logging.debug("Temp filename: '%s'", local_file)
+			res.append({
+				"local_media_path": self.rename_local_file(local_file),
+				"performer": yt.author,
+				"thumb": thumbnail,
+				"canonical_name": stream.title,
+				"media_type": JobType.AUDIO
+			})
+		except Exception:
+			raise Unavailable("Content unavailable")
 
 		return res
 

{warp_beacon-2.3.44.dist-info → warp_beacon-2.3.46.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.3.44
+Version: 2.3.46
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

{warp_beacon-2.3.44.dist-info → warp_beacon-2.3.46.dist-info}/RECORD

@@ -4,12 +4,12 @@ var/warp_beacon/accounts.json,sha256=OsXdncs6h88xrF_AP6_WDCK1waGBn9SR-uYdIeK37GM
 var/warp_beacon/placeholder.gif,sha256=cE5CGJVaop4Sx21zx6j4AyoHU0ncmvQuS2o6hJfEH88,6064
 var/warp_beacon/proxies.json,sha256=VnjlQDXumOEq72ZFjbh6IqHS1TEHqn8HPYAZqWCeSIA,95
 warp_beacon/__init__.py,sha256=_rThNODmz0nDp_n4mWo_HKaNFE5jk1_7cRhHyYaencI,163
-warp_beacon/__version__.py,sha256=4mQiTG01QRZR9DWdi-f-4XbeNcXK_HdGHLDMQE97HYs,24
+warp_beacon/__version__.py,sha256=WwmzdMPCUOj0exfQEPh8jozMYxz4FGxv_V9AJozlyKM,24
 warp_beacon/warp_beacon.py,sha256=7KEtZDj-pdhtl6m-zFLsSojs1ZR4o7L0xbqtdmYPvfE,342
 warp_beacon/compress/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/compress/video.py,sha256=_PDMVYCyzLYxHv1uZmmzGcG_8rjaZr7BTXsXTTy_oS4,2846
 warp_beacon/jobs/__init__.py,sha256=ED8_tPle4iL4kqNW0apAVkgNQtRRTnYfAJwBjO1g0JY,180
-warp_beacon/jobs/abstract.py,sha256=7DZWGI2KfBmAF9qwftmVBrOa2Szd0vrlp24aQKl95ho,3119
+warp_beacon/jobs/abstract.py,sha256=IF2bKx2DuuH_AP07YAvFxxqnuxjIwg2Glsr8TpWX6Z0,3179
 warp_beacon/jobs/download_job.py,sha256=5HiPcnJppFMhO14___3eSkoMygM3y-vhpGkMAuNhK7s,854
 warp_beacon/jobs/types.py,sha256=Ae8zINgbs7cOcYkYoOCOACA7duyhnIGMQAJ_SJB1QRQ,176
 warp_beacon/jobs/upload_job.py,sha256=_ul4psPej1jLEs-BMcMR80GbXDSmm38jE9yoZtecclY,741
@@ -21,17 +21,17 @@ warp_beacon/mediainfo/video.py,sha256=UBZrhTN5IDI-aYu6tsJEILo9nFkjHhkldGVFmvV7tE
 warp_beacon/scheduler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scheduler/instagram_human.py,sha256=K3S14uav0UBZidtREYwgCOCDKyDfWbay5GH-DWMK_JI,5020
 warp_beacon/scheduler/scheduler.py,sha256=ddPhExSgj_RGR1eSDlIm0hVgTVM_76UEMEl9i-eYuQ8,2915
-warp_beacon/scraper/__init__.py,sha256=V_C5SmAmRtjFfVBtTotOHCW3ILMQ8m_ulrBF6ry59_A,16944
+warp_beacon/scraper/__init__.py,sha256=cxRbhXFW69KbAWE8b2-j9pfUBSMSh95aNyf0GFluyig,17188
 warp_beacon/scraper/abstract.py,sha256=28a0aBKZpi8IKptLWdB6RuVbOkrUbrhT7LSZX7QRQtg,2725
-warp_beacon/scraper/account_selector.py,sha256=oqwtyT2_xBYkx_7ysoz0VLa-x8EpiBifmF12MUNLCW8,5316
-warp_beacon/scraper/exceptions.py,sha256=QBHLA8hK1sL8uJu5JhidbXoBXPZtMMtV26nYfHvO4NU,1339
+warp_beacon/scraper/account_selector.py,sha256=FyoZ0-YGuxqWzepIqgwJmPn0AC9u13RQC3F7Cl-BVoc,6284
+warp_beacon/scraper/exceptions.py,sha256=L4UCZqDOLKu5-TPUcTErIbNDLrAQJghkMaPqzTfsm1g,1376
 warp_beacon/scraper/fail_handler.py,sha256=_blvckfTZ4xWVancQKVRXH5ClKGwfrBxMwvXIFZh1qA,975
 warp_beacon/scraper/link_resolver.py,sha256=Rc9ZuMyOo3iPywDHwjngy-WRQ2SXhJwxcg-5ripx7tM,2447
 warp_beacon/scraper/instagram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scraper/instagram/instagram.py,sha256=TpIBS9S3zcLbeGax4CENVo6WP75EE6oIttc-MjWYVjs,13815
 warp_beacon/scraper/youtube/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-warp_beacon/scraper/youtube/abstract.py,sha256=0ZfmPiCtzBbYQ6rkych7Es6GFi2Mw3CjoujassGO0_4,11773
-warp_beacon/scraper/youtube/music.py,sha256=xe68GWbforWWPtgr75wYZA66-cGzK86vS-Cj1tyxVQQ,2812
+warp_beacon/scraper/youtube/abstract.py,sha256=gGtWjsFCcJ8kYDXD4BtWCQ04uYW0jdIwWexJ84p8wFU,11788
+warp_beacon/scraper/youtube/music.py,sha256=-R6Iys8IpglH5UL7F640UQ3ieYHitvGXA0A2GhSemVA,2927
 warp_beacon/scraper/youtube/shorts.py,sha256=TVQP-Dm4ZV3bHd7u25yOpDHRzoXL3jUAszDb6ZW8p0U,1269
 warp_beacon/scraper/youtube/youtube.py,sha256=B1NBz6vv0-zIuf3MDmT2QB7eX2qO8k9orj7-4wU6MBk,2839
 warp_beacon/storage/__init__.py,sha256=0Vajd0oITKJfu2vmNx5uQSt3-L6vwIvUYWJo8HZCjco,3398
@@ -43,9 +43,9 @@ warp_beacon/telegram/handlers.py,sha256=XXIfdV_RCj7tyZMPXchuKmGoDdweOaR08ADDaBPW
 warp_beacon/telegram/placeholder_message.py,sha256=wN9-BRiyrtHG-EvXtZkGJHt2CX71munQ57ITttjt0mw,6400
 warp_beacon/telegram/utils.py,sha256=9uebX53G16mV7ER7WgfdWBLFHHw14S8HBt9URrIskg0,4440
 warp_beacon/uploader/__init__.py,sha256=E9rlZIf7xlQz6MutMOwJ8S5Vm2uheR5nv23Kv8duRQg,5427
-warp_beacon-2.3.44.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-warp_beacon-2.3.44.dist-info/METADATA,sha256=komh1F3rqhR3Zy--VjsQAEU58k-GXXDr6KGpw_mkDDc,22626
-warp_beacon-2.3.44.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
-warp_beacon-2.3.44.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
-warp_beacon-2.3.44.dist-info/top_level.txt,sha256=qGjHVVfyf6lTmbdSA-fQq0rHS1YVS4HoJT3rag5xgPE,1141
-warp_beacon-2.3.44.dist-info/RECORD,,
+warp_beacon-2.3.46.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+warp_beacon-2.3.46.dist-info/METADATA,sha256=p7yMyzJosIJhR3oM0REGyQzBej8DhsXIrMdKqAPg9HY,22626
+warp_beacon-2.3.46.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
+warp_beacon-2.3.46.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
+warp_beacon-2.3.46.dist-info/top_level.txt,sha256=qGjHVVfyf6lTmbdSA-fQq0rHS1YVS4HoJT3rag5xgPE,1141
+warp_beacon-2.3.46.dist-info/RECORD,,