warp-beacon 2.1.1__tar.gz → 2.1.3__tar.gz

{warp_beacon-2.1.1/warp_beacon.egg-info → warp_beacon-2.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: warp_beacon
-Version: 2.1.1
+Version: 2.1.3
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

{warp_beacon-2.1.1 → warp_beacon-2.1.3}/etc/warp_beacon.conf

@@ -1,11 +1,11 @@
 TG_TOKEN=""
 TG_BOT_NAME=""
-TG_BOT_ADMIN_USERNAME=""
+TG_BOT_ADMINS_USERNAMES=""
 TG_API_ID=""
 TG_API_HASH=""
 TG_BOT_NAME=""
-TG_BOT_ADMIN_USERNAME=""
 IG_MAX_RETRIES=10
+IG_REQUEST_TIMEOUT=30
 MONGODB_HOST="mongodb"
 MONGODB_PORT="27017"
 MONGODB_USER="root"

warp_beacon-2.1.3/warp_beacon/__version__.py

@@ -0,0 +1,2 @@
+__version__ = "2.1.3"
+

{warp_beacon-2.1.1 → warp_beacon-2.1.3}/warp_beacon/jobs/abstract.py

@@ -34,6 +34,7 @@ class JobSettings(TypedDict):
 	source_username: str
 	unvailable_error_count: int
 	geoblock_error_count: int
+	account_switches: int
 
 class AbstractJob(ABC):
 	job_id: uuid.UUID = None
@@ -62,6 +63,7 @@ class AbstractJob(ABC):
 	source_usename: str = ""
 	unvailable_error_count: int = 0
 	geoblock_error_count: int = 0
+	account_switches: int = 0
 
 	def __init__(self, **kwargs: Unpack[JobSettings]) -> None:
 		if kwargs:

{warp_beacon-2.1.1 → warp_beacon-2.1.3}/warp_beacon/scraper/__init__.py

@@ -4,7 +4,7 @@ from typing import Optional
 import multiprocessing
 from queue import Empty
 
-from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, Unavailable, FileTooBig, YotubeLiveError, YotubeAgeRestrictedError, IGRateLimitAccured
+from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, Unavailable, FileTooBig, YotubeLiveError, YotubeAgeRestrictedError, IGRateLimitAccured, CaptchaIssue, AllAccountsFailed
 from warp_beacon.mediainfo.video import VideoInfo
 from warp_beacon.mediainfo.audio import AudioInfo
 from warp_beacon.mediainfo.silencer import Silencer
@@ -65,6 +65,17 @@ class AsyncDownloader(object):
 
 		return media_info
 
+	def try_next_account(self, job: DownloadJob, report_error: str = None) -> None:
+		logging.warning("Switching account!")
+		if job.account_switches > self.acc_selector.count_service_accounts(job.job_origin):
+			raise AllAccountsFailed()
+		if report_error:
+			self.acc_selector.bump_acc_fail("rate_limits")
+		self.acc_selector.next()
+		cur_acc = self.acc_selector.get_current()
+		logging.info("Current account: '%s'", str(cur_acc))
+		job.account_switches += 1
+
 	def do_work(self) -> None:
 		logging.info("download worker started")
 		while self.allow_loop.value == 1:
@@ -108,11 +119,14 @@ class AsyncDownloader(object):
 											job_failed=True,
 											job_failed_msg="Unable to access to media under this URL. Seems like the media is private.")
 										)
+										self.send_message_to_admin(
+											f"Task {job.job_id} failed. URL: '{job.url}'. Reason: 'NotFound'."
+										)
 										break
 									except Unavailable as e:
 										logging.warning("Not found or unavailable error occurred!")
 										logging.exception(e)
-										if job.unvailable_error_count > self.acc_selector.count_service_accounts(job.job_origin.value):
+										if job.unvailable_error_count > self.acc_selector.count_service_accounts(job.job_origin):
 											self.uploader.queue_task(job.to_upload_job(
 												job_failed=True,
 												job_failed_msg="Video is unvailable for all your service accounts.")
@@ -128,7 +142,10 @@ class AsyncDownloader(object):
 										logging.exception(e)
 										self.uploader.queue_task(job.to_upload_job(
 											job_failed=True,
-											job_failed_msg="Failed to download content. Please check you Internet connection or retry amount bot configuration settings.")
+											job_failed_msg="Failed to download content due timeout error. Please check you Internet connection, retry amount or request timeout bot configuration settings.")
+										)
+										self.send_message_to_admin(
+											f"Task {job.job_id} failed. URL: '{job.url}'. Reason: 'TimeOut'."
 										)
 										break
 									except FileTooBig as e:
@@ -138,15 +155,22 @@ class AsyncDownloader(object):
 											job_failed=True,
 											job_failed_msg="Unfortunately this file has exceeded the Telegram limits. A file cannot be larger than 2 gigabytes.")
 										)
+										self.send_message_to_admin(
+											f"Task {job.job_id} failed. URL: '{job.url}'. Reason: 'FileTooBig'."
+										)
 										break
 									except IGRateLimitAccured as e:
 										logging.warning("IG ratelimit accured :(")
 										logging.exception(e)
-										logging.warning("Switching Instagram account!")
-										self.acc_selector.bump_acc_fail("rate_limits")
-										self.acc_selector.next()
-										cur_acc = self.acc_selector.get_current()
-										logging.info("Current Instagram account: index: '%d', login: '%s'", cur_acc[0], cur_acc[1]["login"])
+										self.try_next_account(job, report_error="rate_limits")
+										self.job_queue.put(job)
+										break
+									except CaptchaIssue as e:
+										logging.warning("Challange accured!")
+										logging.exception(e)
+										self.try_next_account(job)
+										self.job_queue.put(job)
+										break
 									except YotubeLiveError as e:
 										logging.warning("Youtube Live videos are not supported. Skipping.")
 										logging.exception(e)
@@ -162,6 +186,20 @@ class AsyncDownloader(object):
 											job_failed=True,
 											job_failed_msg="Youtube Age Restricted error. Check your bot Youtube account settings.")
 										)
+										self.send_message_to_admin(
+											f"Task {job.job_id} failed. URL: '{job.url}'. Reason: 'YotubeAgeRestrictedError'."
+										)
+										break
+									except AllAccountsFailed as e:
+										logging.error("All accounts failed!")
+										logging.exception(e)
+										self.uploader.queue_task(job.to_upload_job(
+											job_failed=True,
+											job_failed_msg="All bot accounts failed to download content. Bot administrator noticed about the issue.")
+										)
+										self.send_message_to_admin(
+											f"Task {job.job_id} failed. URL: '{job.url}'. Reason: 'AllAccountsFailed'."
+										)
 										break
 									except (UnknownError, Exception) as e:
 										logging.warning("UnknownError occurred!")
@@ -172,7 +210,7 @@ class AsyncDownloader(object):
 										else:
 											exception_msg = str(e)
 										if "geoblock_required" in exception_msg:
-											if job.geoblock_error_count > self.acc_selector.count_service_accounts(job.job_origin.value):
+											if job.geoblock_error_count > self.acc_selector.count_service_accounts(job.job_origin):
 												self.uploader.queue_task(job.to_upload_job(
 													job_failed=True,
 													job_failed_msg="This content does not accessible for all yout bot accounts. Seems like author blocked certain regions.")

{warp_beacon-2.1.1 → warp_beacon-2.1.3}/warp_beacon/scraper/account_selector.py

@@ -77,5 +77,6 @@ class AccountSelector(object):
 	def get_meta_data(self) -> dict:
 		return self.accounts_meta_data[self.current_module_name][self.index]
 	
-	def count_service_accounts(self, mod_name: str) -> int:
-		return len(self.accounts_meta_data[mod_name])
+	def count_service_accounts(self, mod_name: Origin) -> int:
+		module_name = 'youtube' if next((s for s in ("yt", "youtube", "youtu_be") if s in mod_name.value), None) else 'instagram'
+		return len(self.accounts_meta_data[module_name])

{warp_beacon-2.1.1 → warp_beacon-2.1.3}/warp_beacon/scraper/exceptions.py

@@ -37,6 +37,12 @@ class YotubeAgeRestrictedError(ScraperError):
 class IGRateLimitAccured(ScraperError):
 	pass
 
+class CaptchaIssue(ScraperError):
+	pass
+
+class AllAccountsFailed(ScraperError):
+	pass
+
 class UnknownError(ScraperError):
 	pass
 

{warp_beacon-2.1.1 → warp_beacon-2.1.3}/warp_beacon/scraper/instagram/instagram.py

@@ -17,9 +17,9 @@ from instagrapi.mixins.story import Story
 #from instagrapi.types import Media
 from instagrapi import Client
 from instagrapi.mixins.challenge import ChallengeChoice
-from instagrapi.exceptions import LoginRequired, PleaseWaitFewMinutes, MediaNotFound, ClientNotFoundError, UserNotFound, UnknownError as IGUnknownError
+from instagrapi.exceptions import LoginRequired, PleaseWaitFewMinutes, MediaNotFound, ClientNotFoundError, UserNotFound, ChallengeRequired, ChallengeSelfieCaptcha, UnknownError as IGUnknownError
 
-from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, IGRateLimitAccured, extract_exception_message
+from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, IGRateLimitAccured, CaptchaIssue, extract_exception_message
 from warp_beacon.scraper.abstract import ScraperAbstract
 from warp_beacon.jobs.types import JobType
 from warp_beacon.telegram.utils import Utils
@@ -85,19 +85,7 @@ class InstagramScraper(ScraperAbstract):
 
 	def scrap(self, url: str) -> tuple[str]:
 		self.load_session()
-		try:
-			self.cl.get_timeline_feed()
-		except LoginRequired as e:
-			logging.error("Exception occurred while cheking IG session!")
-			logging.exception(e)
-			old_session = self.cl.get_settings()
-			self.cl.set_settings({})
-			self.setup_device()
-			self.cl.set_uuids(old_session["uuids"])
-			if os.path.exists(self.inst_session_file):
-				os.unlink(self.inst_session_file)
-			time.sleep(5)
-			return self.scrap(url)
+		self._download_hndlr(self.cl.get_timeline_feed)
 		def _scrap() -> tuple[str]:
 			if "stories" in url:
 				# remove URL options
@@ -141,6 +129,10 @@ class InstagramScraper(ScraperAbstract):
 			try:
 				ret_val = func(*args, **kwargs)
 				break
+			except (ChallengeRequired, ChallengeSelfieCaptcha) as e:
+				logging.warning("Instagram wants Challange!")
+				logging.exception(e)
+				raise CaptchaIssue()
 			except LoginRequired as e:
 				logging.error("LoginRequired occurred in download handler!")
 				logging.exception(e)
@@ -171,6 +163,7 @@ class InstagramScraper(ScraperAbstract):
 		return ret_val
 
 	def download_video(self, url: str, media_info: dict) -> dict:
+		self.cl.request_timeout = int(os.environ.get("IG_REQUEST_TIMEOUT", default=60))
 		path = self._download_hndlr(self.cl.video_download_by_url, url, folder='/tmp')
 		return {"local_media_path": str(path), "media_type": JobType.VIDEO, "media_info": {"duration": round(media_info.video_duration)}}
 
@@ -236,7 +229,7 @@ class InstagramScraper(ScraperAbstract):
 		res = []
 		wait_timeout = int(os.environ.get("IG_WAIT_TIMEOUT", default=60))
 		timeout_increment = int(os.environ.get("IG_TIMEOUT_INCREMENT", default=30))
-		ratelimit_threshold = int(os.environ.get("IG_RATELIMIT_TRESHOLD", default=5))
+		ratelimit_threshold = int(os.environ.get("IG_RATELIMIT_TRESHOLD", default=3))
 		please_wait_few_minutes_count = 1
 		while True:
 			try:

{warp_beacon-2.1.1 → warp_beacon-2.1.3}/warp_beacon/scraper/youtube/abstract.py

@@ -17,7 +17,7 @@ from PIL import Image
 
 from warp_beacon.scraper.abstract import ScraperAbstract
 from warp_beacon.mediainfo.abstract import MediaInfoAbstract
-from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, Unavailable, extract_exception_message
+from warp_beacon.scraper.exceptions import TimeOut, Unavailable, extract_exception_message
 
 from pytubefix import YouTube
 from pytubefix.innertube import _default_clients
@@ -52,7 +52,8 @@ def patched_fetch_bearer_token(self) -> None:
 	self.send_message_to_admin_func(
 		f"Please open {verification_url} and input code `{user_code}`.\n\n"
 		"Please select a Google account with verified age.\n"
-		"This will allow you to avoid error the **AgeRestrictedError** when accessing some content.")
+		"This will allow you to avoid error the **AgeRestrictedError** when accessing some content.",
+		yt_auth=True)
 	self.auth_event.wait()
 
 	data = {
@@ -137,12 +138,12 @@ class YoutubeAbstract(ScraperAbstract):
 				# do noting, not interested
 				pass
 			#except http.client.IncompleteRead as e:
-			except (socket.timeout, 
-					ssl.SSLError, 
-					http.client.IncompleteRead, 
-					http.client.HTTPException, 
-					requests.RequestException, 
-					urllib.error.URLError, 
+			except (socket.timeout,
+					ssl.SSLError,
+					http.client.IncompleteRead,
+					http.client.HTTPException,
+					requests.RequestException,
+					urllib.error.URLError,
 					urllib.error.HTTPError) as e:
 				if hasattr(e, "code") and int(e.code) == 403:
 					raise Unavailable(extract_exception_message(e))

{warp_beacon-2.1.1 → warp_beacon-2.1.3}/warp_beacon/telegram/bot.py

@@ -116,29 +116,33 @@ class Bot(object):
 
 		return 0
 
-	async def send_text_to_admin(self, text: str) -> int:
+	async def send_text_to_admin(self, text: str, yt_auth: bool = False) -> list[int]:
 		try:
-			admin = os.environ.get("TG_BOT_ADMIN_USERNAME", None)
-			if not admin:
+			admins = os.environ.get("TG_BOT_ADMINS_USERNAMES", None)
+			if not admins:
 				raise ValueError("Configuration value `TG_BOT_ADMIN_USERNAME` is empty!")
-			message_reply = await self.client.send_message(
-				chat_id=admin,
-				text=text,
-				parse_mode=ParseMode.MARKDOWN,
-				reply_markup=InlineKeyboardMarkup(
-					[
+			
+			msg_ids = []
+			admins_array = admins.split(',')
+			for adm in admins_array:
+				adm = adm.strip()
+				msg_opts = {"chat_id": adm, "text": text, "parse_mode": ParseMode.MARKDOWN}
+				if yt_auth:
+					msg_opts["reply_markup"] = InlineKeyboardMarkup(
 						[
-							InlineKeyboardButton("✅ Done", callback_data="auth_process_done")
+							[
+								InlineKeyboardButton("✅ Done", callback_data="auth_process_done")
+							]
 						]
-					]
-				)
-			)
-			return message_reply.id
+					)
+				message_reply = await self.client.send_message(**msg_opts)
+				msg_ids.append(message_reply.id)
+			return msg_ids
 		except Exception as e:
 			logging.error("Failed to send text message to admin!")
 			logging.exception(e)
 
-		return 0
+		return []
 
 	def build_tg_args(self, job: UploadJob) -> dict:
 		args = {}

{warp_beacon-2.1.1 → warp_beacon-2.1.3/warp_beacon.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: warp_beacon
-Version: 2.1.1
+Version: 2.1.3
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

warp_beacon-2.1.1/warp_beacon/__version__.py

@@ -1,2 +0,0 @@
-__version__ = "2.1.1"
-