warp-beacon 2.0.10__py3-none-any.whl → 2.1.1__py3-none-any.whl

etc/warp_beacon/warp_beacon.conf

@@ -1,14 +1,15 @@
 TG_TOKEN=""
+TG_BOT_NAME=""
+TG_BOT_ADMIN_USERNAME=""
 TG_API_ID=""
 TG_API_HASH=""
 TG_BOT_NAME=""
 TG_BOT_ADMIN_USERNAME=""
-INSTAGRAM_LOGIN=""
-INSTAGRAM_PASSWORD=""
-INSTAGRAM_VERIFICATION_CODE=""
+IG_MAX_RETRIES=10
 MONGODB_HOST="mongodb"
 MONGODB_PORT="27017"
 MONGODB_USER="root"
 MONGODB_PASSWORD="changeme"
-VIDEO_STORAGE_DIR="/var/warp_beacon/videos"
-ENABLE_DONATES=true
+ENABLE_DONATES=true
+SERVICE_ACCOUNTS_FILE=/var/warp_beacon/accounts.json
+FORCE_IPV6=true

var/warp_beacon/accounts.json

@@ -0,0 +1,61 @@
+{
+	"instagram":
+	[
+		{
+			"login": "ig_account0",
+			"password": "ig_password",
+			"imap_server": "imap.gmail.com",
+			"imap_login": "user@gmail.com",
+			"imap_password": "",
+			"auth_details":
+			{
+				"delay_range": [1, 3],
+				"country_code": 7,
+				"locale": "en_US",
+				"timezone_offset": 10800,
+				"user_agent": "Barcelona 291.0.0.31.111 Android (33/13; 600dpi; 1440x3044; samsung; SM-G998B; p3s; exynos2100; en_US; 493450264)",
+				"device":
+				{
+					"app_version": "291.0.0.31.111",
+					"android_version": 33,
+					"android_release": "13.0.0",
+					"dpi": "600dpi",
+					"resolution": "1440x3044",
+					"manufacturer": "Samsung",
+					"device": "p3s",
+					"model": "SM-G998B",
+					"cpu": "exynos2100",
+					"version_code": "493450264"
+				}
+			}
+		},
+		{
+			"login": "ig_account1",
+			"password": "passwd",
+			"imap_server": "imap.gmail.com",
+			"imap_login": "mail_login1",
+			"imap_password": "imap_password1",
+			"auth_details":
+			{
+				"delay_range": [1, 3],
+				"country_code": 7,
+				"locale": "en_US",
+				"timezone_offset": 10800,
+				"user_agent": "Barcelona 291.0.0.31.111 Android (33/13; 600dpi; 1440x3044; samsung; SM-G998B; p3s; exynos2100; en_US; 493450264)",
+				"device":
+				{
+					"app_version": "291.0.0.31.111",
+					"android_version": 33,
+					"android_release": "13.0.0",
+					"dpi": "600dpi",
+					"resolution": "1440x3044",
+					"manufacturer": "Samsung",
+					"device": "p3s",
+					"model": "SM-G998B",
+					"cpu": "exynos2100",
+					"version_code": "493450264"
+				}
+			}
+		}
+	]
+}

warp_beacon/__version__.py

@@ -1,2 +1,2 @@
-__version__ = "2.0.10"
+__version__ = "2.1.1"
 

warp_beacon/jobs/abstract.py

@@ -31,6 +31,9 @@ class JobSettings(TypedDict):
 	canonical_name: str
 	is_message_to_admin: bool
 	message_text: str
+	source_username: str
+	unvailable_error_count: int
+	geoblock_error_count: int
 
 class AbstractJob(ABC):
 	job_id: uuid.UUID = None
@@ -56,6 +59,9 @@ class AbstractJob(ABC):
 	canonical_name: str = ""
 	is_message_to_admin: bool = False
 	message_text: str = ""
+	source_usename: str = ""
+	unvailable_error_count: int = 0
+	geoblock_error_count: int = 0
 
 	def __init__(self, **kwargs: Unpack[JobSettings]) -> None:
 		if kwargs:

warp_beacon/scraper/__init__.py

@@ -1,12 +1,10 @@
 import os
-import time
-import asyncio
 
 from typing import Optional
 import multiprocessing
 from queue import Empty
 
-from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, Unavailable, FileTooBig, YotubeLiveError, YotubeAgeRestrictedError
+from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, Unavailable, FileTooBig, YotubeLiveError, YotubeAgeRestrictedError, IGRateLimitAccured
 from warp_beacon.mediainfo.video import VideoInfo
 from warp_beacon.mediainfo.audio import AudioInfo
 from warp_beacon.mediainfo.silencer import Silencer
@@ -16,9 +14,12 @@ from warp_beacon.jobs import Origin
 from warp_beacon.jobs.download_job import DownloadJob
 from warp_beacon.jobs.upload_job import UploadJob
 from warp_beacon.jobs.types import JobType
+from warp_beacon.scraper.account_selector import AccountSelector
 
 import logging
 
+ACC_FILE = os.environ.get("SERVICE_ACCOUNTS_FILE", default="/var/warp_beacon/instagram_accounts.json")
+
 class AsyncDownloader(object):
 	__JOE_BIDEN_WAKEUP = None
 	workers = []
@@ -27,11 +28,13 @@ class AsyncDownloader(object):
 	uploader = None
 	workers_count = 0
 	auth_event = multiprocessing.Event()
+	acc_selector = None
 
 	def __init__(self, uploader: AsyncUploader, workers_count: int) -> None:
 		self.allow_loop = multiprocessing.Value('i', 1)
 		self.uploader = uploader
 		self.workers_count = workers_count
+		self.acc_selector = AccountSelector(ACC_FILE)
 
 	def __del__(self) -> None:
 		self.stop_all()
@@ -77,18 +80,19 @@ class AsyncDownloader(object):
 						if job.job_origin is not Origin.UNKNOWN:
 							if not job.in_process:
 								actor = None
+								self.acc_selector.set_module(job.job_origin)
 								if job.job_origin is Origin.INSTAGRAM:
-									from warp_beacon.scraper.instagram import InstagramScraper
-									actor = InstagramScraper()
+									from warp_beacon.scraper.instagram.instagram import InstagramScraper
+									actor = InstagramScraper(self.acc_selector.get_current())
 								elif job.job_origin is Origin.YT_SHORTS:
 									from warp_beacon.scraper.youtube.shorts import YoutubeShortsScraper
-									actor = YoutubeShortsScraper()
+									actor = YoutubeShortsScraper(self.acc_selector.get_current())
 								elif job.job_origin is Origin.YT_MUSIC:
 									from warp_beacon.scraper.youtube.music import YoutubeMusicScraper
-									actor = YoutubeMusicScraper()
+									actor = YoutubeMusicScraper(self.acc_selector.get_current())
 								elif job.job_origin is Origin.YOUTUBE:
 									from warp_beacon.scraper.youtube.youtube import YoutubeScraper
-									actor = YoutubeScraper()
+									actor = YoutubeScraper(self.acc_selector.get_current())
 								#self.auth_event = multiprocessing.Event()
 								actor.send_message_to_admin_func = self.send_message_to_admin
 								actor.auth_event = self.auth_event
@@ -97,14 +101,28 @@ class AsyncDownloader(object):
 										logging.info("Downloading URL '%s'", job.url)
 										items = actor.download(job.url)
 										break
-									except (NotFound, Unavailable) as e:
-										logging.warning("Not found or unavailable error occurred!")
+									except NotFound as e:
+										logging.warning("Not found error occurred!")
 										logging.exception(e)
 										self.uploader.queue_task(job.to_upload_job(
 											job_failed=True,
 											job_failed_msg="Unable to access to media under this URL. Seems like the media is private.")
 										)
 										break
+									except Unavailable as e:
+										logging.warning("Not found or unavailable error occurred!")
+										logging.exception(e)
+										if job.unvailable_error_count > self.acc_selector.count_service_accounts(job.job_origin.value):
+											self.uploader.queue_task(job.to_upload_job(
+												job_failed=True,
+												job_failed_msg="Video is unvailable for all your service accounts.")
+											)
+											break
+										job.unvailable_error_count += 1
+										logging.info("Trying to switch account")
+										self.acc_selector.next()
+										self.job_queue.put(job)
+										break
 									except TimeOut as e:
 										logging.warning("Timeout error occurred!")
 										logging.exception(e)
@@ -121,6 +139,14 @@ class AsyncDownloader(object):
 											job_failed_msg="Unfortunately this file has exceeded the Telegram limits. A file cannot be larger than 2 gigabytes.")
 										)
 										break
+									except IGRateLimitAccured as e:
+										logging.warning("IG ratelimit accured :(")
+										logging.exception(e)
+										logging.warning("Switching Instagram account!")
+										self.acc_selector.bump_acc_fail("rate_limits")
+										self.acc_selector.next()
+										cur_acc = self.acc_selector.get_current()
+										logging.info("Current Instagram account: index: '%d', login: '%s'", cur_acc[0], cur_acc[1]["login"])
 									except YotubeLiveError as e:
 										logging.warning("Youtube Live videos are not supported. Skipping.")
 										logging.exception(e)
@@ -146,10 +172,16 @@ class AsyncDownloader(object):
 										else:
 											exception_msg = str(e)
 										if "geoblock_required" in exception_msg:
-											self.uploader.queue_task(job.to_upload_job(
-												job_failed=True,
-												job_failed_msg="This content does not accessible for bot account. Seems like author blocked certain region.")
-											)
+											if job.geoblock_error_count > self.acc_selector.count_service_accounts(job.job_origin.value):
+												self.uploader.queue_task(job.to_upload_job(
+													job_failed=True,
+													job_failed_msg="This content does not accessible for all yout bot accounts. Seems like author blocked certain regions.")
+												)
+												break
+											job.geoblock_error_count += 1
+											logging.info("Trying to switch account")
+											self.acc_selector.next()
+											self.job_queue.put(job)
 											break
 										self.uploader.queue_task(job.to_upload_job(
 											job_failed=True,

warp_beacon/scraper/abstract.py

@@ -1,6 +1,9 @@
 import os
 import pathlib
 
+import socket
+import requests.packages.urllib3.util.connection as urllib3_cn
+
 from abc import ABC, abstractmethod
 from typing import Callable, Union
 
@@ -10,14 +13,21 @@ from pillow_heif import register_heif_opener
 import logging
 
 class ScraperAbstract(ABC):
+	original_gai_family = None
 	send_message_to_admin_func = None
 	auth_event = None
+	account = None
+	account_index = 0
 
-	def __init__(self) -> None:
-		pass
+	def __init__(self, account: tuple) -> None:
+		self.account_index = account[0]
+		self.account = account[1]
+		if os.environ.get("FORCE_IPV6", default="false") == "true":
+			self.force_ipv6()
 
 	def __del__(self) -> None:
-		pass
+		if os.environ.get("FORCE_IPV6", default="false") == "true":
+			self.restore_gai()
 
 	@abstractmethod
 	def download(self, url: str) -> bool:
@@ -63,3 +73,21 @@ class ScraperAbstract(ABC):
 			logging.exception(e)
 
 		return ''
+
+	def force_ipv6(self) -> None:
+		def allowed_gai_family():
+			"""
+			https://github.com/shazow/urllib3/blob/master/urllib3/util/connection.py
+			"""
+			family = socket.AF_INET
+			if urllib3_cn.HAS_IPV6:
+				family = socket.AF_INET6 # force ipv6 only if it is available
+			return family
+		if self.original_gai_family is None:
+			self.original_gai_family = urllib3_cn.allowed_gai_family
+		logging.info("Forcing IPv6 ...")
+		urllib3_cn.allowed_gai_family = allowed_gai_family
+
+	def restore_gai(self) -> None:
+		logging.info("Restoring normal IP stack ...")
+		urllib3_cn.allowed_gai_family = self.original_gai_family

warp_beacon/scraper/account_selector.py

@@ -0,0 +1,81 @@
+import os
+import json
+import re
+
+from itertools import cycle
+
+from warp_beacon.jobs import Origin
+
+import logging
+
+class AccountSelector(object):
+	accounts = []
+	acc_pool = None
+	current = None
+	current_module_name = None
+	index = 0
+	accounts_meta_data = {}
+	session_dir = "/var/warp_beacon"
+
+	def __init__(self, acc_file_path: str) -> None:
+		if os.path.exists(acc_file_path):
+			with open(acc_file_path, 'r', encoding="utf-8") as f:
+				self.accounts = json.loads(f.read())
+			if self.accounts:
+				self.__init_meta_data()
+				self.load_yt_sessions()
+		else:
+			raise ValueError("Accounts file not found")
+
+	def __del__(self) -> None:
+		pass
+
+	#def enrich_service_data(self) -> None:
+	#	for k, v in self.accounts.items():
+
+	def load_yt_sessions(self) -> None:
+		self.accounts["youtube"] = []
+		for f in os.listdir(self.session_dir):
+			if "yt_session" in f and ".json" in f:
+				match = re.search('\d+', f)
+				index = 0
+				if match:
+					index = int(match.group(0))
+				self.accounts["youtube"].insert(index, {"session_file": "%s/%s" % (self.session_dir, f), "index": index})
+		if not self.accounts["youtube"]:
+			self.accounts["youtube"].append({"session_file": "%s/yt_session_0.json" % self.session_dir, "index": 0})
+
+	def __init_meta_data(self) -> None:
+		for module_name, lst in self.accounts.items():
+			if module_name not in self.accounts_meta_data:
+				self.accounts_meta_data[module_name] = []
+			for index, _ in enumerate(lst):
+				self.accounts_meta_data[module_name].insert(index, {"auth_fails": 0, "rate_limits": 0})
+
+	def set_module(self, module_origin: Origin) -> None:
+		module_name = 'youtube' if next((s for s in ("yt", "youtube", "youtu_be") if s in module_origin.value), None) else 'instagram'
+		self.current_module_name = module_name
+		self.acc_pool = cycle(self.accounts[module_name])
+		self.current = next(self.acc_pool)
+		self.index = self.accounts[module_name].index(self.current)
+
+	def next(self) -> dict:
+		self.current = next(self.acc_pool)
+		self.index = self.accounts[self.current_module_name].index(self.current)
+		return self.current
+	
+	def bump_acc_fail(self, key: str, amount: int = 1) -> int:
+		self.accounts_meta_data[self.index][key] += amount
+		return self.accounts_meta_data[self.index][key]
+
+	def how_much(self, key: str) -> int:
+		return self.accounts_meta_data[self.current_module_name][self.index][key]
+	
+	def get_current(self) -> tuple:
+		return (self.index, self.current)
+	
+	def get_meta_data(self) -> dict:
+		return self.accounts_meta_data[self.current_module_name][self.index]
+	
+	def count_service_accounts(self, mod_name: str) -> int:
+		return len(self.accounts_meta_data[mod_name])

warp_beacon/scraper/exceptions.py

@@ -34,6 +34,9 @@ class YotubeLiveError(ScraperError):
 class YotubeAgeRestrictedError(ScraperError):
 	pass
 
+class IGRateLimitAccured(ScraperError):
+	pass
+
 class UnknownError(ScraperError):
 	pass
 

warp_beacon/scraper/{instagram.py → instagram/instagram.py}

@@ -1,64 +1,103 @@
 import os
 import time
-
 import socket
 import ssl
-
+import re
 from typing import Callable, Optional, Union
 
-from pathlib import Path
+import random
+import email
+import imaplib
 import json
-
 import requests
 import urllib3
 from urllib.parse import urljoin, urlparse
 
 from instagrapi.mixins.story import Story
-from instagrapi.types import Media
+#from instagrapi.types import Media
 from instagrapi import Client
+from instagrapi.mixins.challenge import ChallengeChoice
 from instagrapi.exceptions import LoginRequired, PleaseWaitFewMinutes, MediaNotFound, ClientNotFoundError, UserNotFound, UnknownError as IGUnknownError
 
-from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, extract_exception_message
+from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, IGRateLimitAccured, extract_exception_message
 from warp_beacon.scraper.abstract import ScraperAbstract
 from warp_beacon.jobs.types import JobType
 from warp_beacon.telegram.utils import Utils
 
 import logging
 
-INST_SESSION_FILE = "/var/warp_beacon/inst_session.json"
+INST_SESSION_FILE_TPL = "/var/warp_beacon/inst_session_account_%d.json"
 
 class InstagramScraper(ScraperAbstract):
 	cl = None
+	inst_session_file = ""
 
-	def __init__(self) -> None:
-		super().__init__()
+	def __init__(self, account: tuple) -> None:
+		super().__init__(account)
+		#
+		self.inst_session_file = INST_SESSION_FILE_TPL % self.account_index
 		self.cl = Client()
+		self.setup_device()
+		self.cl.challenge_code_handler = self.challenge_code_handler
+		self.cl.change_password_handler = self.change_password_handler
+
+	def setup_device(self) -> None:
+		details = self.account.get("auth_details", {})
+		self.cl.delay_range = details.get("delay_range", [1, 3])
+		self.cl.set_country_code(details.get("country_code", 1))
+		self.cl.set_locale(details.get("locale", "en_US"))
+		self.cl.set_timezone_offset(details.get("timezone_offset", 10800))
+		self.cl.set_user_agent(details.get("user_agent", "Barcelona 291.0.0.31.111 Android (33/13; 600dpi; 1440x3044; samsung; SM-G998B; p3s; exynos2100; en_US; 493450264)"))
+		device = details.get("device", {})
+		self.cl.set_device({
+			"app_version": device.get("app_version", "291.0.0.31.111"),
+			"android_version": device.get("android_version", 33),
+			"android_release": device.get("android_release", "13.0.0"),
+			"dpi": device.get("dpi", "600dpi"),
+			"resolution": device.get("resolution", "1440x3044"),
+			"manufacturer": device.get("manufacturer", "Samsung"),
+			"device": device.get("device", "p3s"),
+			"model": device.get("model", "SM-G998B"),
+			"cpu": device.get("cpu", "exynos2100"),
+			"version_code": device.get("version_code", "493450264")
+		})
 
 	def safe_write_session(self) -> None:
-		tmp_fname = "%s~" % INST_SESSION_FILE
-		with open(tmp_fname, 'w+') as f:
+		tmp_fname = "%s~" % self.inst_session_file
+		with open(tmp_fname, 'w+', encoding="utf-8") as f:
 			f.write(json.dumps(self.cl.get_settings()))
-		if os.path.isfile(INST_SESSION_FILE):
-			os.unlink(INST_SESSION_FILE)
-		os.rename(tmp_fname, INST_SESSION_FILE)
+		if os.path.exists(self.inst_session_file):
+			os.unlink(self.inst_session_file)
+		os.rename(tmp_fname, self.inst_session_file)
 
 	def load_session(self) -> None:
-		if os.path.exists(INST_SESSION_FILE):
-			self.cl.load_settings(INST_SESSION_FILE)
+		if os.path.exists(self.inst_session_file):
+			self.cl.load_settings(self.inst_session_file)
 		else:
 			self.login()
 
 	def login(self) -> None:
-		self.cl = Client()
-		username = os.environ.get("INSTAGRAM_LOGIN", default=None)
-		password = os.environ.get("INSTAGRAM_PASSWORD", default=None)
-		verification_code = os.environ.get("INSTAGRAM_VERIFICATION_CODE", default="")
-		if username is not None and password is not None:
-			self.cl.login(username=username, password=password, verification_code=verification_code)
+		username = self.account["login"]
+		password = self.account["password"]
+		if username and password:
+			self.cl.login(username=username, password=password, verification_code="")
 		self.safe_write_session()
 
 	def scrap(self, url: str) -> tuple[str]:
 		self.load_session()
+		try:
+			self.cl.get_timeline_feed()
+		except LoginRequired as e:
+			logging.error("Exception occurred while cheking IG session!")
+			logging.exception(e)
+			old_session = self.cl.get_settings()
+			self.cl.set_settings({})
+			self.setup_device()
+			self.cl.set_uuids(old_session["uuids"])
+			if os.path.exists(self.inst_session_file):
+				os.unlink(self.inst_session_file)
+			time.sleep(5)
+			return self.scrap(url)
 		def _scrap() -> tuple[str]:
 			if "stories" in url:
 				# remove URL options
@@ -102,6 +141,17 @@ class InstagramScraper(ScraperAbstract):
 			try:
 				ret_val = func(*args, **kwargs)
 				break
+			except LoginRequired as e:
+				logging.error("LoginRequired occurred in download handler!")
+				logging.exception(e)
+				old_session = self.cl.get_settings()
+				self.cl.set_settings({})
+				self.setup_device()
+				self.cl.set_uuids(old_session["uuids"])
+				if os.path.exists(self.inst_session_file):
+					os.unlink(self.inst_session_file)
+				time.sleep(5)
+				self.load_session()
 			except (socket.timeout,
 					ssl.SSLError,
 					requests.exceptions.ConnectionError,
@@ -173,7 +223,7 @@ class InstagramScraper(ScraperAbstract):
 		for media_chunk in Utils.chunker(media_info.resources, 10):
 			chunk = []
 			for media in media_chunk:
-				_media_info = self.cl.media_info(media.pk)
+				_media_info = self._download_hndlr(self.cl.media_info, media.pk)
 				if media.media_type == 1: # photo
 					chunk.append(self.download_photo(url=_media_info.thumbnail_url))
 				elif media.media_type == 2: # video
@@ -184,6 +234,10 @@ class InstagramScraper(ScraperAbstract):
 
 	def download(self, url: str) -> Optional[list[dict]]:
 		res = []
+		wait_timeout = int(os.environ.get("IG_WAIT_TIMEOUT", default=60))
+		timeout_increment = int(os.environ.get("IG_TIMEOUT_INCREMENT", default=30))
+		ratelimit_threshold = int(os.environ.get("IG_RATELIMIT_TRESHOLD", default=5))
+		please_wait_few_minutes_count = 1
 		while True:
 			try:
 				scrap_type, media_id = self.scrap(url)
@@ -205,10 +259,14 @@ class InstagramScraper(ScraperAbstract):
 					res.append(self.download_stories(self.scrap_stories(media_id)))
 				break
 			except PleaseWaitFewMinutes as e:
+				please_wait_few_minutes_count += 1
 				logging.warning("Please wait a few minutes error. Trying to relogin ...")
 				logging.exception(e)
-				wait_timeout = int(os.environ.get("IG_WAIT_TIMEOUT", default=5))
-				logging.info("Waiting %d seconds according configuration option `IG_WAIT_TIMEOUT`", wait_timeout)
+				if please_wait_few_minutes_count >= ratelimit_threshold:
+					logging.warning("IG ratelimit accured")
+					raise IGRateLimitAccured()
+				wait_timeout += timeout_increment
+				logging.info("Waiting %d seconds according configuration option `IG_WAIT_TIMEOUT` with `IG_TIMEOUT_INCREMENT`", wait_timeout)
 				if res:
 					for i in res:
 						if i["media_type"] == JobType.COLLECTION:
@@ -218,10 +276,72 @@ class InstagramScraper(ScraperAbstract):
 						else:
 							if os.path.exists(i["local_media_path"]):
 								os.unlink(i["local_media_path"])
-				os.unlink(INST_SESSION_FILE)
+				if os.path.exists(self.inst_session_file):
+					os.unlink(self.inst_session_file)
+				self.login()
 				time.sleep(wait_timeout)
 			except (MediaNotFound, ClientNotFoundError, UserNotFound) as e:
 				raise NotFound(extract_exception_message(e))
 			except IGUnknownError as e:
 				raise UnknownError(extract_exception_message(e))
 		return res
+	
+	def email_challenge_resolver(self, username: str) -> Optional[str]:
+		logging.info("Started email challenge resolver")
+		mail = imaplib.IMAP4_SSL(self.account.get("imap_server", default="imap.bagrintsev.me"))
+		mail.login(self.account.get("imap_login", default=""), self.account.get("imap_password", default="")) # email server creds
+		mail.select("inbox")
+		_, data = mail.search(None, "(UNSEEN)")
+		ids = data.pop().split()
+		for num in reversed(ids):
+			mail.store(num, "+FLAGS", "\\Seen")  # mark as read
+			_, data = mail.fetch(num, "(RFC822)")
+			msg = email.message_from_string(data[0][1].decode())
+			payloads = msg.get_payload()
+			if not isinstance(payloads, list):
+				payloads = [msg]
+			code = None
+			for payload in payloads:
+				body = ''
+				try:
+					body = payload.get_payload(decode=True).decode()
+				except:
+					continue
+				if "<div" not in body:
+					continue
+				match = re.search(">([^>]*?({u})[^<]*?)<".format(u=username), body)
+				if not match:
+					continue
+				logging.info("Match from email: '%s'", match.group(1))
+				match = re.search(r">(\d{6})<", body)
+				if not match:
+					logging.info('Skip this email, "code" not found')
+					continue
+				code = match.group(1)
+				if code:
+					logging.info("Found IG code at mail server: '%s'", code)
+					return code
+		return None
+
+	def get_code_from_sms(self, username: str) -> Optional[str]:
+		while True:
+			code = input(f"Enter code (6 digits) for {username}: ").strip()
+			if code and code.isdigit():
+				return code
+		return None
+
+	def challenge_code_handler(self, username: str, choice: ChallengeChoice) -> bool:
+		if choice == ChallengeChoice.SMS:
+			return False
+			#return self.get_code_from_sms(username)
+		elif choice == ChallengeChoice.EMAIL:
+			return self.email_challenge_resolver(username)
+		
+		return False
+
+	def change_password_handler(self, username: str) -> str:
+		# Simple way to generate a random string
+		chars = list("abcdefghijklmnopqrstuvwxyz1234567890!&£@#")
+		password = "".join(random.sample(chars, 10))
+		logging.info("Generated new IG password: '%s'", password)
+		return password

warp_beacon/scraper/youtube/abstract.py

@@ -4,7 +4,7 @@ import time
 import socket
 import ssl
 
-from abc import abstractmethod
+#from abc import abstractmethod
 from typing import Callable, Union
 
 import json
@@ -78,10 +78,10 @@ def patched_fetch_bearer_token(self) -> None:
 
 class YoutubeAbstract(ScraperAbstract):
 	DOWNLOAD_DIR = "/tmp"
-	YT_SESSION_FILE = '/var/warp_beacon/yt_session.json'
+	YT_SESSION_FILE = '/var/warp_beacon/yt_session_%d.json'
 
-	def __init__(self) -> None:
-		pass
+	def __init__(self, account: tuple) -> None:
+		super().__init__(account)
 
 	def __del__(self) -> None:
 		pass
@@ -91,7 +91,7 @@ class YoutubeAbstract(ScraperAbstract):
 			raise NameError("No file provided")
 		path_info = pathlib.Path(filename)
 		ext = path_info.suffix
-		old_filename = path_info.stem
+		#old_filename = path_info.stem
 		time_name = str(time.time()).replace('.', '_')
 		new_filename = "%s%s" % (time_name, ext)
 		new_filepath = "%s/%s" % (os.path.dirname(filename), new_filename)
@@ -170,11 +170,9 @@ class YoutubeAbstract(ScraperAbstract):
 		InnerTube.fetch_bearer_token = patched_fetch_bearer_token
 		_default_clients["ANDROID"]["innertube_context"]["context"]["client"]["clientVersion"] = "19.08.35"
 		_default_clients["ANDROID_MUSIC"] = _default_clients["ANDROID"]
-		return YouTube(
-			url=url,
-			#client="WEB",
-			use_oauth=True,
-			allow_oauth_cache=True,
-			token_file=self.YT_SESSION_FILE,
-			on_progress_callback = self.yt_on_progress
-		)
+		yt_opts = {"url": url, "on_progress_callback": self.yt_on_progress}
+		#yt_opts["client"] = "WEB"
+		yt_opts["use_oauth"] = True
+		yt_opts["allow_oauth_cache"] = True
+		yt_opts["token_file"] = self.YT_SESSION_FILE % self.account_index
+		return YouTube(**yt_opts)

warp_beacon/scraper/youtube/music.py

@@ -2,8 +2,6 @@ from warp_beacon.jobs.types import JobType
 from warp_beacon.scraper.youtube.abstract import YoutubeAbstract
 from warp_beacon.scraper.exceptions import NotFound
 
-from pytubefix import YouTube
-
 import logging
 
 class YoutubeMusicScraper(YoutubeAbstract):

warp_beacon/scraper/youtube/shorts.py

@@ -2,8 +2,6 @@ from warp_beacon.jobs.types import JobType
 from warp_beacon.scraper.youtube.abstract import YoutubeAbstract
 from warp_beacon.scraper.exceptions import NotFound
 
-from pytubefix import YouTube
-
 import logging
 
 class YoutubeShortsScraper(YoutubeAbstract):

warp_beacon/scraper/youtube/youtube.py

@@ -2,7 +2,6 @@ from warp_beacon.jobs.types import JobType
 from warp_beacon.scraper.youtube.abstract import YoutubeAbstract
 from warp_beacon.scraper.exceptions import YotubeLiveError, NotFound, YotubeAgeRestrictedError
 
-from pytubefix import YouTube
 from pytubefix.exceptions import AgeRestrictedError
 
 import logging

warp_beacon/telegram/bot.py

@@ -50,7 +50,7 @@ class Bot(object):
 			bot_token=tg_token,
 			api_id=tg_api_id,
 			api_hash=tg_api_hash,
-			workdir='/',
+			workdir='/var/warp_beacon',
 			workers=int(os.environ.get("TG_WORKERS_POOL_SIZE", default=workers_amount))
 		)
 
@@ -373,4 +373,4 @@ class Bot(object):
 		finally:
 			job.remove_files()
 
-		return tg_file_ids
+		return tg_file_ids

warp_beacon/telegram/utils.py

@@ -1,9 +1,9 @@
-import re
+from typing import Union
 
+import re
 import requests
 
-from enum import Enum
-from typing import Union
+from pyrogram.types import Message
 
 from warp_beacon.jobs import Origin
 from warp_beacon.jobs.types import JobType
@@ -14,7 +14,7 @@ class Utils(object):
 	expected_patronum_compiled_re = re.compile(r'Expected ([A-Z]+), got ([A-Z]+) file id instead')
 
 	@staticmethod
-	def extract_file_id(message: "Message") -> Union[None, str]:
+	def extract_file_id(message: Message) -> Union[None, str]:
 		possible_attrs = ("video", "photo", "audio", "animation", "document")
 		for attr in possible_attrs:
 			if hasattr(message, attr):
@@ -73,10 +73,17 @@ class Utils(object):
 		return (seq[pos:pos + size] for pos in range(0, len(seq), size))
 
 	@staticmethod
-	def extract_message_text(message: "Message") -> str:
+	def extract_message_text(message: Message) -> str:
 		if hasattr(message, "text") and message.text:
 			return message.text
 		if hasattr(message, "caption") and message.caption:
 			return message.caption
 
 		return ''
+
+	@staticmethod
+	def extract_message_author(message: Message) -> str:
+		if message.from_user:
+			return message.from_user
+		return ''
+

{warp_beacon-2.0.10.dist-info → warp_beacon-2.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: warp_beacon
-Version: 2.0.10
+Version: 2.1.1
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev
@@ -230,7 +230,7 @@ Requires-Dist: tgcrypto
 Requires-Dist: pyrogram
 Requires-Dist: pillow-heif
 Requires-Dist: pytubefix
-Requires-Dist: av
+Requires-Dist: av ==12.3.0
 Requires-Dist: urlextract
 Requires-Dist: pillow
 Requires-Dist: pymongo

warp_beacon-2.1.1.dist-info/RECORD

@@ -0,0 +1,43 @@
+etc/warp_beacon/warp_beacon.conf,sha256=pEhCjw6VBCN4c61rTJ4RwOKxAggpy0VvTcgObXFZpaI,318
+lib/systemd/system/warp_beacon.service,sha256=lPmHqLqcI2eIV7nwHS0qcALQrznixqJuwwPfa2mDLUA,372
+var/warp_beacon/accounts.json,sha256=rKFQM_b9eoDS4mJ1B_SZNolPLXx1SQdQMdY2F_ZcBt8,1523
+var/warp_beacon/placeholder.gif,sha256=cE5CGJVaop4Sx21zx6j4AyoHU0ncmvQuS2o6hJfEH88,6064
+warp_beacon/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+warp_beacon/__version__.py,sha256=0npkOCld8bseamcx2JJABJTYUri9sDUBFFPsagH-KOs,23
+warp_beacon/warp_beacon.py,sha256=7KEtZDj-pdhtl6m-zFLsSojs1ZR4o7L0xbqtdmYPvfE,342
+warp_beacon/compress/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+warp_beacon/compress/video.py,sha256=_PDMVYCyzLYxHv1uZmmzGcG_8rjaZr7BTXsXTTy_oS4,2846
+warp_beacon/jobs/__init__.py,sha256=ED8_tPle4iL4kqNW0apAVkgNQtRRTnYfAJwBjO1g0JY,180
+warp_beacon/jobs/abstract.py,sha256=fXdYgJcCJhsuxf_4nXzsA9cZt_dGbTr-tjvbzs_m_vQ,2631
+warp_beacon/jobs/download_job.py,sha256=5HiPcnJppFMhO14___3eSkoMygM3y-vhpGkMAuNhK7s,854
+warp_beacon/jobs/types.py,sha256=Ae8zINgbs7cOcYkYoOCOACA7duyhnIGMQAJ_SJB1QRQ,176
+warp_beacon/jobs/upload_job.py,sha256=_ul4psPej1jLEs-BMcMR80GbXDSmm38jE9yoZtecclY,741
+warp_beacon/mediainfo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+warp_beacon/mediainfo/abstract.py,sha256=ZR2JMuRpoh7nDNov9a8YkAfr6BI2HXnXzQtVrLgDxjs,1185
+warp_beacon/mediainfo/audio.py,sha256=ous88kwQj4bDIChN5wnGil5LqTs0IQHH0d-nyrL0-ZM,651
+warp_beacon/mediainfo/silencer.py,sha256=MgUc9Ibbhjhg9GbJMNfJqrdDkMsQShZkQ1sCwvW_-qI,1647
+warp_beacon/mediainfo/video.py,sha256=AIRy_op_BvehsjarM1rvT5Qo0QWwf-Q6xVVd_aCnbJ4,2505
+warp_beacon/scraper/__init__.py,sha256=QZOyrPh5PTgFh70Qqh629RPWi4Ed-kNipiwsSlS7Mnk,12791
+warp_beacon/scraper/abstract.py,sha256=aNZ9ypF9B8BjflcIwi-7wEzIqF-XPeF0xvfX9CP_iIw,2708
+warp_beacon/scraper/account_selector.py,sha256=SbDO7W5WUCmMDokhuSx0yfEQiu5TW9UumwScGHsaGk8,2633
+warp_beacon/scraper/exceptions.py,sha256=5A1viJm8sT_ZcI1IccAxQ9O_eYpJNHt3TmYLM0P22Q8,1238
+warp_beacon/scraper/instagram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+warp_beacon/scraper/instagram/instagram.py,sha256=MNfYOMYzaVG65wrtL63oQu6BTq6i_MuzjBxsoEMlFYA,13327
+warp_beacon/scraper/youtube/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+warp_beacon/scraper/youtube/abstract.py,sha256=Fp_NL8Y5j5Tk9_d5QDfkCQZva7I0JC_1B26RhorMjtw,6065
+warp_beacon/scraper/youtube/music.py,sha256=8TguqXc1BC9Slp09BzCU8dtQ6BnyKwzZSLLFHpqDHtg,1498
+warp_beacon/scraper/youtube/shorts.py,sha256=pHfvEBau8Zp7Ar3LBuPmjqYq8fmjJUQvzeZXujQMpG4,1203
+warp_beacon/scraper/youtube/youtube.py,sha256=K98n2TSJaDZt-xT7mADZL1UEf2exIYm0Wnenn2GAYfI,2250
+warp_beacon/storage/__init__.py,sha256=8XsJXq9X7GDlTaWREF4W1PDX9PH5utwhjf5c5M8Bb7o,3378
+warp_beacon/telegram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+warp_beacon/telegram/bot.py,sha256=Y3QT5iIBdM6ILgN832asCbzr2K0v0hYK7IxwV7aC48M,13100
+warp_beacon/telegram/handlers.py,sha256=MTcHZmWe8RAcZdicnqQewy_SkwujhnaoqJgWHpebfVs,6350
+warp_beacon/telegram/placeholder_message.py,sha256=u5kVfTjGmVYkwA5opniRltHXGpsdSxI41WEde8J5os0,6418
+warp_beacon/telegram/utils.py,sha256=LdCU4ChJHyzpCvyG5v3XcUtUgK3v5by_v8D56VsPeI0,2171
+warp_beacon/uploader/__init__.py,sha256=qODBuIvWtypQQyKl3Y0QiBXC5SipVFL4b64D91EmTyw,4764
+warp_beacon-2.1.1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+warp_beacon-2.1.1.dist-info/METADATA,sha256=oiDULqyo3Qu0FxBilngJEN9xvhKqHVEir6Xp7Tz7ojs,21250
+warp_beacon-2.1.1.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+warp_beacon-2.1.1.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
+warp_beacon-2.1.1.dist-info/top_level.txt,sha256=SIMha34qjGXaijMgtzESgQ-Cu-KbLHbmQacwue_ggTY,917
+warp_beacon-2.1.1.dist-info/RECORD,,

{warp_beacon-2.0.10.dist-info → warp_beacon-2.1.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (72.2.0)
+Generator: setuptools (75.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{warp_beacon-2.0.10.dist-info → warp_beacon-2.1.1.dist-info}/top_level.txt

@@ -13,8 +13,10 @@ warp_beacon/mediainfo/silencer
 warp_beacon/mediainfo/video
 warp_beacon/scraper
 warp_beacon/scraper/abstract
+warp_beacon/scraper/account_selector
 warp_beacon/scraper/exceptions
 warp_beacon/scraper/instagram
+warp_beacon/scraper/instagram/instagram
 warp_beacon/scraper/types
 warp_beacon/scraper/youtube
 warp_beacon/scraper/youtube/abstract

warp_beacon-2.0.10.dist-info/RECORD

@@ -1,40 +0,0 @@
-etc/warp_beacon/warp_beacon.conf,sha256=-QyHfsyo-1Ukxlsz3KSE2vKvLCRm5NtA36_jmfqSAsI,307
-lib/systemd/system/warp_beacon.service,sha256=lPmHqLqcI2eIV7nwHS0qcALQrznixqJuwwPfa2mDLUA,372
-var/warp_beacon/placeholder.gif,sha256=cE5CGJVaop4Sx21zx6j4AyoHU0ncmvQuS2o6hJfEH88,6064
-warp_beacon/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-warp_beacon/__version__.py,sha256=vUAwymcHllnRpWKcmq_zCPb9Ro_Z8h2NwtbWgfkmsVs,24
-warp_beacon/warp_beacon.py,sha256=7KEtZDj-pdhtl6m-zFLsSojs1ZR4o7L0xbqtdmYPvfE,342
-warp_beacon/compress/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-warp_beacon/compress/video.py,sha256=_PDMVYCyzLYxHv1uZmmzGcG_8rjaZr7BTXsXTTy_oS4,2846
-warp_beacon/jobs/__init__.py,sha256=ED8_tPle4iL4kqNW0apAVkgNQtRRTnYfAJwBjO1g0JY,180
-warp_beacon/jobs/abstract.py,sha256=NCwsogk3-m8F6a-SeiY4CEdTgqJn2V_MU4H7_iYuF7k,2463
-warp_beacon/jobs/download_job.py,sha256=5HiPcnJppFMhO14___3eSkoMygM3y-vhpGkMAuNhK7s,854
-warp_beacon/jobs/types.py,sha256=Ae8zINgbs7cOcYkYoOCOACA7duyhnIGMQAJ_SJB1QRQ,176
-warp_beacon/jobs/upload_job.py,sha256=_ul4psPej1jLEs-BMcMR80GbXDSmm38jE9yoZtecclY,741
-warp_beacon/mediainfo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-warp_beacon/mediainfo/abstract.py,sha256=ZR2JMuRpoh7nDNov9a8YkAfr6BI2HXnXzQtVrLgDxjs,1185
-warp_beacon/mediainfo/audio.py,sha256=ous88kwQj4bDIChN5wnGil5LqTs0IQHH0d-nyrL0-ZM,651
-warp_beacon/mediainfo/silencer.py,sha256=MgUc9Ibbhjhg9GbJMNfJqrdDkMsQShZkQ1sCwvW_-qI,1647
-warp_beacon/mediainfo/video.py,sha256=AIRy_op_BvehsjarM1rvT5Qo0QWwf-Q6xVVd_aCnbJ4,2505
-warp_beacon/scraper/__init__.py,sha256=3pwcnYp4fdGG_zdnKGryfbtKJWHOZXuUOTSiCB3mvdQ,11049
-warp_beacon/scraper/abstract.py,sha256=-yMCEW2JUQWUVs4TbiUi6PEw8RyRuTvrJRIUeEpASP0,1723
-warp_beacon/scraper/exceptions.py,sha256=j3sUi3LT3l-mqW8X6sNSpmx5JIioFODWlOZW8I9RJHA,1191
-warp_beacon/scraper/instagram.py,sha256=SHyyNvy5dIPhKllIBEvUhM6yq2zeOw4V6r1cvP69NwU,8580
-warp_beacon/scraper/youtube/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-warp_beacon/scraper/youtube/abstract.py,sha256=ZbCaYAsGowGsrbJDY39Yk_JAhETncw9aE9GYA9sPXTA,5940
-warp_beacon/scraper/youtube/music.py,sha256=Cf7QNgers2zo-cQRxi__NPz7V8ZoWCC-g9b8yZNfJ4E,1529
-warp_beacon/scraper/youtube/shorts.py,sha256=vMEshUBtFwchwp_W2uUoPKXwgll2wFgKdIfvSFKXxLE,1234
-warp_beacon/scraper/youtube/youtube.py,sha256=TmKMdGbhORTBcBYB6V4ItvWKnxWtUfMbXhomonsmAso,2280
-warp_beacon/storage/__init__.py,sha256=8XsJXq9X7GDlTaWREF4W1PDX9PH5utwhjf5c5M8Bb7o,3378
-warp_beacon/telegram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-warp_beacon/telegram/bot.py,sha256=HGsc1H_CxiE0GBxELqATpSWIjahwjU8FuH0geQOruAI,13086
-warp_beacon/telegram/handlers.py,sha256=MTcHZmWe8RAcZdicnqQewy_SkwujhnaoqJgWHpebfVs,6350
-warp_beacon/telegram/placeholder_message.py,sha256=u5kVfTjGmVYkwA5opniRltHXGpsdSxI41WEde8J5os0,6418
-warp_beacon/telegram/utils.py,sha256=1tm_DH1F2snDxSqwZnKD4ijvTrobv_kscgt3w-bWa6g,2027
-warp_beacon/uploader/__init__.py,sha256=qODBuIvWtypQQyKl3Y0QiBXC5SipVFL4b64D91EmTyw,4764
-warp_beacon-2.0.10.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-warp_beacon-2.0.10.dist-info/METADATA,sha256=nsFNDX6H3CtUbCBnDu5E2Wlec9v4FePl6mSQMqp_544,21242
-warp_beacon-2.0.10.dist-info/WHEEL,sha256=HiCZjzuy6Dw0hdX5R3LCFPDmFS4BWl8H-8W39XfmgX4,91
-warp_beacon-2.0.10.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
-warp_beacon-2.0.10.dist-info/top_level.txt,sha256=pu6xG8OO_nCGllnOfAZ6QpVfivtmHVxPlYK8SZzUDqA,840
-warp_beacon-2.0.10.dist-info/RECORD,,