warp-beacon 2.3.51__py3-none-any.whl → 2.4.1__py3-none-any.whl

warp_beacon/__version__.py

@@ -1,2 +1,2 @@
-__version__ = "2.3.51"
+__version__ = "2.4.1"
 

warp_beacon/scraper/__init__.py

@@ -121,6 +121,11 @@ class AsyncDownloader(object):
 							if job.job_origin is Origin.INSTAGRAM:
 								from warp_beacon.scraper.instagram.instagram import InstagramScraper
 								actor = InstagramScraper(selector.get_current(), proxy)
+								selector.inc_ig_request_count()
+								if selector.get_ig_request_count() >= int(os.environ.get("IG_REQUESTS_PER_ACCOUNT", default="20")):
+									logging.info("The account request limit has been reached. Selecting the next account.")
+									selector.reset_ig_request_count()
+									selector.next()
 							elif job.job_origin is Origin.YT_SHORTS:
 								from warp_beacon.scraper.youtube.shorts import YoutubeShortsScraper
 								actor = YoutubeShortsScraper(selector.get_current(), proxy)

warp_beacon/scraper/account_selector.py

@@ -23,6 +23,7 @@ class AccountSelector(object):
 	manager = None
 	account_index = {}
 	current_proxy = None
+	ig_request_count = None
 
 	def __init__(self, manager: multiprocessing.managers.SyncManager, acc_file_path: str, proxy_file_path: str=None) -> None:
 		self.manager = manager
@@ -31,6 +32,7 @@ class AccountSelector(object):
 			with open(acc_file_path, 'r', encoding="utf-8") as f:
 				self.accounts = json.loads(f.read())
 			if self.accounts:
+				self.ig_request_count = self.manager.Value('i', 0)
 				self.__init_meta_data()
 				#self.load_yt_sessions()
 				for acc_type, _ in self.accounts.items():
@@ -170,4 +172,13 @@ class AccountSelector(object):
 		module_name = 'youtube' if next((s for s in ("yt", "youtube", "youtu_be") if s in mod_name.value), None) else 'instagram'
 		if module_name not in self.accounts:
 			return 0
-		return len(self.accounts[module_name])
+		return len(self.accounts[module_name])
+	
+	def inc_ig_request_count(self) -> None:
+		self.ig_request_count.value += 1
+
+	def reset_ig_request_count(self) -> None:
+		self.ig_request_count.value = 0
+
+	def get_ig_request_count(self) -> int:
+		return self.ig_request_count.value

warp_beacon/yt_auth.py

@@ -0,0 +1,197 @@
+import os
+import time
+
+import logging
+
+import json
+import requests
+
+class YtAuth(object):
+	TV_CLIENT_ID = "861556708454-d6dlm3lh05idd8npek18k6be8ba3oc68.apps.googleusercontent.com"
+	TV_CLIENT_SECRET = "SboVhoG9s0rNafixCSGGKXAT"
+	YT_SESSION_FILE_TPL = '/var/warp_beacon/yt_session_%d.json'
+
+	process_start_time = 0
+	account_index = 0
+	yt_session_file = ""
+
+	def __init__(self, account_index: int) -> None:
+		self.account_index = account_index
+		self.yt_session_file = self.YT_SESSION_FILE_TPL % account_index
+
+	def fetch_token(self) -> dict:
+		result = {"user_code": "", "device_code": "", "verification_url": ""}
+		http_code = 0
+		response_text = ''
+		try:
+			logging.info("Fetching YT token ...")
+			self.process_start_time = 0
+			# Subtracting 30 seconds is arbitrary to avoid potential time discrepencies
+			self.process_start_time = int(time.time() - 30)
+			data = {
+				'client_id': self.TV_CLIENT_ID,
+				'scope': 'https://www.googleapis.com/auth/youtube'
+			}
+			response = requests.post(
+				url='https://oauth2.googleapis.com/device/code',
+				headers={
+					"User-Agent": "Mozilla/5.0",
+					"accept-language": "en-US,en",
+					"Content-Type": "application/json"
+				},
+				json=data,
+				timeout=int(os.environ.get("YT_TIMEOUT", "30"))
+			)
+
+			http_code = response.status_code
+			response_text = response.text
+
+			if http_code != 200:
+				logging.error("Invalid YT HTTP code: '%d'", http_code)
+				logging.info("Request dump: '%s'", str(response.__dict__))
+			else:
+				response_data = response.json()
+				result["verification_url"] = response_data['verification_url']
+				result["user_code"] = response_data['user_code']
+				result["device_code"] = response_data['device_code']
+				logging.info("Fetched YT url '%s' and input code '%s'", result["verification_url"], result['user_code'])
+		except Exception as e:
+			logging.error("Youtube authorization failed!")
+			logging.exception(e)
+
+		if http_code != 200:
+			raise ValueError(f"Youtube HTTP response code is {http_code}: {response_text}")
+
+		return result
+
+	def confirm_token(self, device_code: str) -> dict:
+		response_data = {}
+		http_code = 0
+		response_text = ''
+		try:
+			logging.info("Confirming YT auth token ...")
+			self.process_start_time = int(time.time()) - self.process_start_time - 20
+			data = {
+				'client_id': self.TV_CLIENT_ID,
+				'client_secret': self.TV_CLIENT_SECRET,
+				'device_code': device_code,
+				'grant_type': 'urn:ietf:params:oauth:grant-type:device_code'
+			}
+			response = requests.post(
+				url='https://oauth2.googleapis.com/token',
+				headers={
+					"User-Agent": "Mozilla/5.0",
+					"accept-language": "en-US,en",
+					"Content-Type": "application/json"
+				},
+				json=data,
+				timeout=int(os.environ.get("YT_TIMEOUT", "30"))
+			)
+
+			http_code = response.status_code
+			response_text = response.text
+
+			if http_code != 200:
+				logging.error("Invalid YT HTTP code: '%d'", http_code)
+				logging.info("Request dump: '%s'", str(response.__dict__))
+			else:
+				response_data = response.json()
+				response_data["expires"] = self.process_start_time + int(response_data["expires_in"])
+		except Exception as e:
+			logging.error("Failed to confirm token!")
+			logging.exception(e)
+
+		if http_code != 200:
+			raise ValueError(f"Youtube HTTP response code is {http_code}: {response_text}")
+
+		return response_data
+
+	def refresh_token(self, refresh_token: str) -> dict:
+		response_data = {}
+		http_code = 0
+		response_text = ''
+		try:
+			logging.info("Refreshing YT token ...")
+			start_time = int(time.time() - 30)
+			data = {
+				'client_id': self.TV_CLIENT_ID,
+				'client_secret': self.TV_CLIENT_SECRET,
+				'grant_type': 'refresh_token',
+				'refresh_token': refresh_token
+			}
+			response = requests.post(
+				url='https://oauth2.googleapis.com/token',
+				headers={
+					"User-Agent": "Mozilla/5.0",
+					"accept-language": "en-US,en",
+					"Content-Type": "application/json"
+				},
+				json=data,
+				timeout=int(os.environ.get("YT_TIMEOUT", "30"))
+			)
+
+			http_code = response.status_code
+			response_text = response.text
+
+			if http_code != 200:
+				logging.error("Invalid YT HTTP code: '%d'", http_code)
+				logging.info("Request dump: '%s'", str(response.__dict__))
+			else:
+				response_data = response.json()
+				response_data["expires"] = start_time + int(response_data["expires_in"])
+		except Exception as e:
+			logging.error("Failed to refresh YT token")
+			logging.exception(e)
+
+		if http_code != 200:
+			raise ValueError(f"Youtube HTTP response code is {http_code}: {response_text}")
+		
+		return response_data
+
+	def safe_write_session(self, token_data: dict) -> bool:
+		try:
+			tmp_filename = f"{self.yt_session_file}~"
+			
+			if os.path.exists(tmp_filename):
+				os.unlink(tmp_filename)
+			
+			with open(tmp_filename, "w+", encoding="utf-8") as f:
+				f.write(json.dumps(token_data))
+			
+			if os.path.exists(tmp_filename):
+				if os.path.exists(self.yt_session_file):
+					os.unlink(self.yt_session_file)
+				os.rename(src=tmp_filename, dst=self.yt_session_file)
+			return True
+		except Exception as e:
+			logging.error("Failed to write token!")
+			logging.exception(e)
+
+		return False
+	
+	def store_device_code(self, device_code: str) -> bool:
+		try:
+			device_code_file = f"/tmp/yt_device_code_acc_{self.account_index}"
+			logging.info("Storing device code in file '%s'", device_code_file)
+			with open(device_code_file, "w+", encoding="utf-8") as f:
+				f.write(device_code.strip())
+		except Exception as e:
+			logging.error("Failed to store device code!")
+			logging.exception(e)
+			return False
+
+		return True
+	
+	def load_device_code(self) -> str:
+		device_code = ''
+		try:
+			device_code_file = f"/tmp/yt_device_code_acc_{self.account_index}"
+			logging.info("Loading device code from file '%s'", device_code_file)
+			with open(device_code_file, 'r', encoding="utf-8") as f:
+				device_code = f.read().strip()
+			os.unlink(device_code_file)
+		except Exception as e:
+			logging.error("Failed to load device code for account #%d", self.account_index)
+			logging.exception(e)
+
+		return device_code

{warp_beacon-2.3.51.dist-info → warp_beacon-2.4.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.3.51
+Version: 2.4.1
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

{warp_beacon-2.3.51.dist-info → warp_beacon-2.4.1.dist-info}/RECORD

@@ -4,8 +4,9 @@ var/warp_beacon/accounts.json,sha256=OsXdncs6h88xrF_AP6_WDCK1waGBn9SR-uYdIeK37GM
 var/warp_beacon/placeholder.gif,sha256=cE5CGJVaop4Sx21zx6j4AyoHU0ncmvQuS2o6hJfEH88,6064
 var/warp_beacon/proxies.json,sha256=VnjlQDXumOEq72ZFjbh6IqHS1TEHqn8HPYAZqWCeSIA,95
 warp_beacon/__init__.py,sha256=_rThNODmz0nDp_n4mWo_HKaNFE5jk1_7cRhHyYaencI,163
-warp_beacon/__version__.py,sha256=qoauBZVJH1GNZozTtQhg39aKWblw1Jo_CuPkyrihYSg,24
+warp_beacon/__version__.py,sha256=N3GPwYcqNMEhFj_mvMJ1B1wGX85Yq81y2ePqg2BYiRs,23
 warp_beacon/warp_beacon.py,sha256=7KEtZDj-pdhtl6m-zFLsSojs1ZR4o7L0xbqtdmYPvfE,342
+warp_beacon/yt_auth.py,sha256=GUTKqYr_tzDC-07Lx_ahWXSag8EyLxXBUnQbDBIkEmk,6022
 warp_beacon/compress/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/compress/video.py,sha256=_PDMVYCyzLYxHv1uZmmzGcG_8rjaZr7BTXsXTTy_oS4,2846
 warp_beacon/jobs/__init__.py,sha256=ED8_tPle4iL4kqNW0apAVkgNQtRRTnYfAJwBjO1g0JY,180
@@ -21,9 +22,9 @@ warp_beacon/mediainfo/video.py,sha256=UBZrhTN5IDI-aYu6tsJEILo9nFkjHhkldGVFmvV7tE
 warp_beacon/scheduler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scheduler/instagram_human.py,sha256=GRzryMWws4ibJZ_gPGn_Y2I2plcPIVgMWK3Yubqywa0,5020
 warp_beacon/scheduler/scheduler.py,sha256=FPSrR1Z6lWbF1aCAMU2rRYCy1w4zdZ8Q4e7YWQE-Ahk,4041
-warp_beacon/scraper/__init__.py,sha256=bNZI9YBKQB4UyxEjethenXjvp9g1RrWfYgOyepH4O_U,17356
+warp_beacon/scraper/__init__.py,sha256=iqWDK2vzKXkd9RPGYXxQ4O-IrydOBBFR89aX50sP2Yg,17669
 warp_beacon/scraper/abstract.py,sha256=28a0aBKZpi8IKptLWdB6RuVbOkrUbrhT7LSZX7QRQtg,2725
-warp_beacon/scraper/account_selector.py,sha256=HSfmi5H73aWYTjDa8GErCCQpTwWtDLBw92Q-I8TZilg,6282
+warp_beacon/scraper/account_selector.py,sha256=TzAh4fQigsRcS2AomAMvlms8sH5RKKbPT20LJemWGtU,6596
 warp_beacon/scraper/exceptions.py,sha256=L4UCZqDOLKu5-TPUcTErIbNDLrAQJghkMaPqzTfsm1g,1376
 warp_beacon/scraper/fail_handler.py,sha256=_blvckfTZ4xWVancQKVRXH5ClKGwfrBxMwvXIFZh1qA,975
 warp_beacon/scraper/link_resolver.py,sha256=Rc9ZuMyOo3iPywDHwjngy-WRQ2SXhJwxcg-5ripx7tM,2447
@@ -43,9 +44,9 @@ warp_beacon/telegram/handlers.py,sha256=XXIfdV_RCj7tyZMPXchuKmGoDdweOaR08ADDaBPW
 warp_beacon/telegram/placeholder_message.py,sha256=wN9-BRiyrtHG-EvXtZkGJHt2CX71munQ57ITttjt0mw,6400
 warp_beacon/telegram/utils.py,sha256=9uebX53G16mV7ER7WgfdWBLFHHw14S8HBt9URrIskg0,4440
 warp_beacon/uploader/__init__.py,sha256=E9rlZIf7xlQz6MutMOwJ8S5Vm2uheR5nv23Kv8duRQg,5427
-warp_beacon-2.3.51.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-warp_beacon-2.3.51.dist-info/METADATA,sha256=CMn79Q5lJ-opcZwsBPfWIHf3-Zll4vWJAXERZq518t8,22626
-warp_beacon-2.3.51.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-warp_beacon-2.3.51.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
-warp_beacon-2.3.51.dist-info/top_level.txt,sha256=qGjHVVfyf6lTmbdSA-fQq0rHS1YVS4HoJT3rag5xgPE,1141
-warp_beacon-2.3.51.dist-info/RECORD,,
+warp_beacon-2.4.1.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+warp_beacon-2.4.1.dist-info/METADATA,sha256=79HmXMfW3Jw74KnY1NtVVJYU2pgvQTkcD6MVPF48IhE,22625
+warp_beacon-2.4.1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+warp_beacon-2.4.1.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
+warp_beacon-2.4.1.dist-info/top_level.txt,sha256=qGjHVVfyf6lTmbdSA-fQq0rHS1YVS4HoJT3rag5xgPE,1141
+warp_beacon-2.4.1.dist-info/RECORD,,