warp-beacon 2.6.33__py3-none-any.whl → 2.6.35__py3-none-any.whl

warp_beacon/__version__.py

@@ -1,2 +1,2 @@
-__version__ = "2.6.33"
+__version__ = "2.6.35"
 

warp_beacon/scraper/__init__.py

@@ -8,7 +8,6 @@ from queue import Empty
 
 import logging
 
-from warp_beacon.scraper.utils import Utils
 from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, Unavailable, FileTooBig, YoutubeLiveError, \
 	YotubeAgeRestrictedError, IGRateLimitOccurred, CaptchaIssue, AllAccountsFailed, BadProxy
 from warp_beacon.mediainfo.video import VideoInfo
@@ -49,7 +48,6 @@ class AsyncDownloader(object):
 		self.auth_event = multiprocessing.Event()
 		self.manager = multiprocessing.Manager()
 		self.process_context = self.manager.Namespace()
-		self.process_context.ig_session_client_id = Utils.get_ig_session_id()
 		self.allow_loop = self.manager.Value('i', 1)
 		self.scrolling_now = self.manager.Value('i', 0)
 		self.acc_selector = AccountSelector(self.manager, ACC_FILE, PROXY_FILE)
@@ -100,7 +98,7 @@ class AsyncDownloader(object):
 		job.account_switches += 1
 		selector.reset_ig_request_count()
 
-	def do_work(self, selector: AccountSelector, context: Namespace) -> None:
+	def do_work(self, selector: AccountSelector, _: Namespace) -> None:
 		logging.info("download worker started")
 		# pymongo is not fork-safe so new connect to DB required
 		fail_handler = FailHandler(DBClient())
@@ -141,13 +139,12 @@ class AsyncDownloader(object):
 								proxy = selector.get_current_proxy()
 							if job.job_origin is Origin.INSTAGRAM:
 								from warp_beacon.scraper.instagram.instagram import InstagramScraper
-								Utils.maybe_rotate_ig_client_session(context)
-								actor = InstagramScraper(client_session_id=context.ig_session_client_id, account=selector.get_current(), proxy=proxy)
-								selector.inc_ig_request_count()
 								if not job.scroll_content and selector.get_ig_request_count() >= int(os.environ.get("IG_REQUESTS_PER_ACCOUNT", default="10")):
 									logging.info("The account request limit has been reached. Selecting the next account.")
 									selector.reset_ig_request_count()
 									selector.next()
+								actor = InstagramScraper(client_session_id=selector.get_ig_session_id(), account=selector.get_current(), proxy=proxy)
+								selector.inc_ig_request_count()
 							elif job.job_origin is Origin.YT_SHORTS:
 								from warp_beacon.scraper.youtube.shorts import YoutubeShortsScraper
 								actor = YoutubeShortsScraper(selector.get_current(), proxy)
@@ -434,8 +431,7 @@ class AsyncDownloader(object):
 
 	def stop_all(self) -> None:
 		self.allow_loop.value = 0
-		self.acc_selector.save_ig_request_count()
-		Utils.save_ig_session_id(self.process_context.ig_session_client_id)
+		self.acc_selector.save_state()
 		for proc in self.workers:
 			if proc.is_alive():
 				logging.info("stopping process #%d", proc.pid)

warp_beacon/scraper/account_selector.py

@@ -3,29 +3,35 @@ import time
 import random
 import json
 import re
+import uuid
 from typing import Optional, List
-
 from itertools import cycle
+
+import logging
+
 import multiprocessing
 import multiprocessing.managers
 
 from warp_beacon.jobs import Origin
 
-import logging
-
 class AccountSelector(object):
-	accounts = []
-	proxies = []
+	accounts = None
+	proxies = None
 	current = None
 	current_module_name = None
 	accounts_meta_data = None
 	session_dir = "/var/warp_beacon"
 	manager = None
-	account_index = {}
+	account_index = None
 	current_proxy = None
 	ig_request_count = None
+	ig_accounts_session_id = None
 
 	def __init__(self, manager: multiprocessing.managers.SyncManager, acc_file_path: str, proxy_file_path: str=None) -> None:
+		self.accounts = []
+		self.proxies = []
+		self.account_index = {}
+		self.ig_accounts_session_id = self.load_ig_sessions_id()
 		self.manager = manager
 		self.accounts_meta_data = self.manager.dict()
 		if os.path.exists(acc_file_path):
@@ -45,6 +51,31 @@ class AccountSelector(object):
 		else:
 			raise ValueError("Accounts file not found")
 
+	def save_state(self) -> None:
+		self.save_ig_request_count()
+		self.save_ig_sessions_id()
+
+	def save_ig_sessions_id(self) -> None:
+		try:
+			with open(f"{self.session_dir}/ig_session_client_id", "w+", encoding="utf-8") as f:
+				f.write(json.dumps(self.ig_accounts_session_id))
+		except Exception as e:
+			logging.warning("Failed to save session ig_session_client_id!")
+			logging.exception(e)
+
+	def load_ig_sessions_id(self) -> dict:
+		ig_sessions_client_id = {}
+		try:
+			sess_file = f"{self.session_dir}/ig_sessions_client_id.json"
+			if os.path.exists(sess_file):
+				with open(sess_file, 'r', encoding="utf-8") as f:
+					ig_sessions_client_id = json.loads(f.read())
+		except Exception as e:
+			logging.warning("Failed to read session ig_session_client_id!")
+			logging.exception(e)
+
+		return ig_sessions_client_id
+
 	def save_ig_request_count(self) -> None:
 		try:
 			state = {"ig_count": self.ig_request_count.value}
@@ -209,7 +240,7 @@ class AccountSelector(object):
 		return (idx, self.accounts[module_name][idx])
 
 	def get_meta_data(self) -> dict:
-		idx = self.account_index[self.current_module_name].value - 1
+		idx = self.account_index[self.current_module_name].value# - 1
 		return self.accounts_meta_data[self.current_module_name][idx]
 
 	def count_service_accounts(self, mod_name: Origin) -> int:
@@ -225,4 +256,14 @@ class AccountSelector(object):
 		self.ig_request_count.value = 0
 
 	def get_ig_request_count(self) -> int:
-		return self.ig_request_count.value
+		return self.ig_request_count.value
+	
+	def get_ig_session_id(self) -> str:
+		idx = self.account_index[self.current_module_name].value
+		if idx not in self.ig_accounts_session_id:
+			self.ig_accounts_session_id[idx] = str(uuid.uuid4())
+		else:
+			if random.random() > 0.95:
+				self.ig_accounts_session_id[idx] = str(uuid.uuid4())
+				logging.info("Rotated client_session_id — simulating app restart")
+		return self.ig_accounts_session_id[idx]

warp_beacon/scraper/utils.py

@@ -1,40 +1,4 @@
-import os
-from multiprocessing.managers import Namespace
-import random
-import uuid
-import logging
+#import logging
 
 class Utils(object):
-	session_dir = "/var/warp_beacon"
-
-	@staticmethod
-	def get_ig_session_id() -> str:
-		ig_session_client_id = ""
-		try:
-			sess_file = f"{Utils.session_dir}/ig_session_client_id"
-			if os.path.exists(sess_file):
-				with open(sess_file, 'r', encoding="utf-8") as f:
-					ig_session_client_id = f.read().strip()
-		except Exception as e:
-			logging.warning("Failed to read session ig_session_client_id!")
-			logging.exception(e)
-
-		if not ig_session_client_id:
-			ig_session_client_id = str(uuid.uuid4())
-
-		return ig_session_client_id
-	
-	@staticmethod
-	def save_ig_session_id(ig_session_client_id: str) -> None:
-		try:
-			with open(f"{Utils.session_dir}/ig_session_client_id", "w+", encoding="utf-8") as f:
-				f.write(ig_session_client_id)
-		except Exception as e:
-			logging.warning("Failed to save session ig_session_client_id!")
-			logging.exception(e)
-	
-	@staticmethod
-	def maybe_rotate_ig_client_session(context: Namespace) -> None:
-		if random.random() > 0.95:
-			context.ig_session_client_id = str(uuid.uuid4())
-			logging.info("Rotated client_session_id — simulating app restart")
+	pass

{warp_beacon-2.6.33.dist-info → warp_beacon-2.6.35.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.6.33
+Version: 2.6.35
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

{warp_beacon-2.6.33.dist-info → warp_beacon-2.6.35.dist-info}/RECORD

@@ -4,7 +4,7 @@ var/warp_beacon/accounts.json,sha256=OsXdncs6h88xrF_AP6_WDCK1waGBn9SR-uYdIeK37GM
 var/warp_beacon/placeholder.gif,sha256=cE5CGJVaop4Sx21zx6j4AyoHU0ncmvQuS2o6hJfEH88,6064
 var/warp_beacon/proxies.json,sha256=VnjlQDXumOEq72ZFjbh6IqHS1TEHqn8HPYAZqWCeSIA,95
 warp_beacon/__init__.py,sha256=_rThNODmz0nDp_n4mWo_HKaNFE5jk1_7cRhHyYaencI,163
-warp_beacon/__version__.py,sha256=EhSBAgTsvMy3ocyZX2oGpI8vIL6KfXQjHozyx-Obe9Y,24
+warp_beacon/__version__.py,sha256=GBPjTCfxnfCRE053qGQV0boaaA5UUJxjblOZowA_xx0,24
 warp_beacon/warp_beacon.py,sha256=ED43vNzdjDUJ_9qLCbri0bjWLWEJ69BENGj9i7G6AvM,342
 warp_beacon/yt_auth.py,sha256=GUTKqYr_tzDC-07Lx_ahWXSag8EyLxXBUnQbDBIkEmk,6022
 warp_beacon/compress/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -22,13 +22,13 @@ warp_beacon/mediainfo/video.py,sha256=UBZrhTN5IDI-aYu6tsJEILo9nFkjHhkldGVFmvV7tE
 warp_beacon/scheduler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scheduler/instagram_human.py,sha256=0LaRUu0MBBuEOQeFzuq22HIYfJL9pTK_7udsXfef0Fk,8204
 warp_beacon/scheduler/scheduler.py,sha256=9OCh7Ta4wY_aTHGAOOZmaKXg56Ftx1N_aV1g6E3ZLKA,4941
-warp_beacon/scraper/__init__.py,sha256=O8L8KIKt7Xszql0tUPCNMiaxyZduMZjfADAP4V5s9bk,20286
+warp_beacon/scraper/__init__.py,sha256=AaUd30PSJi5CCqP_UQ25Hbz6W7Y0CEyza1eV4ji4xeU,20029
 warp_beacon/scraper/abstract.py,sha256=yP76ae9U73LYW2lDN6XWkXDkWX1h2UVOrkqv0Sqpu2Y,2985
-warp_beacon/scraper/account_selector.py,sha256=7sog5tFPNv6jWJ5m-7nAglAyzd3Qc3G4dfjk8iYFONI,8163
+warp_beacon/scraper/account_selector.py,sha256=dcM2rFAUICr-LYs26HFmXPZsn0yxHplH_mEZU8oNKDk,9570
 warp_beacon/scraper/exceptions.py,sha256=EKwoF0oH2xZWbNU-v8DOaWK5skKwa3s1yTIBdlcfMpc,1452
 warp_beacon/scraper/fail_handler.py,sha256=zcPK3ZVEsu6JmHYcWP7L3naTRK3gWFVRkpP84VBOtJs,964
 warp_beacon/scraper/link_resolver.py,sha256=Rc9ZuMyOo3iPywDHwjngy-WRQ2SXhJwxcg-5ripx7tM,2447
-warp_beacon/scraper/utils.py,sha256=BI0X6Wu2pZ8hKaiOAYooBKjhqE-tl3FfgOTAmzgMp6M,1208
+warp_beacon/scraper/utils.py,sha256=xJfCVhiLjVPoFVupE10sTzX7UpgNWbx2EeYIsoYXGDk,43
 warp_beacon/scraper/instagram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scraper/instagram/captcha.py,sha256=9UYziuqB3Tsat_ET6ex-cnZDbi6yCnsXHSpmE8MuUHk,4651
 warp_beacon/scraper/instagram/instagram.py,sha256=zbkF-1lU5lxJ_2m8i8mGRX0EQMLVSflsOafBM1FEC6s,15588
@@ -46,9 +46,9 @@ warp_beacon/telegram/handlers.py,sha256=uvR6TPHSqdSxigp3wR-ewiE6t3TvVcbVLVcYGwkg
 warp_beacon/telegram/placeholder_message.py,sha256=wN9-BRiyrtHG-EvXtZkGJHt2CX71munQ57ITttjt0mw,6400
 warp_beacon/telegram/utils.py,sha256=1Lq67aRylVJzbwSyvAgjPAGjJZFATkICvAj3TJGuJiM,4635
 warp_beacon/uploader/__init__.py,sha256=j3qcuKhpchseZLGzSsSiogqe6WdMbkK8d3I-ConhNRs,5687
-warp_beacon-2.6.33.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-warp_beacon-2.6.33.dist-info/METADATA,sha256=txwoubgNiyIrmstWZBebz0CvW8QIJY_lcuQNh1AYeAo,22706
-warp_beacon-2.6.33.dist-info/WHEEL,sha256=GHB6lJx2juba1wDgXDNlMTyM13ckjBMKf-OnwgKOCtA,91
-warp_beacon-2.6.33.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
-warp_beacon-2.6.33.dist-info/top_level.txt,sha256=aFsWDQBplsMOyVMGGJ8iu-auZ25z1e_IB4tM2M8kW1A,1187
-warp_beacon-2.6.33.dist-info/RECORD,,
+warp_beacon-2.6.35.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+warp_beacon-2.6.35.dist-info/METADATA,sha256=j1lOtEDstP9jLnR2PmVPO4eTu6kzFEAoqDtdBUFkQFA,22706
+warp_beacon-2.6.35.dist-info/WHEEL,sha256=GHB6lJx2juba1wDgXDNlMTyM13ckjBMKf-OnwgKOCtA,91
+warp_beacon-2.6.35.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
+warp_beacon-2.6.35.dist-info/top_level.txt,sha256=aFsWDQBplsMOyVMGGJ8iu-auZ25z1e_IB4tM2M8kW1A,1187
+warp_beacon-2.6.35.dist-info/RECORD,,