warp-beacon 2.6.31__tar.gz → 2.6.33__tar.gz

{warp_beacon-2.6.31/warp_beacon.egg-info → warp_beacon-2.6.33}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.6.31
+Version: 2.6.33
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

{warp_beacon-2.6.31 → warp_beacon-2.6.33}/setup.py

@@ -87,6 +87,7 @@ setup(
 		"warp_beacon/scraper/youtube/music",
 		"warp_beacon/scraper/fail_handler",
 		"warp_beacon/scraper/link_resolver",
+		"warp_beacon/scraper/utils",
 		"warp_beacon/storage/mongo"
 	],
 	#scripts=['scripts/wait_dc_update.py'],

warp_beacon-2.6.33/warp_beacon/__version__.py

@@ -0,0 +1,2 @@
+__version__ = "2.6.33"
+

{warp_beacon-2.6.31 → warp_beacon-2.6.33}/warp_beacon/scraper/__init__.py

@@ -3,10 +3,12 @@ import os
 import time
 from typing import Optional
 import multiprocessing
+from multiprocessing.managers import Namespace
 from queue import Empty
 
 import logging
 
+from warp_beacon.scraper.utils import Utils
 from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, Unavailable, FileTooBig, YoutubeLiveError, \
 	YotubeAgeRestrictedError, IGRateLimitOccurred, CaptchaIssue, AllAccountsFailed, BadProxy
 from warp_beacon.mediainfo.video import VideoInfo
@@ -39,12 +41,15 @@ class AsyncDownloader(object):
 	acc_selector = None
 	scheduler = None
 	scrolling_now = None
+	process_context = None
 
 	def __init__(self, uploader: AsyncUploader, workers_count: int) -> None:
 		self.workers = []
 		self.job_queue = multiprocessing.Queue()
 		self.auth_event = multiprocessing.Event()
 		self.manager = multiprocessing.Manager()
+		self.process_context = self.manager.Namespace()
+		self.process_context.ig_session_client_id = Utils.get_ig_session_id()
 		self.allow_loop = self.manager.Value('i', 1)
 		self.scrolling_now = self.manager.Value('i', 0)
 		self.acc_selector = AccountSelector(self.manager, ACC_FILE, PROXY_FILE)
@@ -58,7 +63,7 @@ class AsyncDownloader(object):
 
 	def start(self) -> None:
 		for _ in range(self.workers_count):
-			proc = multiprocessing.Process(target=self.do_work, args=(self.acc_selector,))
+			proc = multiprocessing.Process(target=self.do_work, args=(self.acc_selector, self.process_context))
 			self.workers.append(proc)
 			proc.start()
 
@@ -95,7 +100,7 @@ class AsyncDownloader(object):
 		job.account_switches += 1
 		selector.reset_ig_request_count()
 
-	def do_work(self, selector: AccountSelector) -> None:
+	def do_work(self, selector: AccountSelector, context: Namespace) -> None:
 		logging.info("download worker started")
 		# pymongo is not fork-safe so new connect to DB required
 		fail_handler = FailHandler(DBClient())
@@ -136,7 +141,8 @@ class AsyncDownloader(object):
 								proxy = selector.get_current_proxy()
 							if job.job_origin is Origin.INSTAGRAM:
 								from warp_beacon.scraper.instagram.instagram import InstagramScraper
-								actor = InstagramScraper(selector.get_current(), proxy)
+								Utils.maybe_rotate_ig_client_session(context)
+								actor = InstagramScraper(client_session_id=context.ig_session_client_id, account=selector.get_current(), proxy=proxy)
 								selector.inc_ig_request_count()
 								if not job.scroll_content and selector.get_ig_request_count() >= int(os.environ.get("IG_REQUESTS_PER_ACCOUNT", default="10")):
 									logging.info("The account request limit has been reached. Selecting the next account.")
@@ -429,6 +435,7 @@ class AsyncDownloader(object):
 	def stop_all(self) -> None:
 		self.allow_loop.value = 0
 		self.acc_selector.save_ig_request_count()
+		Utils.save_ig_session_id(self.process_context.ig_session_client_id)
 		for proc in self.workers:
 			if proc.is_alive():
 				logging.info("stopping process #%d", proc.pid)

{warp_beacon-2.6.31 → warp_beacon-2.6.33}/warp_beacon/scraper/account_selector.py

@@ -161,7 +161,11 @@ class AccountSelector(object):
 		module_name = self.get_module_name(module_origin)
 		self.current_module_name = module_name
 		if self.current is None:
-			self.current = self.accounts[self.current_module_name][self.account_index[self.current_module_name].value]
+			idx = self.account_index[self.current_module_name].value
+			self.current = self.accounts[self.current_module_name][idx]
+			if not self.current.get("enabled", True):
+				logging.info("Account '%d' is disabled. Probing next ...", idx)
+				self.next()
 		self.current_proxy = self.get_random_account_proxy()
 
 	def next(self) -> dict:
@@ -171,6 +175,9 @@ class AccountSelector(object):
 			idx = 0
 		self.account_index[self.current_module_name].value = idx
 		self.current = self.accounts[self.current_module_name][idx]
+		if not self.current.get("enabled", True):
+			logging.info("Account '%d' is disabled. Probing next ...", idx)
+			return self.next()
 		logging.info("Selected account index is '%d'", idx)
 		return self.current
 

{warp_beacon-2.6.31 → warp_beacon-2.6.33}/warp_beacon/scraper/instagram/instagram.py

@@ -36,8 +36,10 @@ class InstagramScraper(ScraperAbstract):
 	cl = None
 	inst_session_file = ""
 	timeline_cursor = {}
+	client_session_id = ""
 
-	def __init__(self, account: tuple, proxy: dict=None) -> None:
+	def __init__(self, client_session_id: str, account: tuple, proxy: dict=None) -> None:
+		self.client_session_id = client_session_id
 		super().__init__(account, proxy)
 		#
 		self.inst_session_file = INST_SESSION_FILE_TPL % self.account_index
@@ -53,6 +55,8 @@ class InstagramScraper(ScraperAbstract):
 		self.cl.change_password_handler = self.change_password_handler
 
 	def setup_device(self) -> None:
+		if not self.client_session_id:
+			self.client_session_id = self.cl.generate_uuid()
 		details = self.account.get("auth_details", {})
 		self.cl.delay_range = details.get("delay_range", [1, 3])
 		self.cl.set_country_code(details.get("country_code", 1))
@@ -76,7 +80,7 @@ class InstagramScraper(ScraperAbstract):
 		self.cl.set_uuids({
 			"phone_id": uuids.get("phone_id", self.cl.generate_uuid()),
 			"uuid": uuids.get("uuid", self.cl.generate_uuid()),
-			"client_session_id": uuids.get("client_session_id", self.cl.generate_uuid()),
+			"client_session_id": self.client_session_id,
 			"advertising_id": uuids.get("advertising_id", self.cl.generate_uuid()),
 			"device_id": uuids.get("device_id", self.cl.generate_uuid())
 		})

warp_beacon-2.6.33/warp_beacon/scraper/utils.py

@@ -0,0 +1,40 @@
+import os
+from multiprocessing.managers import Namespace
+import random
+import uuid
+import logging
+
+class Utils(object):
+	session_dir = "/var/warp_beacon"
+
+	@staticmethod
+	def get_ig_session_id() -> str:
+		ig_session_client_id = ""
+		try:
+			sess_file = f"{Utils.session_dir}/ig_session_client_id"
+			if os.path.exists(sess_file):
+				with open(sess_file, 'r', encoding="utf-8") as f:
+					ig_session_client_id = f.read().strip()
+		except Exception as e:
+			logging.warning("Failed to read session ig_session_client_id!")
+			logging.exception(e)
+
+		if not ig_session_client_id:
+			ig_session_client_id = str(uuid.uuid4())
+
+		return ig_session_client_id
+	
+	@staticmethod
+	def save_ig_session_id(ig_session_client_id: str) -> None:
+		try:
+			with open(f"{Utils.session_dir}/ig_session_client_id", "w+", encoding="utf-8") as f:
+				f.write(ig_session_client_id)
+		except Exception as e:
+			logging.warning("Failed to save session ig_session_client_id!")
+			logging.exception(e)
+	
+	@staticmethod
+	def maybe_rotate_ig_client_session(context: Namespace) -> None:
+		if random.random() > 0.95:
+			context.ig_session_client_id = str(uuid.uuid4())
+			logging.info("Rotated client_session_id — simulating app restart")

{warp_beacon-2.6.31 → warp_beacon-2.6.33/warp_beacon.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.6.31
+Version: 2.6.33
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

{warp_beacon-2.6.31 → warp_beacon-2.6.33}/warp_beacon.egg-info/SOURCES.txt

@@ -41,6 +41,7 @@ warp_beacon/scraper/account_selector.py
 warp_beacon/scraper/exceptions.py
 warp_beacon/scraper/fail_handler.py
 warp_beacon/scraper/link_resolver.py
+warp_beacon/scraper/utils.py
 warp_beacon/scraper/instagram/__init__.py
 warp_beacon/scraper/instagram/captcha.py
 warp_beacon/scraper/instagram/instagram.py

{warp_beacon-2.6.31 → warp_beacon-2.6.33}/warp_beacon.egg-info/top_level.txt

@@ -23,6 +23,7 @@ warp_beacon/scraper/instagram
 warp_beacon/scraper/instagram/instagram
 warp_beacon/scraper/link_resolver
 warp_beacon/scraper/types
+warp_beacon/scraper/utils
 warp_beacon/scraper/youtube
 warp_beacon/scraper/youtube/abstract
 warp_beacon/scraper/youtube/music

warp_beacon-2.6.31/warp_beacon/__version__.py

@@ -1,2 +0,0 @@
-__version__ = "2.6.31"
-