warp-beacon 2.6.87__py3-none-any.whl → 2.6.88__py3-none-any.whl

warp_beacon/__version__.py

@@ -1,2 +1,2 @@
-__version__ = "2.6.87"
+__version__ = "2.6.88"
 

warp_beacon/scheduler/instagram_human.py

@@ -17,7 +17,9 @@ class InstagramHuman(object):
 		self.operations_count = 0
 
 	def watch_content(self, media: list) -> None:
-		for m in media[:random.randint(1, 15)]:
+		if not media:
+			return
+		for m in media[:random.randint(1, len(media))]:
 			try:
 				logging.info("Wathing content with pk '%s'", str(m.pk))
 				content = self.scrapler.cl.media_info_v1(m.pk)
@@ -30,7 +32,7 @@ class InstagramHuman(object):
 
 	def scroll_content(self, last_pk: int) -> None:
 		timeline_initialized = False
-		if random.random() > 0.7:
+		if random.random() > 0.5:
 			timeline_initialized = True
 			self.scrapler.timeline_cursor = self.scrapler.download_hndlr(self.scrapler.cl.get_timeline_feed, reason="cold_start_fetch")
 			logging.info("Starting to watch related reels with media_pk '%d'", last_pk)
@@ -38,7 +40,7 @@ class InstagramHuman(object):
 			self.operations_count += 1
 			self.watch_content(media)
 		
-		if random.random() > 0.9:
+		if random.random() > 0.7:
 			time.sleep(random.uniform(2, 20))
 			if not timeline_initialized:
 				self.scrapler.timeline_cursor = self.scrapler.download_hndlr(self.scrapler.cl.get_timeline_feed, reason="cold_start_fetch")
@@ -75,7 +77,7 @@ class InstagramHuman(object):
 				self.scrapler.download_hndlr(self.scrapler.cl.notification_like_and_comment_on_photo_user_tagged, "everyone")
 				self.operations_count += 1
 				self.random_pause()
-			if random.random() > 0.5:
+			if random.random() > 0.2:
 				logging.info("Simulation updating reels tray feed ...")
 				self.scrapler.download_hndlr(self.scrapler.cl.get_reels_tray_feed, "pull_to_refresh")
 				self.operations_count += 1
@@ -98,7 +100,7 @@ class InstagramHuman(object):
 				self.random_pause()
 			if random.random() > 0.4:
 				logging.info("Watching reels ...")
-				reels = self.scrapler.download_hndlr(self.scrapler.cl.reels)
+				reels = self.scrapler.download_hndlr(self.scrapler.cl.reels, amount=random.randint(4, 15))
 				self.operations_count += 1
 				self.watch_content(reels)
 				self.random_pause()
@@ -126,11 +128,11 @@ class InstagramHuman(object):
 				self.random_pause()
 			if random.random() > 0.4:
 				logging.info("Watching reels ...")
-				reels = self.scrapler.download_hndlr(self.scrapler.cl.reels)
+				reels = self.scrapler.download_hndlr(self.scrapler.cl.reels, amount=random.randint(4, 15))
 				self.operations_count += 1
 				self.watch_content(reels)
 				self.random_pause()
-			if random.random() > 0.6:
+			if random.random() > 0.4:
 				logging.info("Simulation profile view ...")
 				self.profile_view()
 				self.random_pause()
@@ -145,9 +147,9 @@ class InstagramHuman(object):
 				self.scrapler.download_hndlr(self.scrapler.cl.direct_active_presence)
 				self.operations_count += 1
 				self.random_pause(short=True)
-			if random.random() > 0.8:
+			if random.random() > 0.5:
 				logging.info("Watching reels ...")
-				reels = self.scrapler.download_hndlr(self.scrapler.cl.reels)
+				reels = self.scrapler.download_hndlr(self.scrapler.cl.reels, amount=random.randint(4, 15))
 				self.operations_count += 1
 				self.watch_content(reels)
 				self.random_pause()
@@ -179,14 +181,14 @@ class InstagramHuman(object):
 				#self.scrapler.download_hndlr(self.scrapler.cl.user_info, target_user_id)
 				self.scrapler.download_hndlr(self.scrapler.cl.user_info_v1, target_user_id)
 				self.operations_count += 1
-				time.sleep(random.uniform(2, 5))
+				self.random_pause()
 			elif isinstance(random_friend, str):
 				target_user_id = self.scrapler.download_hndlr(self.scrapler.cl.user_id_from_username, random_friend)
 				logging.info("user_info with target_user_id = '%s' ...", target_user_id)
 				#self.scrapler.download_hndlr(self.scrapler.cl.user_info, target_user_id)
 				self.scrapler.download_hndlr(self.scrapler.cl.user_info_v1, target_user_id)
 				self.operations_count += 1
-				time.sleep(random.uniform(2, 5))
+				self.random_pause()
 			
 			if random.random() > 0.5:
 				logging.info("Checking direct ...")
@@ -202,7 +204,7 @@ class InstagramHuman(object):
 			
 			if random.random() > 0.5:
 				logging.info("user_medias with target_user_id = '%s' ...", target_user_id)
-				self.scrapler.download_hndlr(self.scrapler.cl.user_medias_v1, target_user_id, amount=random.randint(1, 5))
+				self.scrapler.download_hndlr(self.scrapler.cl.user_medias_v1, target_user_id, amount=random.randint(1, 10))
 				self.operations_count += 1
 				self.random_pause()
 		except Exception as e:

warp_beacon/scheduler/scheduler.py

@@ -126,7 +126,7 @@ class IGScheduler(object):
 
 	def handle_time_planning(self) -> None:
 		if int(self.state.get("remaining", 0)) <= 0:
-			self.state["remaining"] = randrange(9292, 26200)
+			self.state["remaining"] = randrange(5292, 12253)
 
 	def do_work(self) -> None:
 		logging.info("Scheduler thread started ...")

warp_beacon/scraper/__init__.py

@@ -130,7 +130,7 @@ class AsyncDownloader(object):
 								proxy = selector.get_current_proxy()
 							if job.job_origin is Origin.INSTAGRAM:
 								from warp_beacon.scraper.instagram.instagram import InstagramScraper
-								if not job.scroll_content and selector.get_ig_request_count() >= int(os.environ.get("IG_REQUESTS_PER_ACCOUNT", default="10")):
+								if not job.scroll_content and selector.get_ig_request_count() >= int(os.environ.get("IG_REQUESTS_PER_ACCOUNT", default="20")):
 									logging.info("The account request limit has been reached. Selecting the next account.")
 									selector.reset_ig_request_count()
 									selector.next()
@@ -164,7 +164,7 @@ class AsyncDownloader(object):
 										break
 									if job.session_validation and job.job_origin in (Origin.INSTAGRAM, Origin.YOUTUBE):
 										if job.job_origin is Origin.INSTAGRAM:
-											if selector.get_ig_request_count() >= int(os.environ.get("IG_REQUESTS_PER_ACCOUNT", default="10")):
+											if selector.get_ig_request_count() >= int(os.environ.get("IG_REQUESTS_PER_ACCOUNT", default="20")):
 												logging.info("The account request limit has been reached. Selecting the next account.")
 												selector.reset_ig_request_count()
 												selector.next()

warp_beacon/scraper/utils.py

@@ -4,10 +4,10 @@ from fake_useragent import UserAgent
 
 class ScraperUtils(object):
 	@staticmethod
-	def get_ua_dict() -> dict:
+	def get_ua_dict(browsers: list = ['Facebook', 'Android'], platforms: list = ['mobile', 'tablet'], os: list =['Android', 'iOS']) -> dict:
 		random_client = None
 		try:
-			ua = UserAgent(browsers=['Facebook', 'Android'], platforms=['mobile', 'tablet'], os=['Android', 'iOS'])
+			ua = UserAgent(browsers=browsers, platforms=platforms, os=os)
 			random_client = ua.getRandom
 			logging.info("Select random UA: %s", random_client)
 		except Exception as e:
@@ -16,5 +16,5 @@ class ScraperUtils(object):
 		return random_client
 
 	@staticmethod
-	def get_ua() -> str:
-		return ScraperUtils.get_ua_dict()["useragent"]
+	def get_ua(browsers: list = ['Facebook', 'Android'], platforms: list = ['mobile', 'tablet'], os: list =['Android', 'iOS']) -> str:
+		return ScraperUtils.get_ua_dict(browsers=browsers, platforms=platforms, os=os)["useragent"]

warp_beacon/scraper/youtube/abstract.py

@@ -29,6 +29,7 @@ from warp_beacon.scraper.abstract import ScraperAbstract
 from warp_beacon.scraper.exceptions import (BadProxy, TimeOut, Unavailable,
 											extract_exception_message)
 from warp_beacon.yt_auth import YtAuth
+from warp_beacon.scraper.utils import ScraperUtils
 
 class YoutubeAbstract(ScraperAbstract):
 	DOWNLOAD_DIR = "/tmp"
@@ -226,7 +227,10 @@ class YoutubeAbstract(ScraperAbstract):
 		return ret_val
 
 	def yt_on_progress(self, stream: Stream, chunk: bytes, bytes_remaining: int) -> None:
-		total_size = stream.filesize or stream.filesize_approx
+		total_size = int(stream.filesize or stream.filesize_approx or 0)
+		if not total_size:
+			logging.warning("[Download worker]: total_size is '%d'", total_size)
+			return
 		bytes_downloaded = total_size - bytes_remaining
 		percentage_of_completion = bytes_downloaded / (total_size or 1) * 100
 		if total_size == 0 or percentage_of_completion >= self._download_progress_threshold:
@@ -286,6 +290,28 @@ class YoutubeAbstract(ScraperAbstract):
 				yt_opts["proxies"] = self.build_proxies(proxy_dsn)
 		return YouTube(**yt_opts)
 	
+	def yt_dlp_on_progress(self, params: dict) -> None:
+		if params.get("status", "") == "downloading":
+			total_size = int(params.get("total_bytes") or params.get("total_bytes_estimate") or 0)
+			if not total_size or total_size < 0:
+				logging.warning("[Download worker][yt_dlp]: total_size is '%d'", total_size)
+				return
+			bytes_downloaded = int(params.get("downloaded_bytes", 0))
+			percentage_of_completion = bytes_downloaded / (total_size or 1) * 100
+			if total_size == 0 or percentage_of_completion >= self._download_progress_threshold:
+				msg = {
+					"action": "report_download_status",
+					"current": bytes_downloaded,
+					"total": total_size,
+					"message_id": self.job.placeholder_message_id,
+					"chat_id": self.job.chat_id,
+					"completed": percentage_of_completion >= 100
+				}
+				self.status_pipe.send(msg)
+				logging.debug("[Download worker][yt_dlp] Downloaded %d%%", percentage_of_completion)
+				if total_size > 0:
+					self._download_progress_threshold += 20
+
 	def build_yt_dlp(self, timeout: int = 60) -> yt_dlp.YoutubeDL:
 		auth_data = {}
 		with open(self.YT_SESSION_FILE % self.account_index, 'r', encoding="utf-8") as f:
@@ -297,9 +323,24 @@ class YoutubeAbstract(ScraperAbstract):
 			'format': 'bestvideo+bestaudio/best',
 			'merge_output_format': 'mp4',
 			'noplaylist': True,
-			'tv_auth': auth_data
+			'progress_hooks': [self.yt_dlp_on_progress],
+			'http_headers': {
+				"Accept-Language": "en-US,en;q=0.9",
+				'User-Agent': ScraperUtils.get_ua(
+					browsers=["Google", "Chrome", "Firefox"],
+					os=["Windows", "Linux", "Ubuntu", "Chrome OS", "Mac OS X"],
+					platforms=["desktop"]
+				)
+			}
 		}
 
+		if auth_data and auth_data.get("access_token", None):
+			ydl_opts["http_headers"]["Authorization"] = f'Bearer {auth_data["access_token"]}'
+
+		yt_dlp_cookies_file = os.environ.get("YT_DLP_COOKIES_FILE", default="/var/warp_beacon/yt_dlp_cookies.txt")
+		if yt_dlp_cookies_file and os.path.exists(yt_dlp_cookies_file):
+			ydl_opts['cookiefile'] = yt_dlp_cookies_file
+
 		if self.proxy:
 			proxy_dsn = self.proxy.get("dsn", "")
 			logging.info("Using proxy DSN '%s'", proxy_dsn)

{warp_beacon-2.6.87.dist-info → warp_beacon-2.6.88.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.6.87
+Version: 2.6.88
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

{warp_beacon-2.6.87.dist-info → warp_beacon-2.6.88.dist-info}/RECORD

@@ -4,7 +4,7 @@ var/warp_beacon/accounts.json,sha256=OsXdncs6h88xrF_AP6_WDCK1waGBn9SR-uYdIeK37GM
 var/warp_beacon/placeholder.gif,sha256=cE5CGJVaop4Sx21zx6j4AyoHU0ncmvQuS2o6hJfEH88,6064
 var/warp_beacon/proxies.json,sha256=VnjlQDXumOEq72ZFjbh6IqHS1TEHqn8HPYAZqWCeSIA,95
 warp_beacon/__init__.py,sha256=_rThNODmz0nDp_n4mWo_HKaNFE5jk1_7cRhHyYaencI,163
-warp_beacon/__version__.py,sha256=FgjYXoAFmDQi-ZsqKjg6shGTSafYxf_8wotLCLckBXA,24
+warp_beacon/__version__.py,sha256=LBpQqsZ-TT6c9n00IAHECMU-8pg2jKcbCReuRuJTohg,24
 warp_beacon/warp_beacon.py,sha256=ED43vNzdjDUJ_9qLCbri0bjWLWEJ69BENGj9i7G6AvM,342
 warp_beacon/yt_auth.py,sha256=GUTKqYr_tzDC-07Lx_ahWXSag8EyLxXBUnQbDBIkEmk,6022
 warp_beacon/compress/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -20,21 +20,21 @@ warp_beacon/mediainfo/audio.py,sha256=ous88kwQj4bDIChN5wnGil5LqTs0IQHH0d-nyrL0-Z
 warp_beacon/mediainfo/silencer.py,sha256=qxMuViOoVwUYb60uCVvqHiGrqByR1_4_rqMT-XdMkwc,1813
 warp_beacon/mediainfo/video.py,sha256=UBZrhTN5IDI-aYu6tsJEILo9nFkjHhkldGVFmvV7tEI,2480
 warp_beacon/scheduler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-warp_beacon/scheduler/instagram_human.py,sha256=4Cik8bX9QYb3rtJ51matR78xxg8mXpaO9d5tGYVsIlE,8208
-warp_beacon/scheduler/scheduler.py,sha256=cr3oKANu_iOw4vCLzr8Mh0HdZIqYx3I8i4ohfoQkw2U,4995
-warp_beacon/scraper/__init__.py,sha256=2-h_HGb7Gp5H2-qyfONaubTOv1lzR4SWfjLV7iX9cP0,20103
+warp_beacon/scheduler/instagram_human.py,sha256=JF5K8jgdwW0ncpKsEAj2yWMP9Y40kKgCM_i3HoHwNnM,8307
+warp_beacon/scheduler/scheduler.py,sha256=Bf4sGXjX75Dox3q-yzUHhagtzUAj3hl5GzfnZya-_io,4995
+warp_beacon/scraper/__init__.py,sha256=a-nnFAPvXtmJFQJOEbN2UL92FllK_7NTPM3Yoz_hYjk,20103
 warp_beacon/scraper/abstract.py,sha256=_Cr_MaSQ56Wp6n00PcqiM2WDa5DlRKXUnQ9C15bPObw,3692
 warp_beacon/scraper/account_selector.py,sha256=VmXH8SHZqQ6a2mrt2agzDWepE3v46QrFLL3OqVwJ2A4,9731
 warp_beacon/scraper/exceptions.py,sha256=EKwoF0oH2xZWbNU-v8DOaWK5skKwa3s1yTIBdlcfMpc,1452
 warp_beacon/scraper/fail_handler.py,sha256=zcPK3ZVEsu6JmHYcWP7L3naTRK3gWFVRkpP84VBOtJs,964
 warp_beacon/scraper/link_resolver.py,sha256=Rc9ZuMyOo3iPywDHwjngy-WRQ2SXhJwxcg-5ripx7tM,2447
-warp_beacon/scraper/utils.py,sha256=fEqKFlWkZTyThU1rokvMEHIVdc8bk70kCWo5LPvnZYE,1045
+warp_beacon/scraper/utils.py,sha256=Kk_lDmdJiCSaHmOV80OFK05O1wRL2H0agH98JiO8dyg,1268
 warp_beacon/scraper/instagram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scraper/instagram/captcha.py,sha256=9UYziuqB3Tsat_ET6ex-cnZDbi6yCnsXHSpmE8MuUHk,4651
 warp_beacon/scraper/instagram/instagram.py,sha256=64XxGui7bPuiJT-Hqlg1KQT9GsbX1YMXzEUJdzvkxLQ,17141
 warp_beacon/scraper/instagram/wb_instagrapi.py,sha256=OECHz1eK6J5bfw2H84CdywwxBNegkNb_UhDr0MtNtZ4,5237
 warp_beacon/scraper/youtube/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-warp_beacon/scraper/youtube/abstract.py,sha256=WLbQKtrKSwpZUQlpjL2XI7z8XxoRCtVxrRjSXFJmXuM,13230
+warp_beacon/scraper/youtube/abstract.py,sha256=0ub4O8kNBzcrYCbMaNoNQzwE5vBdpNcSYNtxT-cBXe4,15024
 warp_beacon/scraper/youtube/music.py,sha256=5AeSBQyUgVCJT2hoBCV2WvlyuV9US09SYJhmBG_P9F8,2755
 warp_beacon/scraper/youtube/shorts.py,sha256=1GtoYUlxAwcgSQcn80u5ehNJytH5AN5dPOicmX-XD8E,1705
 warp_beacon/scraper/youtube/youtube.py,sha256=x9v9p1coA9TvBhxjNAofGu4UBkAEdYPE2ePRnU-5tK0,7233
@@ -52,9 +52,9 @@ warp_beacon/telegram/progress_file_reader.py,sha256=e3equyNKlKs764AD-iE9QRsh3YDH
 warp_beacon/telegram/types.py,sha256=Kvdng6uCF1HRoqQgGW1ZYYPJoVuYkFb-LDvMBbW5Hjk,89
 warp_beacon/telegram/utils.py,sha256=1Lq67aRylVJzbwSyvAgjPAGjJZFATkICvAj3TJGuJiM,4635
 warp_beacon/uploader/__init__.py,sha256=j3qcuKhpchseZLGzSsSiogqe6WdMbkK8d3I-ConhNRs,5687
-warp_beacon-2.6.87.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-warp_beacon-2.6.87.dist-info/METADATA,sha256=d99T_Qpshl5vd8q5Ikhz44db0tH5b8Lpe4fLaxoTR4o,22736
-warp_beacon-2.6.87.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
-warp_beacon-2.6.87.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
-warp_beacon-2.6.87.dist-info/top_level.txt,sha256=5YQRN46STNg81V_3jdzZ6bftkMxhe1hTPSFvJugDu84,1405
-warp_beacon-2.6.87.dist-info/RECORD,,
+warp_beacon-2.6.88.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+warp_beacon-2.6.88.dist-info/METADATA,sha256=pdq3qKn6nykzvnMC11AcwO_KgOcGOglLGEsTGVXKmV4,22736
+warp_beacon-2.6.88.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
+warp_beacon-2.6.88.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
+warp_beacon-2.6.88.dist-info/top_level.txt,sha256=5YQRN46STNg81V_3jdzZ6bftkMxhe1hTPSFvJugDu84,1405
+warp_beacon-2.6.88.dist-info/RECORD,,