warp-beacon 2.6.85__py3-none-any.whl → 2.6.87__py3-none-any.whl

warp_beacon/__version__.py

@@ -1,2 +1,2 @@
-__version__ = "2.6.85"
+__version__ = "2.6.87"
 

warp_beacon/scheduler/instagram_human.py

@@ -17,7 +17,7 @@ class InstagramHuman(object):
 		self.operations_count = 0
 
 	def watch_content(self, media: list) -> None:
-		for m in media[:random.randint(1, 3)]:
+		for m in media[:random.randint(1, 15)]:
 			try:
 				logging.info("Wathing content with pk '%s'", str(m.pk))
 				content = self.scrapler.cl.media_info_v1(m.pk)
@@ -34,7 +34,7 @@ class InstagramHuman(object):
 			timeline_initialized = True
 			self.scrapler.timeline_cursor = self.scrapler.download_hndlr(self.scrapler.cl.get_timeline_feed, reason="cold_start_fetch")
 			logging.info("Starting to watch related reels with media_pk '%d'", last_pk)
-			media = self.scrapler.download_hndlr(self.scrapler.cl.reels, amount=random.randint(4, 10), last_media_pk=last_pk)
+			media = self.scrapler.download_hndlr(self.scrapler.cl.reels, amount=random.randint(4, 15), last_media_pk=last_pk)
 			self.operations_count += 1
 			self.watch_content(media)
 		
@@ -43,7 +43,7 @@ class InstagramHuman(object):
 			if not timeline_initialized:
 				self.scrapler.timeline_cursor = self.scrapler.download_hndlr(self.scrapler.cl.get_timeline_feed, reason="cold_start_fetch")
 			logging.info("Starting to explore reels with media_pk '%d'", last_pk)
-			media = self.scrapler.download_hndlr(self.scrapler.cl.explore_reels, amount=random.randint(4, 10), last_media_pk=last_pk)
+			media = self.scrapler.download_hndlr(self.scrapler.cl.explore_reels, amount=random.randint(4, 15), last_media_pk=last_pk)
 			self.operations_count += 1
 			self.watch_content(media)
 

warp_beacon/scheduler/scheduler.py

@@ -15,10 +15,8 @@ class IGScheduler(object):
 	yt_sessions_dir = "/var/warp_beacon"
 
 	def __init__(self, downloader: warp_beacon.scraper.AsyncDownloader) -> None:
-		self.downloader = None
 		self.running = True
 		self.thread = None
-		self.event = None
 		self.state = {"remaining": randrange(8400, 26200), "yt_sess_exp": []}
 		self.downloader = downloader
 		self.event = threading.Event()
@@ -108,6 +106,9 @@ class IGScheduler(object):
 	
 	def validate_yt_session(self) -> bool:
 		try:
+			if self.downloader.yt_validate_event.is_set():
+				return True
+			self.downloader.yt_validate_event.set()
 			logging.info("Setting YT validate task ...")
 			self.downloader.queue_task(warp_beacon.jobs.download_job.DownloadJob.build(
 				session_validation=True,
@@ -147,7 +148,7 @@ class IGScheduler(object):
 					self.handle_time_planning()
 
 				start_time = time.time()
-				logging.info("Next scheduler activity in '%s' seconds", int(min_val))
+				logging.info("Next scheduler activity in '%d' seconds", int(min_val))
 				logging.info("IG timeout '%d' secs", int(self.state["remaining"]))
 				self.event.wait(timeout=min_val)
 				self.event.clear()

warp_beacon/scraper/__init__.py

@@ -33,18 +33,6 @@ PROXY_FILE = os.environ.get("PROXY_FILE", default="/var/warp_beacon/proxies.json
 class AsyncDownloader(object):
 	TG_FILE_LIMIT = 2147483648 # 2 GiB
 	__JOE_BIDEN_WAKEUP = None
-	workers = None
-	allow_loop = None
-	job_queue = None
-	uploader = None
-	workers_count = 0
-	auth_event = None
-	manager = None
-	acc_selector = None
-	scheduler = None
-	scrolling_now = None
-	process_context = None
-	status_pipe = None
 
 	def __init__(self, uploader: AsyncUploader, pipe_connection: multiprocessing.connection.Connection, workers_count: int) -> None:
 		self.workers = []
@@ -58,6 +46,7 @@ class AsyncDownloader(object):
 		self.uploader = uploader
 		self.workers_count = workers_count
 		self.status_pipe = pipe_connection
+		self.yt_validate_event = multiprocessing.Event()
 		if os.environ.get("TG_PREMIUM", default="false") == "true":
 			self.TG_FILE_LIMIT = 4294967296 # 4 GiB
 
@@ -160,6 +149,7 @@ class AsyncDownloader(object):
 							actor.request_yt_auth = self.request_yt_auth
 							actor.auth_event = self.auth_event
 							actor.status_pipe = self.status_pipe
+							actor.yt_validate_event = self.yt_validate_event
 							# job retry loop
 							while self.allow_loop.value == 1:
 								try:

warp_beacon/scraper/abstract.py

@@ -1,23 +1,28 @@
-import os
-import time
-import pathlib
-from abc import ABC, abstractmethod
-from typing import Callable, Union
 import logging
 import multiprocessing
 import multiprocessing.connection
+import os
 import socket
-import requests.packages.urllib3.util.connection as urllib3_cn
+import pathlib
+import time
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Callable, Union
 
 from PIL import Image
 from pillow_heif import register_heif_opener
 
+import requests.packages.urllib3.util.connection as urllib3_cn
+
+if TYPE_CHECKING:
+	from multiprocessing.synchronize import Event as EventType
+
 class ScraperAbstract(ABC):
 	def __init__(self, account: tuple, proxy: dict = None) -> None:
 		self.original_gai_family = None
 		self.send_message_to_admin_func: Callable = lambda: None
 		self.request_yt_auth: Callable = lambda: None
 		self.status_pipe: multiprocessing.connection.Connection = None
+		self.yt_validate_event: EventType = None
 		self.auth_event = None
 		self.account = None
 		self.account_index = 0

warp_beacon/scraper/instagram/instagram.py

@@ -12,9 +12,9 @@ import logging
 import email
 import imaplib
 import json
+from urllib.parse import urljoin, urlparse
 import requests
 import urllib3
-from urllib.parse import urljoin, urlparse
 
 from instagrapi import exceptions
 from instagrapi.exceptions import UnknownError as IGUnknownError
@@ -31,6 +31,7 @@ from warp_beacon.jobs.download_job import DownloadJob
 from warp_beacon.telegram.utils import Utils
 from warp_beacon.scraper.instagram.wb_instagrapi import WBClient
 from warp_beacon.telegram.types import ReportType
+from warp_beacon.scraper.utils import ScraperUtils
 
 INST_SESSION_FILE_TPL = "/var/warp_beacon/inst_session_account_%d.json"
 
@@ -61,10 +62,7 @@ class InstagramScraper(ScraperAbstract):
 			"Accept": "*/*",
 			"Accept-Encoding": "gzip, deflate, br",
 			"Accept-Language": "en-US,en;q=0.9",
-			"User-Agent": (
-				"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
-				"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
-			)
+			"User-Agent": ScraperUtils.get_ua()
 		})
 		self.cl.set_progress_callback(self.download_progress)
 
@@ -96,7 +94,7 @@ class InstagramScraper(ScraperAbstract):
 			"uuid": uuids.get("uuid", self.cl.generate_uuid()),
 			"client_session_id": self.client_session_id,
 			"advertising_id": uuids.get("advertising_id", self.cl.generate_uuid()),
-			"device_id": uuids.get("device_id", self.cl.generate_uuid())
+			"device_id": uuids.get("device_id", self.cl.generate_android_device_id())
 		})
 
 	def safe_write_session(self) -> None:

warp_beacon/scraper/instagram/wb_instagrapi.py

@@ -7,6 +7,8 @@ import requests
 from instagrapi import Client
 from instagrapi.exceptions import VideoNotDownload
 
+from warp_beacon.scraper.utils import ScraperUtils
+
 class WBClient(Client):
 	"""
 	patched instagrapi
@@ -17,10 +19,7 @@ class WBClient(Client):
 		self.session = requests.Session()
 		# may be I should remove '"Sec-Fetch-*", "Upgrade-Insecure-Requests", "DNT"' ?
 		self.session.headers.update({
-			"User-Agent": (
-				"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
-				"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
-			),
+			"User-Agent": ScraperUtils.get_ua(),
 			"Accept": (
 				"text/html,application/xhtml+xml,application/xml;"
 				"q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8"
@@ -29,12 +28,12 @@ class WBClient(Client):
 			"Accept-Encoding": "gzip, deflate, br",
 			"Referer": "https://www.instagram.com/",
 			"Connection": "keep-alive",
-			"Sec-Fetch-Site": "same-origin",
-			"Sec-Fetch-Mode": "navigate",
-			"Sec-Fetch-User": "?1",
-			"Sec-Fetch-Dest": "document",
-			"Upgrade-Insecure-Requests": "1",
-			"DNT": "1",
+			#"Sec-Fetch-Site": "same-origin",
+			#"Sec-Fetch-Mode": "navigate",
+			#"Sec-Fetch-User": "?1",
+			#"Sec-Fetch-Dest": "document",
+			#"Upgrade-Insecure-Requests": "1",
+			#"DNT": "1",
 		})
 		self.essential_params = {"oe", "oh", "_nc_ht", "_nc_cat", "_nc_oc", "_nc_ohc", "_nc_gid"}
 

warp_beacon/scraper/utils.py

@@ -1,4 +1,20 @@
-#import logging
+import logging
 
-class Utils(object):
-	pass
+from fake_useragent import UserAgent
+
+class ScraperUtils(object):
+	@staticmethod
+	def get_ua_dict() -> dict:
+		random_client = None
+		try:
+			ua = UserAgent(browsers=['Facebook', 'Android'], platforms=['mobile', 'tablet'], os=['Android', 'iOS'])
+			random_client = ua.getRandom
+			logging.info("Select random UA: %s", random_client)
+		except Exception as e:
+			logging.warning("Exception occurrd while generating random client UA!", exc_info=e)
+			random_client = {'useragent': 'Mozilla/5.0 (Linux; Android 14; SM-S911B Build/UP1A.231005.007; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/133.0.6943.117 Mobile Safari/537.36 [FB_IAB/FB4A;FBAV/502.0.0.66.79;IABMV/1;]', 'percent': 0.017937771404345798, 'type': 'mobile', 'device_brand': 'Samsung', 'browser': 'Facebook', 'browser_version': '502.0.0', 'browser_version_major_minor': 502.0, 'os': 'Android', 'os_version': '14', 'platform': 'Linux aarch64'}
+		return random_client
+
+	@staticmethod
+	def get_ua() -> str:
+		return ScraperUtils.get_ua_dict()["useragent"]

warp_beacon/scraper/youtube/abstract.py

@@ -63,6 +63,10 @@ class YoutubeAbstract(ScraperAbstract):
 			logging.error("Failed to refresh Youtube session!")
 			logging.exception(e)
 
+		# avoid task acquiring in parallel worker
+		self.yt_validate_event.clear()
+		#time.sleep(35)
+
 		return 0
 	
 	def get_video_id(self, url: str) -> Optional[str]:

{warp_beacon-2.6.85.dist-info → warp_beacon-2.6.87.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.6.85
+Version: 2.6.87
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev
@@ -241,6 +241,7 @@ Requires-Dist: yt_dlp
 Requires-Dist: pydub
 Requires-Dist: SpeechRecognition
 Requires-Dist: playwright
+Requires-Dist: fake-useragent
 Dynamic: author
 Dynamic: home-page
 Dynamic: license-file

{warp_beacon-2.6.85.dist-info → warp_beacon-2.6.87.dist-info}/RECORD

@@ -4,7 +4,7 @@ var/warp_beacon/accounts.json,sha256=OsXdncs6h88xrF_AP6_WDCK1waGBn9SR-uYdIeK37GM
 var/warp_beacon/placeholder.gif,sha256=cE5CGJVaop4Sx21zx6j4AyoHU0ncmvQuS2o6hJfEH88,6064
 var/warp_beacon/proxies.json,sha256=VnjlQDXumOEq72ZFjbh6IqHS1TEHqn8HPYAZqWCeSIA,95
 warp_beacon/__init__.py,sha256=_rThNODmz0nDp_n4mWo_HKaNFE5jk1_7cRhHyYaencI,163
-warp_beacon/__version__.py,sha256=SOOh9zn6VImWIcDqQ_G6RuKAQKjuNYLH_vqVb-KgrWI,24
+warp_beacon/__version__.py,sha256=FgjYXoAFmDQi-ZsqKjg6shGTSafYxf_8wotLCLckBXA,24
 warp_beacon/warp_beacon.py,sha256=ED43vNzdjDUJ_9qLCbri0bjWLWEJ69BENGj9i7G6AvM,342
 warp_beacon/yt_auth.py,sha256=GUTKqYr_tzDC-07Lx_ahWXSag8EyLxXBUnQbDBIkEmk,6022
 warp_beacon/compress/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -20,21 +20,21 @@ warp_beacon/mediainfo/audio.py,sha256=ous88kwQj4bDIChN5wnGil5LqTs0IQHH0d-nyrL0-Z
 warp_beacon/mediainfo/silencer.py,sha256=qxMuViOoVwUYb60uCVvqHiGrqByR1_4_rqMT-XdMkwc,1813
 warp_beacon/mediainfo/video.py,sha256=UBZrhTN5IDI-aYu6tsJEILo9nFkjHhkldGVFmvV7tEI,2480
 warp_beacon/scheduler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-warp_beacon/scheduler/instagram_human.py,sha256=Li8R8Fm9sZCXCJnS4RrQnQJTsPlBPENQowRe_23Bu-Y,8207
-warp_beacon/scheduler/scheduler.py,sha256=C-19tIb6iiHfboKNPDjg9tzhJQ3mNEbHywW9foe7nKw,4931
-warp_beacon/scraper/__init__.py,sha256=buXmGcJnwCi0qGZAh3srcSbww0XvKyTACp_DW11e86Q,20225
-warp_beacon/scraper/abstract.py,sha256=hRyBbdZ5HK7QyyXlbFAgk3FZe2SIil0WBZ9vJziZv_s,3554
+warp_beacon/scheduler/instagram_human.py,sha256=4Cik8bX9QYb3rtJ51matR78xxg8mXpaO9d5tGYVsIlE,8208
+warp_beacon/scheduler/scheduler.py,sha256=cr3oKANu_iOw4vCLzr8Mh0HdZIqYx3I8i4ohfoQkw2U,4995
+warp_beacon/scraper/__init__.py,sha256=2-h_HGb7Gp5H2-qyfONaubTOv1lzR4SWfjLV7iX9cP0,20103
+warp_beacon/scraper/abstract.py,sha256=_Cr_MaSQ56Wp6n00PcqiM2WDa5DlRKXUnQ9C15bPObw,3692
 warp_beacon/scraper/account_selector.py,sha256=VmXH8SHZqQ6a2mrt2agzDWepE3v46QrFLL3OqVwJ2A4,9731
 warp_beacon/scraper/exceptions.py,sha256=EKwoF0oH2xZWbNU-v8DOaWK5skKwa3s1yTIBdlcfMpc,1452
 warp_beacon/scraper/fail_handler.py,sha256=zcPK3ZVEsu6JmHYcWP7L3naTRK3gWFVRkpP84VBOtJs,964
 warp_beacon/scraper/link_resolver.py,sha256=Rc9ZuMyOo3iPywDHwjngy-WRQ2SXhJwxcg-5ripx7tM,2447
-warp_beacon/scraper/utils.py,sha256=xJfCVhiLjVPoFVupE10sTzX7UpgNWbx2EeYIsoYXGDk,43
+warp_beacon/scraper/utils.py,sha256=fEqKFlWkZTyThU1rokvMEHIVdc8bk70kCWo5LPvnZYE,1045
 warp_beacon/scraper/instagram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scraper/instagram/captcha.py,sha256=9UYziuqB3Tsat_ET6ex-cnZDbi6yCnsXHSpmE8MuUHk,4651
-warp_beacon/scraper/instagram/instagram.py,sha256=RMEr_Tn-awB9Lzv8jTEmExxQ6dJ-D9wfkXKpUUWx2IE,17187
-warp_beacon/scraper/instagram/wb_instagrapi.py,sha256=Dh0SlCZcrgLC4IsIqKUV3xszQXKlrdsll0nCAxoiXHI,5289
+warp_beacon/scraper/instagram/instagram.py,sha256=64XxGui7bPuiJT-Hqlg1KQT9GsbX1YMXzEUJdzvkxLQ,17141
+warp_beacon/scraper/instagram/wb_instagrapi.py,sha256=OECHz1eK6J5bfw2H84CdywwxBNegkNb_UhDr0MtNtZ4,5237
 warp_beacon/scraper/youtube/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-warp_beacon/scraper/youtube/abstract.py,sha256=K2ovycq1lkx0a2R1FddO3B0IcaVsVzni_fdXzajGdwg,13134
+warp_beacon/scraper/youtube/abstract.py,sha256=WLbQKtrKSwpZUQlpjL2XI7z8XxoRCtVxrRjSXFJmXuM,13230
 warp_beacon/scraper/youtube/music.py,sha256=5AeSBQyUgVCJT2hoBCV2WvlyuV9US09SYJhmBG_P9F8,2755
 warp_beacon/scraper/youtube/shorts.py,sha256=1GtoYUlxAwcgSQcn80u5ehNJytH5AN5dPOicmX-XD8E,1705
 warp_beacon/scraper/youtube/youtube.py,sha256=x9v9p1coA9TvBhxjNAofGu4UBkAEdYPE2ePRnU-5tK0,7233
@@ -52,9 +52,9 @@ warp_beacon/telegram/progress_file_reader.py,sha256=e3equyNKlKs764AD-iE9QRsh3YDH
 warp_beacon/telegram/types.py,sha256=Kvdng6uCF1HRoqQgGW1ZYYPJoVuYkFb-LDvMBbW5Hjk,89
 warp_beacon/telegram/utils.py,sha256=1Lq67aRylVJzbwSyvAgjPAGjJZFATkICvAj3TJGuJiM,4635
 warp_beacon/uploader/__init__.py,sha256=j3qcuKhpchseZLGzSsSiogqe6WdMbkK8d3I-ConhNRs,5687
-warp_beacon-2.6.85.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-warp_beacon-2.6.85.dist-info/METADATA,sha256=560e5jLxBdbimadJ7ZBtoxxZkqdVtfd_xcvv4yxl7nM,22706
-warp_beacon-2.6.85.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
-warp_beacon-2.6.85.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
-warp_beacon-2.6.85.dist-info/top_level.txt,sha256=5YQRN46STNg81V_3jdzZ6bftkMxhe1hTPSFvJugDu84,1405
-warp_beacon-2.6.85.dist-info/RECORD,,
+warp_beacon-2.6.87.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+warp_beacon-2.6.87.dist-info/METADATA,sha256=d99T_Qpshl5vd8q5Ikhz44db0tH5b8Lpe4fLaxoTR4o,22736
+warp_beacon-2.6.87.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
+warp_beacon-2.6.87.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
+warp_beacon-2.6.87.dist-info/top_level.txt,sha256=5YQRN46STNg81V_3jdzZ6bftkMxhe1hTPSFvJugDu84,1405
+warp_beacon-2.6.87.dist-info/RECORD,,

{warp_beacon-2.6.85.dist-info → warp_beacon-2.6.87.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.3.1)
+Generator: setuptools (80.4.0)
 Root-Is-Purelib: true
 Tag: py3-none-any