warp-beacon 2.6.84__tar.gz → 2.6.86__tar.gz

{warp_beacon-2.6.84/warp_beacon.egg-info → warp_beacon-2.6.86}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.6.84
+Version: 2.6.86
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

warp_beacon-2.6.86/warp_beacon/__version__.py

@@ -0,0 +1,2 @@
+__version__ = "2.6.86"
+

{warp_beacon-2.6.84 → warp_beacon-2.6.86}/warp_beacon/scheduler/instagram_human.py

@@ -17,7 +17,7 @@ class InstagramHuman(object):
 		self.operations_count = 0
 
 	def watch_content(self, media: list) -> None:
-		for m in media[:random.randint(1, 3)]:
+		for m in media[:random.randint(1, 15)]:
 			try:
 				logging.info("Wathing content with pk '%s'", str(m.pk))
 				content = self.scrapler.cl.media_info_v1(m.pk)
@@ -34,7 +34,7 @@ class InstagramHuman(object):
 			timeline_initialized = True
 			self.scrapler.timeline_cursor = self.scrapler.download_hndlr(self.scrapler.cl.get_timeline_feed, reason="cold_start_fetch")
 			logging.info("Starting to watch related reels with media_pk '%d'", last_pk)
-			media = self.scrapler.download_hndlr(self.scrapler.cl.reels, amount=random.randint(4, 10), last_media_pk=last_pk)
+			media = self.scrapler.download_hndlr(self.scrapler.cl.reels, amount=random.randint(4, 15), last_media_pk=last_pk)
 			self.operations_count += 1
 			self.watch_content(media)
 		
@@ -43,7 +43,7 @@ class InstagramHuman(object):
 			if not timeline_initialized:
 				self.scrapler.timeline_cursor = self.scrapler.download_hndlr(self.scrapler.cl.get_timeline_feed, reason="cold_start_fetch")
 			logging.info("Starting to explore reels with media_pk '%d'", last_pk)
-			media = self.scrapler.download_hndlr(self.scrapler.cl.explore_reels, amount=random.randint(4, 10), last_media_pk=last_pk)
+			media = self.scrapler.download_hndlr(self.scrapler.cl.explore_reels, amount=random.randint(4, 15), last_media_pk=last_pk)
 			self.operations_count += 1
 			self.watch_content(media)
 

{warp_beacon-2.6.84 → warp_beacon-2.6.86}/warp_beacon/scheduler/scheduler.py

@@ -15,10 +15,8 @@ class IGScheduler(object):
 	yt_sessions_dir = "/var/warp_beacon"
 
 	def __init__(self, downloader: warp_beacon.scraper.AsyncDownloader) -> None:
-		self.downloader = None
 		self.running = True
 		self.thread = None
-		self.event = None
 		self.state = {"remaining": randrange(8400, 26200), "yt_sess_exp": []}
 		self.downloader = downloader
 		self.event = threading.Event()
@@ -108,6 +106,9 @@ class IGScheduler(object):
 	
 	def validate_yt_session(self) -> bool:
 		try:
+			if self.downloader.yt_validate_event.is_set():
+				return True
+			self.downloader.yt_validate_event.set()
 			logging.info("Setting YT validate task ...")
 			self.downloader.queue_task(warp_beacon.jobs.download_job.DownloadJob.build(
 				session_validation=True,
@@ -147,7 +148,7 @@ class IGScheduler(object):
 					self.handle_time_planning()
 
 				start_time = time.time()
-				logging.info("Next scheduler activity in '%s' seconds", int(min_val))
+				logging.info("Next scheduler activity in '%d' seconds", int(min_val))
 				logging.info("IG timeout '%d' secs", int(self.state["remaining"]))
 				self.event.wait(timeout=min_val)
 				self.event.clear()

{warp_beacon-2.6.84 → warp_beacon-2.6.86}/warp_beacon/scraper/__init__.py

@@ -33,18 +33,6 @@ PROXY_FILE = os.environ.get("PROXY_FILE", default="/var/warp_beacon/proxies.json
 class AsyncDownloader(object):
 	TG_FILE_LIMIT = 2147483648 # 2 GiB
 	__JOE_BIDEN_WAKEUP = None
-	workers = None
-	allow_loop = None
-	job_queue = None
-	uploader = None
-	workers_count = 0
-	auth_event = None
-	manager = None
-	acc_selector = None
-	scheduler = None
-	scrolling_now = None
-	process_context = None
-	status_pipe = None
 
 	def __init__(self, uploader: AsyncUploader, pipe_connection: multiprocessing.connection.Connection, workers_count: int) -> None:
 		self.workers = []
@@ -58,6 +46,7 @@ class AsyncDownloader(object):
 		self.uploader = uploader
 		self.workers_count = workers_count
 		self.status_pipe = pipe_connection
+		self.yt_validate_event = multiprocessing.Event()
 		if os.environ.get("TG_PREMIUM", default="false") == "true":
 			self.TG_FILE_LIMIT = 4294967296 # 4 GiB
 
@@ -160,6 +149,7 @@ class AsyncDownloader(object):
 							actor.request_yt_auth = self.request_yt_auth
 							actor.auth_event = self.auth_event
 							actor.status_pipe = self.status_pipe
+							actor.yt_validate_event = self.yt_validate_event
 							# job retry loop
 							while self.allow_loop.value == 1:
 								try:

{warp_beacon-2.6.84 → warp_beacon-2.6.86}/warp_beacon/scraper/abstract.py

@@ -1,23 +1,28 @@
-import os
-import time
-import pathlib
-from abc import ABC, abstractmethod
-from typing import Callable, Union
 import logging
 import multiprocessing
 import multiprocessing.connection
+import os
 import socket
-import requests.packages.urllib3.util.connection as urllib3_cn
+import pathlib
+import time
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Callable, Union
 
 from PIL import Image
 from pillow_heif import register_heif_opener
 
+import requests.packages.urllib3.util.connection as urllib3_cn
+
+if TYPE_CHECKING:
+	from multiprocessing.synchronize import Event as EventType
+
 class ScraperAbstract(ABC):
 	def __init__(self, account: tuple, proxy: dict = None) -> None:
 		self.original_gai_family = None
 		self.send_message_to_admin_func: Callable = lambda: None
 		self.request_yt_auth: Callable = lambda: None
 		self.status_pipe: multiprocessing.connection.Connection = None
+		self.yt_validate_event: EventType = None
 		self.auth_event = None
 		self.account = None
 		self.account_index = 0

{warp_beacon-2.6.84 → warp_beacon-2.6.86}/warp_beacon/scraper/instagram/instagram.py

@@ -12,9 +12,9 @@ import logging
 import email
 import imaplib
 import json
+from urllib.parse import urljoin, urlparse
 import requests
 import urllib3
-from urllib.parse import urljoin, urlparse
 
 from instagrapi import exceptions
 from instagrapi.exceptions import UnknownError as IGUnknownError
@@ -341,7 +341,7 @@ class InstagramScraper(ScraperAbstract):
 					#})
 					media_info = self.download_hndlr(self.cl.media_info_v1, media_id)
 					logging.info("media_type is '%d', product_type is '%s'", media_info.media_type, media_info.product_type)
-					if media_info.media_type == 2 and media_info.product_type == "clips": # Reels
+					if media_info.media_type == 2 and media_info.product_type in ("clips", "ad"): # Reels
 						res.append(self.download_video(url=media_info.video_url, media_info=media_info))
 					elif media_info.media_type == 1: # Photo
 						res.append(self.download_photo(url=media_info.thumbnail_url, media_info=media_info))

{warp_beacon-2.6.84 → warp_beacon-2.6.86}/warp_beacon/scraper/youtube/abstract.py

@@ -63,6 +63,10 @@ class YoutubeAbstract(ScraperAbstract):
 			logging.error("Failed to refresh Youtube session!")
 			logging.exception(e)
 
+		# avoid task acquiring in parallel worker
+		self.yt_validate_event.clear()
+		#time.sleep(35)
+
 		return 0
 	
 	def get_video_id(self, url: str) -> Optional[str]:

{warp_beacon-2.6.84 → warp_beacon-2.6.86/warp_beacon.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.6.84
+Version: 2.6.86
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

warp_beacon-2.6.84/warp_beacon/__version__.py

@@ -1,2 +0,0 @@
-__version__ = "2.6.84"
-