warp-beacon 2.8.2__tar.gz → 2.8.4__tar.gz

{warp_beacon-2.8.2/warp_beacon.egg-info → warp_beacon-2.8.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.8.2
+Version: 2.8.4
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

warp_beacon-2.8.4/warp_beacon/__version__.py

@@ -0,0 +1,2 @@
+__version__ = "2.8.4"
+

{warp_beacon-2.8.2 → warp_beacon-2.8.4}/warp_beacon/scheduler/instagram_human.py

@@ -151,7 +151,7 @@ class InstagramHuman(object):
 
 	def scroll_content(self, last_pk: int) -> None:
 		#timeline_initialized = False
-		if random.random() > 0.5:
+		if random.random() > 0.95:
 			#timeline_initialized = True
 			#self.browse_timeline()
 			logging.info("Starting to watch related reels with media_pk '%d'", last_pk)
@@ -159,7 +159,7 @@ class InstagramHuman(object):
 			self.operations_count += 1
 			self.watch_content(media)
 		
-		if random.random() > 0.7:
+		if random.random() > 0.95:
 			time.sleep(random.uniform(2, 20))
 			#if not timeline_initialized:
 			#	self.browse_timeline()

{warp_beacon-2.8.2 → warp_beacon-2.8.4}/warp_beacon/scraper/X/X.py

@@ -7,6 +7,7 @@ import requests
 import yt_dlp
 from playwright.sync_api import sync_playwright
 
+from warp_beacon.telegram.utils import Utils
 from warp_beacon.scraper.utils import ScraperUtils
 from warp_beacon.scraper.X.types import XMediaType
 from warp_beacon.jobs.types import JobType
@@ -82,24 +83,40 @@ class XScraper(XAbstract):
 					media_type = XMediaType.IMAGE
 				else:
 					raise
-
+		
+		images = []
 		if media_type == XMediaType.IMAGE:
 			job_type = JobType.IMAGE
 			images, post_text = self.download_images(url, timeout)
 			if images:
-				#if len(images) > 1:
-				#	job_type = JobType.COLLECTION
-				local_file = images[0]
-
-		if local_file:
+				if len(images) > 1:
+					job_type = JobType.COLLECTION
+				else:
+					local_file = images[0]
+
+		if job_type == JobType.COLLECTION:
+			chunks = []
+			for media_chunk in Utils.chunker(images, 10):
+				chunk = []
+				for media in media_chunk:
+					chunk.append({
+						"local_media_path": media,
+						"canonical_name": post_text,
+						"media_type": JobType.IMAGE
+					})
+				chunks.append(chunk)
 			res.append({
-				"local_media_path": local_file,
-				"performer": media_info.get("uploader", "Unknown"),
-				'progress_hooks': [self.dlp_on_progress],
-				#"thumb": thumbnail,
-				"canonical_name": post_text,
-				"media_type": job_type
+				"media_type": JobType.COLLECTION,
+				"items": chunks
 			})
+		else:
+			if local_file:
+				res.append({
+					"local_media_path": local_file,
+					"performer": media_info.get("uploader", "Unknown"),
+					"canonical_name": post_text,
+					"media_type": job_type
+				})
 
 		return res
 
@@ -197,6 +214,7 @@ class XScraper(XAbstract):
 					"username": parsed.username,
 					"password": parsed.password
 				}
+				logging.info("[X] build proxy: %s", proxy)
 
 		with sync_playwright() as p:
 			with p.chromium.launch(headless=True) as browser:
@@ -206,8 +224,9 @@ class XScraper(XAbstract):
 
 					#page.wait_for_timeout(3000)
 					page.wait_for_selector("img[src*='pbs.twimg.com/media']", timeout=(timeout*1000))
-					text_element = page.wait_for_selector('[data-testid="tweetText"]', timeout=(timeout*1000))
-					text = text_element.inner_text()
+					text_element = page.query_selector('[data-testid="tweetText"]')
+					if text_element:
+						text = text_element.inner_text()
 
 					image_elements = page.query_selector_all("img")
 					image_urls = []

{warp_beacon-2.8.2 → warp_beacon-2.8.4}/warp_beacon/scraper/X/abstract.py

@@ -24,7 +24,7 @@ from warp_beacon.telegram.types import ReportType
 
 class XAbstract(ScraperAbstract):
 	DOWNLOAD_DIR = "/tmp"
-	X_MAX_RETRIES_DEFAULT = 8
+	X_MAX_RETRIES_DEFAULT = 2
 	X_PAUSE_BEFORE_RETRY_DEFAULT = 3
 	X_TIMEOUT_DEFAULT = 15
 	X_TIMEOUT_INCREMENT_DEFAULT = 20

{warp_beacon-2.8.2 → warp_beacon-2.8.4/warp_beacon.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.8.2
+Version: 2.8.4
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

warp_beacon-2.8.2/warp_beacon/__version__.py

@@ -1,2 +0,0 @@
-__version__ = "2.8.2"
-