warp-beacon 2.8.2__py3-none-any.whl → 2.8.3__py3-none-any.whl

warp_beacon/__version__.py

@@ -1,2 +1,2 @@
-__version__ = "2.8.2"
+__version__ = "2.8.3"
 

warp_beacon/scheduler/instagram_human.py

@@ -151,7 +151,7 @@ class InstagramHuman(object):
 
 	def scroll_content(self, last_pk: int) -> None:
 		#timeline_initialized = False
-		if random.random() > 0.5:
+		if random.random() > 0.95:
 			#timeline_initialized = True
 			#self.browse_timeline()
 			logging.info("Starting to watch related reels with media_pk '%d'", last_pk)
@@ -159,7 +159,7 @@ class InstagramHuman(object):
 			self.operations_count += 1
 			self.watch_content(media)
 		
-		if random.random() > 0.7:
+		if random.random() > 0.95:
 			time.sleep(random.uniform(2, 20))
 			#if not timeline_initialized:
 			#	self.browse_timeline()

warp_beacon/scraper/X/X.py

@@ -7,6 +7,7 @@ import requests
 import yt_dlp
 from playwright.sync_api import sync_playwright
 
+from warp_beacon.telegram.utils import Utils
 from warp_beacon.scraper.utils import ScraperUtils
 from warp_beacon.scraper.X.types import XMediaType
 from warp_beacon.jobs.types import JobType
@@ -82,24 +83,40 @@ class XScraper(XAbstract):
 					media_type = XMediaType.IMAGE
 				else:
 					raise
-
+		
+		images = []
 		if media_type == XMediaType.IMAGE:
 			job_type = JobType.IMAGE
 			images, post_text = self.download_images(url, timeout)
 			if images:
-				#if len(images) > 1:
-				#	job_type = JobType.COLLECTION
-				local_file = images[0]
-
-		if local_file:
+				if len(images) > 1:
+					job_type = JobType.COLLECTION
+				else:
+					local_file = images[0]
+
+		if job_type == JobType.COLLECTION:
+			chunks = []
+			for media_chunk in Utils.chunker(images, 10):
+				chunk = []
+				for media in media_chunk:
+					chunk.append({
+						"local_media_path": media,
+						"canonical_name": post_text,
+						"media_type": JobType.IMAGE
+					})
+				chunks.append(chunk)
 			res.append({
-				"local_media_path": local_file,
-				"performer": media_info.get("uploader", "Unknown"),
-				'progress_hooks': [self.dlp_on_progress],
-				#"thumb": thumbnail,
-				"canonical_name": post_text,
-				"media_type": job_type
+				"media_type": JobType.COLLECTION,
+				"items": chunks
 			})
+		else:
+			if local_file:
+				res.append({
+					"local_media_path": local_file,
+					"performer": media_info.get("uploader", "Unknown"),
+					"canonical_name": post_text,
+					"media_type": job_type
+				})
 
 		return res
 
@@ -200,13 +217,13 @@ class XScraper(XAbstract):
 
 		with sync_playwright() as p:
 			with p.chromium.launch(headless=True) as browser:
-				with browser.new_context(proxy=proxy, ignore_https_errors=True) as context:
+				with browser.new_context(proxy=proxy) as context:
 					page = context.new_page()
 					page.goto(url, wait_until="networkidle", timeout=(timeout*1000))
 
 					#page.wait_for_timeout(3000)
 					page.wait_for_selector("img[src*='pbs.twimg.com/media']", timeout=(timeout*1000))
-					text_element = page.wait_for_selector('[data-testid="tweetText"]', timeout=(timeout*1000))
+					text_element = page.query_selector('[data-testid="tweetText"]')
 					text = text_element.inner_text()
 
 					image_elements = page.query_selector_all("img")

warp_beacon/scraper/X/abstract.py

@@ -24,7 +24,7 @@ from warp_beacon.telegram.types import ReportType
 
 class XAbstract(ScraperAbstract):
 	DOWNLOAD_DIR = "/tmp"
-	X_MAX_RETRIES_DEFAULT = 8
+	X_MAX_RETRIES_DEFAULT = 2
 	X_PAUSE_BEFORE_RETRY_DEFAULT = 3
 	X_TIMEOUT_DEFAULT = 15
 	X_TIMEOUT_INCREMENT_DEFAULT = 20

{warp_beacon-2.8.2.dist-info → warp_beacon-2.8.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.8.2
+Version: 2.8.3
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

{warp_beacon-2.8.2.dist-info → warp_beacon-2.8.3.dist-info}/RECORD

@@ -4,7 +4,7 @@ var/warp_beacon/accounts.json,sha256=OsXdncs6h88xrF_AP6_WDCK1waGBn9SR-uYdIeK37GM
 var/warp_beacon/placeholder.gif,sha256=cE5CGJVaop4Sx21zx6j4AyoHU0ncmvQuS2o6hJfEH88,6064
 var/warp_beacon/proxies.json,sha256=VnjlQDXumOEq72ZFjbh6IqHS1TEHqn8HPYAZqWCeSIA,95
 warp_beacon/__init__.py,sha256=_rThNODmz0nDp_n4mWo_HKaNFE5jk1_7cRhHyYaencI,163
-warp_beacon/__version__.py,sha256=_SZLUMceWXLlOgtC4XoBV-qybNYGrvvvBX_c96Do1fM,23
+warp_beacon/__version__.py,sha256=wFes6herhJvqX5NDKqYVfqjb67DmWdV9Toohi3aNFW4,23
 warp_beacon/warp_beacon.py,sha256=ADCR30uGXIsDrt9WoiI9Ghu2QtWs0qZIK6x3pQKM_B4,1109
 warp_beacon/yt_auth.py,sha256=GUTKqYr_tzDC-07Lx_ahWXSag8EyLxXBUnQbDBIkEmk,6022
 warp_beacon/compress/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -20,7 +20,7 @@ warp_beacon/mediainfo/audio.py,sha256=ous88kwQj4bDIChN5wnGil5LqTs0IQHH0d-nyrL0-Z
 warp_beacon/mediainfo/silencer.py,sha256=qxMuViOoVwUYb60uCVvqHiGrqByR1_4_rqMT-XdMkwc,1813
 warp_beacon/mediainfo/video.py,sha256=UBZrhTN5IDI-aYu6tsJEILo9nFkjHhkldGVFmvV7tEI,2480
 warp_beacon/scheduler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-warp_beacon/scheduler/instagram_human.py,sha256=WuS3KX9Gj-Rjjglxe7uVrm7WT9pygFnNS-G5-NpeJwY,12949
+warp_beacon/scheduler/instagram_human.py,sha256=lOytnonvqtB_8z6TVzrVJ1prqqgE4fktZZl-xpn7SOA,12951
 warp_beacon/scheduler/scheduler.py,sha256=Bf4sGXjX75Dox3q-yzUHhagtzUAj3hl5GzfnZya-_io,4995
 warp_beacon/scraper/__init__.py,sha256=o9-HQEf4yQVNtWuJN4NcLUovejiHhP_KkQ1Xf5EaQvU,20670
 warp_beacon/scraper/abstract.py,sha256=pWbaTu-gDZgi-iFjqMR_uGzPl5KLv-4gTdJ9w6cD4sk,3802
@@ -29,9 +29,9 @@ warp_beacon/scraper/exceptions.py,sha256=EKwoF0oH2xZWbNU-v8DOaWK5skKwa3s1yTIBdlc
 warp_beacon/scraper/fail_handler.py,sha256=5ODu4b8ndZWAcHIXrcUufsWFihetzNUoAi8IgAkreyQ,998
 warp_beacon/scraper/link_resolver.py,sha256=Rc9ZuMyOo3iPywDHwjngy-WRQ2SXhJwxcg-5ripx7tM,2447
 warp_beacon/scraper/utils.py,sha256=AOZmDki2Pbr84IG-j_wN2UghKCiWFVDYdx6HJl0JTBs,1258
-warp_beacon/scraper/X/X.py,sha256=iS_24pYTXKArcHy6XJVJgu6w9sicsHSgREyn6ZuB8YI,6846
+warp_beacon/scraper/X/X.py,sha256=dAw6kb0YYQ4-Y3Ya4r4F5XBgZ4KmjVIfXFNx96Xb138,7188
 warp_beacon/scraper/X/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-warp_beacon/scraper/X/abstract.py,sha256=Mmz69rQ0LTzQZkPCjrvdj2YbKfavXU8d1GujDv8VJmo,5475
+warp_beacon/scraper/X/abstract.py,sha256=pCzZPTCtn8pRbBx2SeuBUpMkEHqnOLtwLBAHYceL12Q,5475
 warp_beacon/scraper/X/types.py,sha256=i36Nu2cHpHCkvoeobBQC3B13Ke_N40tgCCApcm_FBFY,76
 warp_beacon/scraper/instagram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scraper/instagram/captcha.py,sha256=9UYziuqB3Tsat_ET6ex-cnZDbi6yCnsXHSpmE8MuUHk,4651
@@ -56,9 +56,9 @@ warp_beacon/telegram/progress_file_reader.py,sha256=e3equyNKlKs764AD-iE9QRsh3YDH
 warp_beacon/telegram/types.py,sha256=Kvdng6uCF1HRoqQgGW1ZYYPJoVuYkFb-LDvMBbW5Hjk,89
 warp_beacon/telegram/utils.py,sha256=zTF8VQfAWetBSjAPbmNe_Zi_LN5fAcWptJKjLaFNHaE,5073
 warp_beacon/uploader/__init__.py,sha256=1enK6qMWaTZEaK456JwaKOfvCvznHA8cjgceOsrF6Po,5732
-warp_beacon-2.8.2.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-warp_beacon-2.8.2.dist-info/METADATA,sha256=8gbtqun7YoGc-aolBKe1_nSz0q52KZEhNU3xErMZECg,23214
-warp_beacon-2.8.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-warp_beacon-2.8.2.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
-warp_beacon-2.8.2.dist-info/top_level.txt,sha256=RraB0PWGvRK2zPYkuICKNgStLG1C5s7rPHHJEHJbkgA,1510
-warp_beacon-2.8.2.dist-info/RECORD,,
+warp_beacon-2.8.3.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+warp_beacon-2.8.3.dist-info/METADATA,sha256=TeoJgsE_VnzVPdEzsKzoBvUWetvXgK9-9O2LAUPZ8Tw,23214
+warp_beacon-2.8.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+warp_beacon-2.8.3.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
+warp_beacon-2.8.3.dist-info/top_level.txt,sha256=RraB0PWGvRK2zPYkuICKNgStLG1C5s7rPHHJEHJbkgA,1510
+warp_beacon-2.8.3.dist-info/RECORD,,