warp-beacon 2.6.96__py3-none-any.whl → 2.6.97__py3-none-any.whl

warp_beacon/__version__.py

@@ -1,2 +1,2 @@
-__version__ = "2.6.96"
+__version__ = "2.6.97"
 

warp_beacon/scheduler/instagram_human.py

@@ -1,5 +1,6 @@
 import time
 import random
+from typing import Optional
 from datetime import datetime
 
 import logging
@@ -8,33 +9,109 @@ from instagrapi.types import UserShort
 from warp_beacon.scraper.instagram.instagram import InstagramScraper
 
 class InstagramHuman(object):
-	scrapler = None
 	default_profiles = ["nasa", "natgeo", "9gag", "spotify", "nba"]
-	operations_count = 0
 
 	def __init__(self, scrapler: InstagramScraper) -> None:
 		self.scrapler = scrapler
 		self.operations_count = 0
 
+	def browse_timeline(self) -> Optional[dict]:
+		feed = None
+		items = []
+		try:
+			reason = random.choice(["cold_start_fetch", "pull_to_refresh"])
+			feed = self.scrapler.cl.get_timeline_feed(reason=reason)
+			self.operations_count += 1
+			items = feed.get("feed_items", [])
+		except Exception as e:
+			logging.warning("Failed to get timeline feed!", exc_info=e)
+			return
+
+		seen = []
+		if items:
+			for item in items:
+				media = item.get("media_or_ad")
+				logging.info("Item content: %s", media)
+				if not media:
+					continue
+				media_id = media.get("id")
+				#user_id = media.get("user", {}).get("pk")
+
+				if media_id:
+					seen.append(str(media_id))
+					if random.random() < 0.5:
+						try:
+							#self.scrapler.cl.media_like(media_id)
+							self.scrapler.cl.media_comments(media_id)
+							self.operations_count += 1
+						except Exception as e:
+							logging.warning("Failed to see comments to media '%s'", media_id, exc_info=e)
+					self.random_pause()
+
+		if seen:
+			try:
+				self.scrapler.cl.media_seen(seen)
+				self.operations_count += 1
+			except Exception as e:
+				logging.warning("Failed to mark timeline feed as seen", exc_info=e)
+
+		return feed
+
+	def watch_stories(self) -> None:
+		logging.info("Simulating stories watch ...")
+		stories = None
+		try:
+			stories = self.scrapler.cl.user_stories_v1(self.scrapler.cl.user_id)
+			self.operations_count += 1
+		except Exception as e:
+			logging.warning("Failed to get user stories!", exc_info=e)
+
+		if not stories:
+			return
+
+		seen = []
+		for m in stories[:random.randint(1, len(stories))]:
+			try:
+				logging.info("Wathing story with pk '%s'", str(m.pk))
+				seen.append(str(m.id))
+				self.random_pause()
+			except Exception as e:
+				logging.warning("Exception while watching content", exc_info=e)
+
+		if seen:
+			try:
+				self.scrapler.cl.media_seen(seen)
+				self.operations_count += 1
+				logging.info("Marked %d stories as seen", len(seen))
+			except Exception as e:
+				logging.warning("Failed to mark seen watched watch stories!", exc_info=e)
+
 	def watch_content(self, media: list) -> None:
 		if not media:
 			return
+		seen = []
 		for m in media[:random.randint(1, len(media))]:
 			try:
 				logging.info("Wathing content with pk '%s'", str(m.pk))
 				content = self.scrapler.cl.media_info_v1(m.pk)
+				seen.append(str(content.id))
 				logging.info("Watched content with id '%s'", str(content.pk))
 				self.operations_count += 1
 				self.random_pause()
 			except Exception as e:
 				logging.warning("Exception while watching content")
 				logging.exception(e)
+		try:
+			self.scrapler.cl.media_seen(seen)
+		except Exception as e:
+			logging.warning("Failed to mark seen watched videos!", exc_info=e)
 
 	def scroll_content(self, last_pk: int) -> None:
 		timeline_initialized = False
 		if random.random() > 0.5:
 			timeline_initialized = True
-			self.scrapler.timeline_cursor = self.scrapler.download_hndlr(self.scrapler.cl.get_timeline_feed, reason="cold_start_fetch")
+			#self.scrapler.timeline_cursor = self.scrapler.download_hndlr(self.scrapler.cl.get_timeline_feed, reason="cold_start_fetch")
+			self.scrapler.timeline_cursor = self.browse_timeline()
 			logging.info("Starting to watch related reels with media_pk '%d'", last_pk)
 			media = self.scrapler.download_hndlr(self.scrapler.cl.reels, amount=random.randint(4, 10), last_media_pk=last_pk)
 			self.operations_count += 1
@@ -43,7 +120,8 @@ class InstagramHuman(object):
 		if random.random() > 0.7:
 			time.sleep(random.uniform(2, 20))
 			if not timeline_initialized:
-				self.scrapler.timeline_cursor = self.scrapler.download_hndlr(self.scrapler.cl.get_timeline_feed, reason="cold_start_fetch")
+				#self.scrapler.timeline_cursor = self.scrapler.download_hndlr(self.scrapler.cl.get_timeline_feed, reason="cold_start_fetch")
+				self.scrapler.timeline_cursor = self.browse_timeline()
 			logging.info("Starting to explore reels with media_pk '%d'", last_pk)
 			media = self.scrapler.download_hndlr(self.scrapler.cl.explore_reels, amount=random.randint(4, 10), last_media_pk=last_pk)
 			self.operations_count += 1
@@ -65,15 +143,13 @@ class InstagramHuman(object):
 	def morning_routine(self) -> None:
 		try:
 			logging.info("Starting morning activity simulation")
-			self.scrapler.timeline_cursor = self.scrapler.download_hndlr(self.scrapler.cl.get_timeline_feed, "pull_to_refresh", self.scrapler.timeline_cursor.get("next_max_id"))
-			self.operations_count += 1
+			#self.scrapler.timeline_cursor = self.scrapler.download_hndlr(self.scrapler.cl.get_timeline_feed, "pull_to_refresh", self.scrapler.timeline_cursor.get("next_max_id"))
+			#self.operations_count += 1
+			self.scrapler.timeline_cursor = self.browse_timeline()
 			time.sleep(random.uniform(3, 7))
 			if random.random() > 0.5:
-				logging.info("Checking direct ...")
-				self.scrapler.download_hndlr(self.scrapler.cl.direct_active_presence)
-				self.operations_count += 1
-				time.sleep(random.uniform(2, 5))
-			if random.random() > 0.3:
+				self.check_direct()
+			if random.random() > 0.6:
 				self.scrapler.download_hndlr(self.scrapler.cl.notification_like_and_comment_on_photo_user_tagged, "everyone")
 				self.operations_count += 1
 				self.random_pause()
@@ -82,6 +158,8 @@ class InstagramHuman(object):
 				self.scrapler.download_hndlr(self.scrapler.cl.get_reels_tray_feed, "pull_to_refresh")
 				self.operations_count += 1
 				self.random_pause()
+			if random.random() > 0.4:
+				self.watch_stories()
 			if random.random() > 0.8:
 				self.profile_view()
 		except Exception as e:
@@ -91,13 +169,19 @@ class InstagramHuman(object):
 	def daytime_routine(self) -> None:
 		try:
 			logging.info("Starting day fast check activity simulation")
-			self.scrapler.download_hndlr(self.scrapler.cl.get_timeline_feed, "pull_to_refresh")
-			self.operations_count += 1
+			#self.scrapler.download_hndlr(self.scrapler.cl.get_timeline_feed, "pull_to_refresh")
+			#self.operations_count += 1
+			self.browse_timeline()
 			time.sleep(random.uniform(2, 5))
 			if random.random() > 0.5:
 				self.scrapler.download_hndlr(self.scrapler.cl.get_reels_tray_feed, "pull_to_refresh")
 				self.operations_count += 1
 				self.random_pause()
+
+			if random.random() > 0.4:
+				self.watch_stories()
+				self.random_pause()
+
 			if random.random() > 0.4:
 				logging.info("Watching reels ...")
 				reels = self.scrapler.download_hndlr(self.scrapler.cl.reels, amount=random.randint(4, 15))
@@ -111,21 +195,18 @@ class InstagramHuman(object):
 	def evening_routine(self) -> None:
 		try:
 			logging.info("Starting evening active user simulation")
-			self.scrapler.download_hndlr(self.scrapler.cl.get_timeline_feed, "pull_to_refresh")
-			self.operations_count += 1
+			self.browse_timeline()
 			time.sleep(random.uniform(2, 5))
-			self.scrapler.download_hndlr(self.scrapler.cl.get_reels_tray_feed, "pull_to_refresh")
-			self.operations_count += 1
-			time.sleep(random.uniform(2, 5))
-			if random.random() > 0.5:
-				self.scrapler.download_hndlr(self.scrapler.cl.direct_active_presence)
-				self.operations_count += 1
-				time.sleep(random.uniform(2, 5))
 			if random.random() > 0.5:
+				self.check_direct()
+			if random.random() > 0.6:
 				logging.info("Checking notifications, tags ...")
 				self.scrapler.download_hndlr(self.scrapler.cl.notification_like_and_comment_on_photo_user_tagged, "everyone")
 				self.operations_count += 1
 				self.random_pause()
+			if random.random() > 0.4:
+				self.watch_stories()
+				self.random_pause()
 			if random.random() > 0.4:
 				logging.info("Watching reels ...")
 				reels = self.scrapler.download_hndlr(self.scrapler.cl.reels, amount=random.randint(4, 10))
@@ -144,9 +225,10 @@ class InstagramHuman(object):
 		try:
 			logging.info("Starting night activity simulation")
 			if random.random() > 0.7:
-				self.scrapler.download_hndlr(self.scrapler.cl.direct_active_presence)
-				self.operations_count += 1
-				self.random_pause(short=True)
+				self.check_direct()
+			if random.random() > 0.7:
+				self.watch_stories()
+				self.random_pause()
 			if random.random() > 0.5:
 				logging.info("Watching reels ...")
 				reels = self.scrapler.download_hndlr(self.scrapler.cl.reels, amount=random.randint(4, 15))
@@ -162,6 +244,30 @@ class InstagramHuman(object):
 		logging.info("Pause for '%.2f' sec ...", round(pause, 2))
 		time.sleep(pause)
 
+	def check_direct(self) -> None:
+		logging.info("Checking direct ...")
+		self.scrapler.download_hndlr(self.scrapler.cl.direct_active_presence)
+		self.operations_count += 1
+		self.random_pause()
+		threads = self.scrapler.download_hndlr(self.scrapler.cl.direct_threads, amount=random.randint(3, 7))
+		self.operations_count += 1
+		for thread in threads:
+			try:
+				messages = self.scrapler.cl.direct_messages(thread.id, amount=random.randint(5, 15))
+				self.operations_count += 1
+				if not messages:
+					continue
+				msg_sample = random.sample(messages, k=random.randint(1, min(len(messages), 5)))
+				for msg in msg_sample:
+					if random.random() < 0.85:
+						self.scrapler.cl.direct_message_seen(msg.thread_id, msg.id)
+						self.operations_count += 1
+					self.random_pause()
+				self.random_pause()
+			except Exception as e:
+				logging.warning("Failed to read thread %s", thread.id)
+				logging.exception(e)
+
 	def profile_view(self) -> None:
 		try:
 			logging.info("profile_view ...")
@@ -190,11 +296,10 @@ class InstagramHuman(object):
 				self.operations_count += 1
 				self.random_pause()
 			
+			#self.scrapler.cl.explore_page_media_info
+
 			if random.random() > 0.5:
-				logging.info("Checking direct ...")
-				self.scrapler.download_hndlr(self.scrapler.cl.direct_active_presence)
-				self.operations_count += 1
-				self.random_pause()
+				self.check_direct()
 
 			if random.random() > 0.3:
 				logging.info("Checking notifications, tags ...")

warp_beacon/scraper/instagram/instagram.py

@@ -282,7 +282,7 @@ class InstagramScraper(ScraperAbstract):
 			if len(st_parts) > 1:
 				effective_story_id = st_parts[0]
 		logging.info("Effective story id is '%s'", effective_story_id)
-		effective_url = "https://www.instagram.com/stories/%s/%s/" % (story_info.user.username, effective_story_id)
+		effective_url = f"https://www.instagram.com/stories/{story_info.user.username}/{effective_story_id}/"
 		if story_info.media_type == 1: # photo
 			path = str(self.download_hndlr(self.cl.story_download_by_url, url=story_info.thumbnail_url, folder='/tmp'))
 			path_lowered = path.lower()
@@ -357,6 +357,10 @@ class InstagramScraper(ScraperAbstract):
 						res.append(self.download_photo(url=media_info.thumbnail_url, media_info=media_info))
 					elif media_info.media_type == 8: # Album
 						res.append(self.download_album(media_info=media_info))
+					try:
+						self.cl.media_seen([media_info.id])
+					except Exception as e:
+						logging.warning("Failed to mark seen with id = '%s'", media_info.id, exc_info=e)
 				elif scrap_type == "story":
 					story_info = self.cl.story_info(media_id)
 					logging.info("media_type for story is '%d'", story_info.media_type)

warp_beacon/scraper/instagram/wb_instagrapi.py

@@ -1,11 +1,17 @@
 import logging
 from typing import Callable
+from copy import deepcopy
 from urllib.parse import urlparse, parse_qs, urlencode, urlunparse
 from pathlib import Path
 import requests
 
 from instagrapi import Client
-from instagrapi.exceptions import VideoNotDownload
+from instagrapi.types import Media
+from instagrapi.exceptions import (
+	ClientError,
+	ClientLoginRequired,
+	VideoNotDownload
+)
 
 from warp_beacon.scraper.utils import ScraperUtils
 
@@ -160,4 +166,28 @@ class WBClient(Client):
 							)
 						except Exception as e:
 							logging.warning("Progress callback raised an exception!", exc_info=e)
-		return path.resolve()
+		return path.resolve()
+	
+	def media_info(self, media_pk: str, use_cache: bool = True) -> Media:
+		"""
+		Get Media Information from PK
+
+		Parameters
+		----------
+		media_pk: str
+			Unique identifier of the media
+		use_cache: bool, optional
+			Whether or not to use information from cache, default value is True
+
+		Returns
+		-------
+		Media
+			An object of Media type
+		"""
+		media_pk = self.media_pk(media_pk)
+		if not use_cache or media_pk not in self._medias_cache:
+			media = self.media_info_v1(media_pk)
+			self._medias_cache[media_pk] = media
+		return deepcopy(
+			self._medias_cache[media_pk]
+		)  # return copy of cache (dict changes protection)

{warp_beacon-2.6.96.dist-info → warp_beacon-2.6.97.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.6.96
+Version: 2.6.97
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

{warp_beacon-2.6.96.dist-info → warp_beacon-2.6.97.dist-info}/RECORD

@@ -4,7 +4,7 @@ var/warp_beacon/accounts.json,sha256=OsXdncs6h88xrF_AP6_WDCK1waGBn9SR-uYdIeK37GM
 var/warp_beacon/placeholder.gif,sha256=cE5CGJVaop4Sx21zx6j4AyoHU0ncmvQuS2o6hJfEH88,6064
 var/warp_beacon/proxies.json,sha256=VnjlQDXumOEq72ZFjbh6IqHS1TEHqn8HPYAZqWCeSIA,95
 warp_beacon/__init__.py,sha256=_rThNODmz0nDp_n4mWo_HKaNFE5jk1_7cRhHyYaencI,163
-warp_beacon/__version__.py,sha256=EAP5WKHPDylNbR_6ZQze_0fZlcnJeLdRc2t63qJe6RM,24
+warp_beacon/__version__.py,sha256=TS8oRh0prYIEOMSCk2cuClcx0weLzgMtkvKhmNkLh6k,24
 warp_beacon/warp_beacon.py,sha256=ADCR30uGXIsDrt9WoiI9Ghu2QtWs0qZIK6x3pQKM_B4,1109
 warp_beacon/yt_auth.py,sha256=GUTKqYr_tzDC-07Lx_ahWXSag8EyLxXBUnQbDBIkEmk,6022
 warp_beacon/compress/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -20,7 +20,7 @@ warp_beacon/mediainfo/audio.py,sha256=ous88kwQj4bDIChN5wnGil5LqTs0IQHH0d-nyrL0-Z
 warp_beacon/mediainfo/silencer.py,sha256=qxMuViOoVwUYb60uCVvqHiGrqByR1_4_rqMT-XdMkwc,1813
 warp_beacon/mediainfo/video.py,sha256=UBZrhTN5IDI-aYu6tsJEILo9nFkjHhkldGVFmvV7tEI,2480
 warp_beacon/scheduler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-warp_beacon/scheduler/instagram_human.py,sha256=nZh76wSwogvG015z_cfwFY5xGlEfSRauRtDf_Aej1RI,8293
+warp_beacon/scheduler/instagram_human.py,sha256=BY2qtJE1QTr5oz1-8xC4wkqn9lU_tgy91OPM7QeVbBU,11169
 warp_beacon/scheduler/scheduler.py,sha256=Bf4sGXjX75Dox3q-yzUHhagtzUAj3hl5GzfnZya-_io,4995
 warp_beacon/scraper/__init__.py,sha256=k3M0X_m5f7b_DbBB3Ahk62ewEYr5AkqJL0PJXf0G4mI,20140
 warp_beacon/scraper/abstract.py,sha256=pWbaTu-gDZgi-iFjqMR_uGzPl5KLv-4gTdJ9w6cD4sk,3802
@@ -31,8 +31,8 @@ warp_beacon/scraper/link_resolver.py,sha256=Rc9ZuMyOo3iPywDHwjngy-WRQ2SXhJwxcg-5
 warp_beacon/scraper/utils.py,sha256=Kk_lDmdJiCSaHmOV80OFK05O1wRL2H0agH98JiO8dyg,1268
 warp_beacon/scraper/instagram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scraper/instagram/captcha.py,sha256=9UYziuqB3Tsat_ET6ex-cnZDbi6yCnsXHSpmE8MuUHk,4651
-warp_beacon/scraper/instagram/instagram.py,sha256=WWMhgHcJn_cZ-nHwpcSYubObHf5lT5wK57Jad88e8_Q,17702
-warp_beacon/scraper/instagram/wb_instagrapi.py,sha256=OECHz1eK6J5bfw2H84CdywwxBNegkNb_UhDr0MtNtZ4,5237
+warp_beacon/scraper/instagram/instagram.py,sha256=xDjKJa_TdXC-Xk8ji5oGKQmo7LrpRNSshLcuKuFJrCM,17863
+warp_beacon/scraper/instagram/wb_instagrapi.py,sha256=M5NCtLwdUvByjmDBZMWljgB275R0LSBFblsGpapluD0,5968
 warp_beacon/scraper/youtube/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scraper/youtube/abstract.py,sha256=AnKWFhfUvPikQ4tZflb0wvnZnkb8ums6eTZNx5iCbdM,15233
 warp_beacon/scraper/youtube/music.py,sha256=5AeSBQyUgVCJT2hoBCV2WvlyuV9US09SYJhmBG_P9F8,2755
@@ -52,9 +52,9 @@ warp_beacon/telegram/progress_file_reader.py,sha256=e3equyNKlKs764AD-iE9QRsh3YDH
 warp_beacon/telegram/types.py,sha256=Kvdng6uCF1HRoqQgGW1ZYYPJoVuYkFb-LDvMBbW5Hjk,89
 warp_beacon/telegram/utils.py,sha256=1Lq67aRylVJzbwSyvAgjPAGjJZFATkICvAj3TJGuJiM,4635
 warp_beacon/uploader/__init__.py,sha256=j3qcuKhpchseZLGzSsSiogqe6WdMbkK8d3I-ConhNRs,5687
-warp_beacon-2.6.96.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-warp_beacon-2.6.96.dist-info/METADATA,sha256=q2LA7qNaB271J8U5YYaRtRVDe_4gv2HrCDMJRRKW1V0,23215
-warp_beacon-2.6.96.dist-info/WHEEL,sha256=A8Eltl-h0W-qZDVezsLjjslosEH_pdYC2lQ0JcbgCzs,91
-warp_beacon-2.6.96.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
-warp_beacon-2.6.96.dist-info/top_level.txt,sha256=5YQRN46STNg81V_3jdzZ6bftkMxhe1hTPSFvJugDu84,1405
-warp_beacon-2.6.96.dist-info/RECORD,,
+warp_beacon-2.6.97.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+warp_beacon-2.6.97.dist-info/METADATA,sha256=9H9zq1a_2KPH5HBpGZnMi4QHv38xQhZZhx0zAHA1SfE,23215
+warp_beacon-2.6.97.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
+warp_beacon-2.6.97.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
+warp_beacon-2.6.97.dist-info/top_level.txt,sha256=5YQRN46STNg81V_3jdzZ6bftkMxhe1hTPSFvJugDu84,1405
+warp_beacon-2.6.97.dist-info/RECORD,,

{warp_beacon-2.6.96.dist-info → warp_beacon-2.6.97.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.7.0)
+Generator: setuptools (80.7.1)
 Root-Is-Purelib: true
 Tag: py3-none-any